RealtimeQuickStart.java example

Explorer
pinot-master
/**
 * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.linkedin.pinot.tools;

import java.io.File;
import org.apache.commons.io.FileUtils;
import org.json.JSONException;
import com.google.common.collect.Lists;
import com.linkedin.pinot.common.utils.KafkaStarterUtils;
import com.linkedin.pinot.common.utils.ZkStarter;
import com.linkedin.pinot.tools.Quickstart.color;
import com.linkedin.pinot.tools.admin.command.QuickstartRunner;
import com.linkedin.pinot.tools.streams.MeetupRsvpStream;
import static com.linkedin.pinot.tools.Quickstart.prettyprintResponse;
import static com.linkedin.pinot.tools.Quickstart.printStatus;
import kafka.server.KafkaServerStartable;


public class RealtimeQuickStart {
  private File _quickStartDataDir;
  private ZkStarter.ZookeeperInstance _zookeeperInstance;

  public void execute() throws JSONException, Exception {
    _quickStartDataDir = new File("quickStartData" + System.currentTimeMillis());
    String quickStartDataDirName = _quickStartDataDir.getName();

    if (!_quickStartDataDir.exists()) {
      _quickStartDataDir.mkdir();
    }

    File schema = new File(quickStartDataDirName + "/rsvp_pinot_schema.json");
    File tableCreate = new File(quickStartDataDirName + "/rsvp_create_table_request.json");

    FileUtils.copyURLToFile(RealtimeQuickStart.class.getClassLoader().getResource("sample_data/rsvp_pinot_schema.json"),
        schema);

    FileUtils.copyURLToFile(
        RealtimeQuickStart.class.getClassLoader().getResource("sample_data/rsvp_create_table_request.json"),
        tableCreate);

    printStatus(color.CYAN, "Starting Kafka");

    _zookeeperInstance = ZkStarter.startLocalZkServer();

    final KafkaServerStartable kafkaStarter =
        KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID,
            KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration());

    KafkaStarterUtils.createTopic("meetupRSVPEvents", KafkaStarterUtils.DEFAULT_ZK_STR, 10);

    File tempDir = new File("/tmp/" + String.valueOf(System.currentTimeMillis()));

    QuickstartTableRequest request = new QuickstartTableRequest("meetupRsvp", schema, tableCreate);
    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);

    runner.startAll();

    printStatus(color.CYAN, "Starting controller, server and broker");

    runner.addSchema();
    runner.addTable();
    printStatus(color.CYAN, "Added schema and table");

    printStatus(color.YELLOW, "Realtime quickstart setup complete");

    final MeetupRsvpStream meetupRSVPProvider = new MeetupRsvpStream(schema);
    meetupRSVPProvider.run();
    printStatus(color.CYAN, "Starting meetup data stream and publishing to kafka");

    // lets wait for a few events to get populated
    Thread.sleep(5000);

    String q1 = "select count(*) from meetupRsvp limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q1);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q1)));
    printStatus(color.GREEN, "***************************************************");

    String q2 = "select sum(rsvp_count) from meetupRsvp group by group_city top 10 limit 0";
    printStatus(color.YELLOW, "Top 10 cities with the most rsvp");
    printStatus(color.CYAN, "Query : " + q2);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q2)));
    printStatus(color.GREEN, "***************************************************");

    String q3 = "select * from meetupRsvp order by mtime limit 10";
    printStatus(color.YELLOW, "Show 10 most recent rsvps");
    printStatus(color.CYAN, "Query : " + q3);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q3)));
    printStatus(color.GREEN, "***************************************************");

    String q4 = "select sum(rsvp_count) from meetupRsvp group by event_name top 10 limit 0";
    printStatus(color.YELLOW, "Show top 10 rsvp'ed events");
    printStatus(color.CYAN, "Query : " + q4);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q4)));
    printStatus(color.GREEN, "***************************************************");

    String q5 = "select count(*) from meetupRsvp limit 0";
    printStatus(color.YELLOW, "Total number of documents in the table");
    printStatus(color.CYAN, "Query : " + q5);
    printStatus(color.YELLOW, prettyprintResponse(runner.runQuery(q5)));
    printStatus(color.GREEN, "***************************************************");

    printStatus(color.GREEN, "you can always go to http://localhost:9000/query/ to play around in the query console");

    Runtime.getRuntime().addShutdownHook(new Thread() {
      @Override
      public void run() {
        try {
          printStatus(color.GREEN, "***** shutting down realtime quickstart *****");
          meetupRSVPProvider.stopPublishing();
          FileUtils.deleteDirectory(_quickStartDataDir);
          runner.stop();
          runner.clean();
          KafkaStarterUtils.stopServer(kafkaStarter);
          ZkStarter.stopLocalZkServer(_zookeeperInstance);
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
    });

    long st = System.currentTimeMillis();
    while (true) {
      if (System.currentTimeMillis() - st >= (60 * 60) * 1000) {
        break;
      }
    }

    printStatus(color.YELLOW, "running since an hour, stopping now");

  }

  public static void main(String[] args) {
    RealtimeQuickStart rst = new RealtimeQuickStart();
    try {
      rst.execute();
    } catch (Exception e) {
      System.out.println(e.getMessage());
      e.printStackTrace();
    }
  }

}