UserProfileExample.java

/**
 * Copyright 2013 Cloudera Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.cloudera.cdk.data.hbase.avro.example;

import java.io.IOException;
import java.util.HashMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTablePool;

import com.cloudera.cdk.data.PartitionKey;
import com.cloudera.cdk.data.hbase.avro.SpecificAvroDao;
import com.cloudera.cdk.data.hbase.impl.Dao;
import com.cloudera.cdk.data.hbase.impl.EntityScanner;
import com.cloudera.cdk.data.hbase.impl.SchemaManager;
import com.cloudera.cdk.data.hbase.manager.DefaultSchemaManager;
import com.cloudera.cdk.data.hbase.tool.SchemaTool;

/**
 * This is an example that demonstrates basic CDK HBase functionality. It uses a
 * fictional "user profile" use case that needs basic user profile data (first
 * name, last name, etc...) and user action log data (login, profile changed,
 * etc...) persisted in a wide user table.
 * 
 * By using a wide user table, we are able to atomically fetch user profile and
 * action data with a single HBase request, as well as atomically update both
 * profile and action log data.
 * 
 * The basic CDK HBase functionality demonstrated includes basic scanning,
 * putting, composite DAOs, and optimistic concurrency control (OCC).
 */
public class UserProfileExample {

  /**
   * The user profile DAO
   */
  private final Dao<UserProfileModel> userProfileDao;

  /**
   * The user actions DAO. User actions are stored in the same table along side
   * the user profiles.
   */
  private final Dao<UserActionsModel> userActionsDao;

  /**
   * A composite dao that encapsulates the two entity types that can be stored
   * in the user_profile table (UserProfileModel and UserActionsModel).
   * UserProfileActionsModel is the compsite type this dao returns.
   */
  private final Dao<UserProfileActionsModel> userProfileActionsDao;

  /**
   * The constructor will start by registering the schemas with the meta store
   * table in HBase, and create the required tables to run.
   */
  public UserProfileExample() {
    Configuration conf = HBaseConfiguration.create();
    HTablePool pool = new HTablePool(conf, 10);
    SchemaManager schemaManager = new DefaultSchemaManager(pool);

    registerSchemas(conf, schemaManager);

    userProfileDao = new SpecificAvroDao<UserProfileModel>(pool,
        "cdk_example_user_profiles", "UserProfileModel", schemaManager);
    userActionsDao = new SpecificAvroDao<UserActionsModel>(pool,
        "cdk_example_user_profiles", "UserActionsModel", schemaManager);
    userProfileActionsDao = SpecificAvroDao.buildCompositeDaoWithEntityManager(
        pool, "cdk_example_user_profiles", UserProfileActionsModel.class,
        schemaManager);
  }

  /**
   * Print all user profiles.
   * 
   * This method demonstrates how to open a scanner that will scan the entire
   * table. It has no start or stop keys specified.
   */
  public void printUserProfies() {
    EntityScanner<UserProfileModel> scanner = userProfileDao.getScanner();
    scanner.open();
    try {
      for (UserProfileModel entity : scanner) {
        System.out.println(entity.toString());
      }
    } finally {
      // scanners need to be closed.
      scanner.close();
    }
  }

  /**
   * Print the user profiles and actions for all users with the provided last
   * name
   * 
   * This method demonstrates how to open a scanner with a start key. It's using
   * the composite dao, so the records it returns will be a composite of both
   * the profile model and actions model.
   * 
   * @param lastName
   *          The last name of users to scan.
   */
  public void printUserProfileActionsForLastName(String lastName) {
    // Create a partial key that will allow us to start the scanner from the
    // first user record that has last name equal to the one provided.
    PartitionKey startKey = userProfileActionsDao.getPartitionStrategy()
        .partitionKey("lastName");

    // Get the scanner with the start key. Null for stopKey in the getScanner
    // method indicates that the scanner will scan to the end of the table. Our
    // loop will break out when it encounters a record without the last name.

    EntityScanner<UserProfileActionsModel> scanner = userProfileActionsDao
        .getScanner(startKey, null);
    scanner.open();
    try {
      // scan until we find a last name not equal to the one provided
      for (UserProfileActionsModel entity : scanner) {
        if (!entity.getUserProfileModel().getLastName().equals(lastName)) {
          // last name of row different, break out of the scan.
          break;
        }
        System.out.println(entity.toString());
      }
    } finally {
      // scanners need to be closed.
      scanner.close();
    }
  }

  /**
   * Create a fresh new user record.
   * 
   * This method demonstrates creating both a UserProfileModel and a
   * UserActionsModel atomically in a single row. When creating a user profile,
   * we add the "created" action to the actions map. This shows how we can use
   * the CompositeDao to accomplish this.
   * 
   * @param firstName
   *          The first name of the user we are creating
   * @param lastName
   *          The last name of the user we are creating
   * @param married
   *          True if this person is married. Otherwise false.
   */
  public void create(String firstName, String lastName, boolean married) {
    long ts = System.currentTimeMillis();

    UserProfileModel profileModel = UserProfileModel.newBuilder()
        .setFirstName(firstName).setLastName(lastName).setMarried(married)
        .setCreated(ts).build();
    UserActionsModel actionsModel = UserActionsModel.newBuilder()
        .setFirstName(firstName).setLastName(lastName)
        .setActions(new HashMap<String, String>()).build();
    actionsModel.getActions().put("profile_created", Long.toString(ts));

    UserProfileActionsModel profileActionsModel = UserProfileActionsModel
        .newBuilder().setUserProfileModel(profileModel)
        .setUserActionsModel(actionsModel).build();

    if (!userProfileActionsDao.put(profileActionsModel)) {
      // If put returns false, a user already existed at this row
      System.out
          .println("Creating a new user profile failed due to a write conflict.");
    }
  }

  /**
   * Update the married status of a new user record.
   * 
   * This method demonstrates updating both a UserProfileModel and a
   * UserActionsModel with a single HBase request using the composite dao. It
   * performs a get/update/put operation, which is protected by the
   * check_conflict field on UserProfileModel from colliding with another
   * get/update/put operation.
   * 
   * @param firstName
   *          The first name of the user we are updating
   * @param lastName
   *          The last name of the user we are updating
   * @param married
   *          True if this person is married. Otherwise false.
   */
  public void updateUserProfile(String firstName, String lastName,
      boolean married) {
    // Get the timestamp we'll use to set the value of the profile_updated
    // action.
    long ts = System.currentTimeMillis();

    // Construct the key we'll use to fetch the user.
    PartitionKey key = userProfileActionsDao.getPartitionStrategy()
        .partitionKey(lastName, firstName);

    // Get the profile and actions entity from the composite dao.
    UserProfileActionsModel profileActionsModel = userProfileActionsDao
        .get(key);

    // Updating the married status is hairy since our avro compiler isn't setup
    // to compile setters for fields. We have to construct a clone through the
    // builder.
    UserProfileActionsModel updatedProfileActionsModel = UserProfileActionsModel
        .newBuilder(profileActionsModel)
        .setUserProfileModel(
            UserProfileModel
                .newBuilder(profileActionsModel.getUserProfileModel())
                .setMarried(married).build()).build();
    // Since maps are mutable, we can update the actions map without having to
    // go through the builder like above.
    updatedProfileActionsModel.getUserActionsModel().getActions()
        .put("profile_updated", Long.toString(ts));

    if (!userProfileActionsDao.put(updatedProfileActionsModel)) {
      // If put returns false, a write conflict occurred where someone else
      // updated the row between the times we did the get and put.
      System.out
          .println("Updating the user profile failed due to a write conflict");
    }
  }

  /**
   * Add an action to the user profile.
   * 
   * This method demonstrates how we can use a keyAsColumn map field (the
   * actions field of the UserActionsModel) to add values to the map without
   * having to do a get/update/put operation. When doing the put, it won't
   * remove columns that exist in the row that aren't in the new map we are
   * putting. It will just add the additional columns we are now putting to the
   * row.
   * 
   * @param firstName
   *          The first name of the user we are updating
   * @param lastName
   *          The last name of the user we are updating
   * @param actionType
   *          A string representing the action type which is the key of the map
   * @param actionValue
   *          A string representing the action value.
   */
  public void addAction(String firstName, String lastName, String actionType,
      String actionValue) {
    // Create a new UserActionsModel, and add a new actions map to it with a
    // single action value. Even if one exists in this row, since it has a lone
    // keyAsColumn field, it won't remove any actions that already exist in the
    // actions column family.
    UserActionsModel actionsModel = UserActionsModel.newBuilder()
        .setLastName(lastName).setFirstName(firstName)
        .setActions(new HashMap<String, String>()).build();
    actionsModel.getActions().put(actionType, actionValue);

    // Perform the put.
    userActionsDao.put(actionsModel);
  }

  /**
   * Uses SchemaTool to register the required schemas and create the required
   * tables.
   * 
   * @param conf
   *          The HBaseConfiguration.
   * @param schemaManager
   *          The schema manager SchemaTool needs to create the schemas.
   */
  private void registerSchemas(Configuration conf, SchemaManager schemaManager) {
    HBaseAdmin admin;
    try {
      // Construct an HBaseAdmin object (required by schema tool), and delete it
      // if it exists so we start fresh.
      admin = new HBaseAdmin(conf);
      if (admin.tableExists("cdk_example_user_profiles")) {
        admin.disableTable("cdk_example_user_profiles");
        admin.deleteTable("cdk_example_user_profiles");
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    // Use the SchemaTool to create the schemas that are in the example-models
    // directory, and create the table and column families required by those
    // schemas.
    SchemaTool tool = new SchemaTool(admin, schemaManager);
    tool.createOrMigrateSchemaDirectory("classpath:example-models", true);
  }

  /**
   * The main driver method. Doesn't require any arguments.
   * 
   * @param args
   */
  public static void main(String[] args) {
    UserProfileExample example = new UserProfileExample();

    // Let's create some user profiles
    example.create("John", "Doe", true);
    example.create("Jane", "Doe", false);
    example.create("Foo", "Bar", false);

    // Now print those user profiles. This doesn't include actions
    example.printUserProfies();

    // Now we'll add some user actions to each user
    example.addAction("Jane", "Doe", "last_login", "2013-07-30 00:00:00");
    example.addAction("Jane", "Doe", "ad_click", "example.com_ad_id");
    example.addAction("Foo", "Bar", "last_login", "2013-07-30 00:00:00");

    // Print the user profiles and actions for the Does. This will include the
    // above actions, as well as a profile_created action set when creating the
    // user profiles.
    example.printUserProfileActionsForLastName("Doe");

    // Update Jane to a married status.
    example.updateUserProfile("Jane", "Doe", true);

    // Reprint the user profiles and actions. Jane should now have married true,
    // as well as a new profile_updated timestamp.
    example.printUserProfileActionsForLastName("Doe");
  }
}