/*************************************************************************** * Copyright (C) 2011 by H-Store Project * * Brown University * * Massachusetts Institute of Technology * * Yale University * * * * http://hstore.cs.brown.edu/ * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR * * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * * OTHER DEALINGS IN THE SOFTWARE. * ***************************************************************************/ package edu.brown.benchmark.seats; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.collections15.map.ListOrderedMap; import org.apache.log4j.Logger; import org.voltdb.CatalogContext; import org.voltdb.VoltTable; import org.voltdb.VoltType; import org.voltdb.catalog.Column; import org.voltdb.catalog.Table; import org.voltdb.client.Client; import org.voltdb.client.ClientResponse; import org.voltdb.types.TimestampType; import edu.brown.api.BenchmarkComponent; import edu.brown.benchmark.seats.procedures.LoadConfig; import edu.brown.catalog.CatalogUtil; import edu.brown.hstore.Hstoreservice.Status; import edu.brown.logging.LoggerUtil; import edu.brown.logging.LoggerUtil.LoggerBoolean; import edu.brown.rand.AbstractRandomGenerator; import edu.brown.rand.RandomDistribution.FlatHistogram; import edu.brown.statistics.Histogram; import edu.brown.statistics.ObjectHistogram; import edu.brown.utils.JSONUtil; import edu.brown.utils.StringUtil; public class SEATSProfile { private static final Logger LOG = Logger.getLogger(SEATSProfile.class); private static final LoggerBoolean debug = new LoggerBoolean(); private static final LoggerBoolean trace = new LoggerBoolean(); static { LoggerUtil.attachObserver(LOG, debug, trace); } // ---------------------------------------------------------------- // PERSISTENT DATA MEMBERS // ---------------------------------------------------------------- /** * Data Scale Factor */ protected double scale_factor; /** * For each airport id, store the last id of the customer that uses this airport * as their local airport. The customer ids will be stored as follows in the dbms: * <16-bit AirportId><48-bit CustomerId> */ protected final ObjectHistogram<Long> airport_max_customer_id = new ObjectHistogram<Long>(); /** * The date when flights total data set begins */ protected TimestampType flight_start_date = new TimestampType(); /** * The date for when the flights are considered upcoming and are eligible for reservations */ protected TimestampType flight_upcoming_date; /** * The number of days in the past that our flight data set includes. */ protected long flight_past_days; /** * The number of days in the future (from the flight_upcoming_date) that our flight data set includes */ protected long flight_future_days; /** * The number of FLIGHT rows created. */ protected long num_flights = 0l; /** * The number of CUSTOMER rows */ protected long num_customers = 0l; /** * The number of RESERVATION rows */ protected long num_reservations = 0l; /** * TODO **/ protected final Map<String, Histogram<String>> histograms = new HashMap<String, Histogram<String>>(); /** * Each AirportCode will have a histogram of the number of flights * that depart from that airport to all the other airports */ protected final Map<String, Histogram<String>> airport_histograms = new HashMap<String, Histogram<String>>(); protected final Map<String, Map<String, Long>> code_id_xref = new HashMap<String, Map<String, Long>>(); // ---------------------------------------------------------------- // TRANSIENT DATA MEMBERS // ---------------------------------------------------------------- /** * TableName -> TableCatalog */ protected transient final CatalogContext catalogContext; /** * Key -> Id Mappings */ protected transient final Map<String, String> code_columns = new HashMap<String, String>(); /** * Foreign Key Mappings * Column Name -> Xref Mapper */ protected transient final Map<String, String> fkey_value_xref = new HashMap<String, String>(); /** * Specialized random number generator */ protected transient final AbstractRandomGenerator rng; /** * Depart Airport Code -> Arrive Airport Code * Random number generators based on the flight distributions */ private final Map<String, FlatHistogram<String>> airport_distributions = new HashMap<String, FlatHistogram<String>>(); // ---------------------------------------------------------------- // CONSTRUCTOR // ---------------------------------------------------------------- public SEATSProfile(CatalogContext catalogContext, AbstractRandomGenerator rng) { this.catalogContext = catalogContext; this.rng = rng; // Tuple Code to Tuple Id Mapping for (String xref[] : SEATSConstants.CODE_TO_ID_COLUMNS) { assert(xref.length == 3); String tableName = xref[0]; String codeCol = xref[1]; String idCol = xref[2]; if (this.code_columns.containsKey(codeCol) == false) { this.code_columns.put(codeCol, idCol); this.code_id_xref.put(idCol, new HashMap<String, Long>()); if (debug.val) LOG.debug(String.format("Added %s mapping from Code Column '%s' to Id Column '%s'", tableName, codeCol, idCol)); } } // FOR // Foreign Key Code to Ids Mapping // In this data structure, the key will be the name of the dependent column // and the value will be the name of the foreign key parent column // We then use this in conjunction with the Key->Id mapping to turn a code into // a foreign key column id. For example, if the child table AIRPORT has a column with a foreign // key reference to COUNTRY.CO_ID, then the data file for AIRPORT will have a value // 'USA' in the AP_CO_ID column. We can use mapping to get the id number for 'USA'. // Long winded and kind of screwy, but hey what else are you going to do? for (Table catalog_tbl : catalogContext.database.getTables()) { for (Column catalog_col : catalog_tbl.getColumns()) { Column catalog_fkey_col = CatalogUtil.getForeignKeyParent(catalog_col); if (catalog_fkey_col != null && this.code_id_xref.containsKey(catalog_fkey_col.getName())) { this.fkey_value_xref.put(catalog_col.getName(), catalog_fkey_col.getName()); if (debug.val) LOG.debug(String.format("Added ForeignKey mapping from %s to %s", catalog_col.fullName(), catalog_fkey_col.fullName())); } } // FOR } // FOR } // ---------------------------------------------------------------- // SAVE / LOAD PROFILE // ---------------------------------------------------------------- /** * Save the profile information into the database */ protected final void saveProfile(BenchmarkComponent baseClient) { // CONFIG_PROFILE Table catalog_tbl = catalogContext.database.getTables().get(SEATSConstants.TABLENAME_CONFIG_PROFILE); VoltTable vt = CatalogUtil.getVoltTable(catalog_tbl); assert(vt != null); vt.addRow( this.scale_factor, // CFP_SCALE_FACTOR this.airport_max_customer_id.toJSONString(), // CFP_AIPORT_MAX_CUSTOMER this.flight_start_date, // CFP_FLIGHT_START this.flight_upcoming_date, // CFP_FLIGHT_UPCOMING this.flight_past_days, // CFP_FLIGHT_PAST_DAYS this.flight_future_days, // CFP_FLIGHT_FUTURE_DAYS this.num_flights, // CFP_NUM_FLIGHTS this.num_customers, // CFP_NUM_CUSTOMERS this.num_reservations, // CFP_NUM_RESERVATIONS JSONUtil.toJSONString(this.code_id_xref) // CFP_CODE_ID_XREF ); if (debug.val) LOG.debug(String.format("Saving profile information into %s\n%s", catalog_tbl, this)); baseClient.loadVoltTable(catalog_tbl.getName(), vt); // CONFIG_HISTOGRAMS catalog_tbl = catalogContext.database.getTables().get(SEATSConstants.TABLENAME_CONFIG_HISTOGRAMS); vt = CatalogUtil.getVoltTable(catalog_tbl); assert(vt != null); for (Entry<String, Histogram<String>> e : this.airport_histograms.entrySet()) { vt.addRow( e.getKey(), // CFH_NAME e.getValue().toJSONString(), // CFH_DATA 1 // CFH_IS_AIRPORT ); } // FOR if (debug.val) LOG.debug("Saving airport histogram information into " + catalog_tbl); baseClient.loadVoltTable(catalog_tbl.getName(), vt); for (Entry<String, Histogram<String>> e : this.histograms.entrySet()) { vt.addRow( e.getKey(), // CFH_NAME e.getValue().toJSONString(), // CFH_DATA 0 // CFH_IS_AIRPORT ); } // FOR if (debug.val) LOG.debug("Saving benchmark histogram information into " + catalog_tbl); baseClient.loadVoltTable(catalog_tbl.getName(), vt); return; } protected static void clearCachedProfile() { cachedProfile = null; } private SEATSProfile copy(SEATSProfile other) { this.scale_factor = other.scale_factor; this.airport_max_customer_id.put(other.airport_max_customer_id); this.flight_start_date = other.flight_start_date; this.flight_upcoming_date = other.flight_upcoming_date; this.flight_past_days = other.flight_past_days; this.flight_future_days = other.flight_future_days; this.num_flights = other.num_flights; this.num_customers = other.num_customers; this.num_reservations = other.num_reservations; this.code_id_xref.putAll(other.code_id_xref); this.airport_histograms.putAll(other.airport_histograms); this.histograms.putAll(other.histograms); return (this); } /** * Load the profile information stored in the database */ private static SEATSProfile cachedProfile; protected final void loadProfile(Client client) { synchronized (SEATSProfile.class) { // Check whether we have a cached Profile we can copy from if (cachedProfile != null) { if (debug.val) LOG.debug("Using cached SEATSProfile"); this.copy(cachedProfile); return; } // Otherwise we have to go fetch everything again if (debug.val) LOG.debug("Loading SEATSProfile for the first time"); ClientResponse response = null; try { response = client.callProcedure(LoadConfig.class.getSimpleName()); } catch (Exception ex) { throw new RuntimeException("Failed retrieve data from " + SEATSConstants.TABLENAME_CONFIG_PROFILE, ex); } assert(response != null); assert(response.getStatus() == Status.OK) : "Unexpected " + response; VoltTable results[] = response.getResults(); int result_idx = 0; // CONFIG_PROFILE this.loadConfigProfile(results[result_idx++]); // CONFIG_HISTOGRAMS this.loadConfigHistograms(results[result_idx++]); // CODE XREFS for (int i = 0; i < SEATSConstants.CODE_TO_ID_COLUMNS.length; i++) { String codeCol = SEATSConstants.CODE_TO_ID_COLUMNS[i][1]; String idCol = SEATSConstants.CODE_TO_ID_COLUMNS[i][2]; this.loadCodeXref(results[result_idx++], codeCol, idCol); } // FOR // CACHED FLIGHT IDS // this.loadCachedFlights(results[result_idx++]); if (debug.val) LOG.debug("Loaded profile:\n" + this.toString()); if (trace.val) LOG.trace("Airport Max Customer Id:\n" + this.airport_max_customer_id); cachedProfile = new SEATSProfile(this.catalogContext, this.rng).copy(this); } // SYNCH } private final void loadConfigProfile(VoltTable vt) { boolean adv = vt.advanceRow(); assert(adv) : "No data in " + SEATSConstants.TABLENAME_CONFIG_PROFILE + ". " + "Did you forget to load the database first?"; int col = 0; this.scale_factor = vt.getDouble(col++); JSONUtil.fromJSONString(this.airport_max_customer_id, vt.getString(col++)); this.flight_start_date = vt.getTimestampAsTimestamp(col++); this.flight_upcoming_date = vt.getTimestampAsTimestamp(col++); this.flight_past_days = vt.getLong(col++); this.flight_future_days = vt.getLong(col++); this.num_flights = vt.getLong(col++); this.num_customers = vt.getLong(col++); this.num_reservations = vt.getLong(col++); if (debug.val) LOG.debug(String.format("Loaded %s data", SEATSConstants.TABLENAME_CONFIG_PROFILE)); } private final void loadConfigHistograms(VoltTable vt) { while (vt.advanceRow()) { int col = 0; String name = vt.getString(col++); ObjectHistogram<String> h = JSONUtil.fromJSONString(new ObjectHistogram<String>(), vt.getString(col++)); boolean is_airline = (vt.getLong(col++) == 1); if (is_airline) { this.airport_histograms.put(name, h); if (trace.val) LOG.trace(String.format("Loaded %d records for %s airport histogram", h.getValueCount(), name)); } else { this.histograms.put(name, h); if (trace.val) LOG.trace(String.format("Loaded %d records for %s histogram", h.getValueCount(), name)); } } // WHILE if (debug.val) LOG.debug(String.format("Loaded %s data", SEATSConstants.TABLENAME_CONFIG_HISTOGRAMS)); } private final void loadCodeXref(VoltTable vt, String codeCol, String idCol) { Map<String, Long> m = this.code_id_xref.get(idCol); while (vt.advanceRow()) { long id = vt.getLong(0); String code = vt.getString(1); m.put(code, id); } // WHILE if (debug.val) LOG.debug(String.format("Loaded %d xrefs for %s -> %s", m.size(), codeCol, idCol)); } // private final void loadCachedFlights(VoltTable vt) { // while (vt.advanceRow()) { // long f_id = vt.getLong(0); // FlightId flight_id = new FlightId(f_id); // this.cached_flight_ids.add(flight_id); // } // WHILE // if (debug.val) // LOG.debug(String.format("Loaded %d cached FlightIds", this.cached_flight_ids.size())); // } // ---------------------------------------------------------------- // DATA ACCESS METHODS // ---------------------------------------------------------------- private Map<String, Long> getCodeXref(String col_name) { Map<String, Long> m = this.code_id_xref.get(col_name); assert(m != null) : "Invalid code xref mapping column '" + col_name + "'"; assert(m.isEmpty() == false) : "Empty code xref mapping for column '" + col_name + "'\n" + StringUtil.formatMaps(this.code_id_xref); return (m); } // ----------------------------------------------------------------- // FLIGHTS // ----------------------------------------------------------------- public long getRandomFlightId() { return (long)this.rng.nextInt((int)this.num_flights); } public long getFlightIdCount() { return (this.num_flights); } // ---------------------------------------------------------------- // HISTOGRAM METHODS // ---------------------------------------------------------------- /** * Return the histogram for the given name * @param name * @return */ protected Histogram<String> getHistogram(String name) { Histogram<String> h = this.histograms.get(name); assert(h != null) : "Invalid histogram '" + name + "'"; return (h); } /** * * @param airport_code * @return */ public Histogram<String> getFightsPerAirportHistogram(String airport_code) { return (this.airport_histograms.get(airport_code)); } /** * Returns the number of histograms that we have loaded * Does not include the airport_histograms * @return */ public int getHistogramCount() { return (this.histograms.size()); } // ---------------------------------------------------------------- // RANDOM GENERATION METHODS // ---------------------------------------------------------------- /** * Return a random airport id * @return */ public long getRandomAirportId() { return (this.rng.number(1, (int)this.getAirportCount())); } public long getRandomOtherAirport(long airport_id) { String code = this.getAirportCode(airport_id); FlatHistogram<String> f = this.airport_distributions.get(code); if (f == null) { synchronized (this.airport_distributions) { f = this.airport_distributions.get(code); if (f == null) { Histogram<String> h = this.airport_histograms.get(code); assert(h != null); f = new FlatHistogram<String>(rng, h); this.airport_distributions.put(code, f); } } // SYCH } assert(f != null); String other = f.nextValue(); return this.getAirportId(other); } /** * Return a random date in the future (after the start of upcoming flights) * @return */ public TimestampType getRandomUpcomingDate() { TimestampType upcoming_start_date = this.flight_upcoming_date; int offset = rng.nextInt((int)this.flight_future_days); return (new TimestampType(upcoming_start_date.getTime() + (offset * SEATSConstants.MICROSECONDS_PER_DAY))); } // /** // * Return a random FlightId from our set of cached ids // * @return // */ // public FlightId getRandomFlightId() { // assert(this.cached_flight_ids.isEmpty() == false); // if (trace.val) // LOG.trace("Attempting to get a random FlightId"); // int idx = rng.nextInt(this.cached_flight_ids.size()); // FlightId flight_id = this.cached_flight_ids.get(idx); // if (trace.val) // LOG.trace("Got random " + flight_id); // return (flight_id); // } // ---------------------------------------------------------------- // AIRLINE METHODS // ---------------------------------------------------------------- public Collection<Long> getAirlineIds() { Map<String, Long> m = this.getCodeXref("AL_ID"); return (m.values()); } public Collection<String> getAirlineCodes() { Map<String, Long> m = this.getCodeXref("AL_ID"); return (m.keySet()); } public Long getAirlineId(String airline_code) { Map<String, Long> m = this.getCodeXref("AL_ID"); return (m.get(airline_code)); } public int incrementAirportCustomerCount(long airport_id) { int next_id = (int)this.airport_max_customer_id.get(airport_id, 0); this.airport_max_customer_id.put(airport_id); return (next_id); } // ---------------------------------------------------------------- // CUSTOMER METHODS // ---------------------------------------------------------------- public Long getCustomerIdCount(Long airport_id) { return (this.airport_max_customer_id.get(airport_id)); } public long getCustomerIdCount() { return (this.num_customers); } /** * Return a random customer id based out of any airport * @return */ public long getRandomCustomerId() { return (long)this.rng.nextInt((int)this.num_customers); } // ---------------------------------------------------------------- // AIRPORT METHODS // ---------------------------------------------------------------- /** * Return all the airport ids that we know about * @return */ public Collection<Long> getAirportIds() { Map<String, Long> m = this.getCodeXref("AP_ID"); return (m.values()); } public Long getAirportId(String airport_code) { Map<String, Long> m = this.getCodeXref("AP_ID"); return (m.get(airport_code)); } public String getAirportCode(long airport_id) { Map<String, Long> m = this.getCodeXref("AP_ID"); for (Entry<String, Long> e : m.entrySet()) { if (e.getValue() == airport_id) return (e.getKey()); } return (null); } public Collection<String> getAirportCodes() { return (this.getCodeXref("AP_ID").keySet()); } /** * Return the number of airports that are part of this profile * @return */ public int getAirportCount() { return (this.getAirportCodes().size()); } public ObjectHistogram<String> getAirportCustomerHistogram() { ObjectHistogram<String> h = new ObjectHistogram<String>(); if (debug.val) LOG.debug("Generating Airport-CustomerCount histogram [numAirports=" + this.getAirportCount() + "]"); for (Long airport_id : this.airport_max_customer_id.values()) { String airport_code = this.getAirportCode(airport_id); int count = this.airport_max_customer_id.get(airport_id).intValue(); h.put(airport_code, count); } // FOR return (h); } /** * Get the list airport codes that have flights * @return */ public Collection<String> getAirportsWithFlights() { return this.airport_histograms.keySet(); } public boolean hasFlights(String airport_code) { Histogram<String> h = this.getFightsPerAirportHistogram(airport_code); if (h != null) { return (h.getSampleCount() > 0); } return (false); } // ----------------------------------------------------------------- // FLIGHT DATES // ----------------------------------------------------------------- /** * The date in which the flight data set begins * @return */ public TimestampType getFlightStartDate() { return this.flight_start_date; } /** * * @param start_date */ public void setFlightStartDate(TimestampType start_date) { this.flight_start_date = start_date; } /** * The date in which the flight data set begins * @return */ public TimestampType getFlightUpcomingDate() { return (this.flight_upcoming_date); } /** * * @param startDate */ public void setFlightUpcomingDate(TimestampType upcoming_date) { this.flight_upcoming_date = upcoming_date; } /** * The date in which upcoming flights begin * @return */ public long getFlightPastDays() { return (this.flight_past_days); } /** * * @param flight_start_date */ public void setFlightPastDays(long flight_past_days) { this.flight_past_days = flight_past_days; } /** * The date in which upcoming flights begin * @return */ public long getFlightFutureDays() { return (this.flight_future_days); } /** * * @param flight_start_date */ public void setFlightFutureDays(long flight_future_days) { this.flight_future_days = flight_future_days; } public long getNextReservationId(int clientId) { // Offset it by the client id so that we can ensure it's unique long r_id = this.num_reservations++ | clientId<<52; assert(r_id != VoltType.NULL_BIGINT) : String.format("Unexpected null reservation id %d [clientId=%d]\n%s", r_id, clientId, this); return (r_id); } @Override public String toString() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Scale Factor", this.scale_factor); m.put("# of Reservations", this.num_reservations); m.put("Flight Start Date", this.flight_start_date); m.put("Flight Upcoming Date", this.flight_upcoming_date); m.put("Flight Past Days", this.flight_past_days); m.put("Flight Future Days", this.flight_future_days); m.put("Num Flights", this.num_flights); m.put("Num Customers", this.num_customers); m.put("Num Reservations", this.num_reservations); return (StringUtil.formatMaps(m)); } }