// This file is part of OpenTSDB.
// Copyright (C) 2013 The OpenTSDB Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or (at your
// option) any later version. This program is distributed in the hope that it
// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
// General Public License for more details. You should have received a copy
// of the GNU Lesser General Public License along with this program. If not,
// see <http://www.gnu.org/licenses/>.
package net.opentsdb.meta;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import org.hbase.async.DeleteRequest;
import org.hbase.async.GetRequest;
import org.hbase.async.HBaseException;
import org.hbase.async.KeyValue;
import org.hbase.async.PutRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
import net.opentsdb.core.TSDB;
import net.opentsdb.uid.UniqueId;
import net.opentsdb.uid.UniqueId.UniqueIdType;
import net.opentsdb.utils.JSON;
import net.opentsdb.utils.JSONException;
/**
* UIDMeta objects are associated with the UniqueId of metrics, tag names
* or tag values. When a new metric, tagk or tagv is generated, a UIDMeta object
* will also be written to storage with only the uid, type and name filled out.
* <p>
* Users are allowed to edit the following fields:
* <ul><li>display_name</li>
* <li>description</li>
* <li>notes</li>
* <li>custom</li></ul>
* The {@code name}, {@code uid}, {@code type} and {@code created} fields can
* only be modified by the system and are usually done so on object creation.
* <p>
* When you call {@link #syncToStorage} on this object, it will verify that the
* UID object this meta data is linked with still exists. Then it will fetch the
* existing data and copy changes, overwriting the user fields if specific
* (e.g. via a PUT command). If overwriting is not called for (e.g. a POST was
* issued), then only the fields provided by the user will be saved, preserving
* all of the other fields in storage. Hence the need for the {@code changed}
* hash map and the {@link #syncMeta} method.
* <p>
* Note that the HBase specific storage code will be removed once we have a DAL
* @since 2.0
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY)
public final class UIDMeta {
private static final Logger LOG = LoggerFactory.getLogger(UIDMeta.class);
/** Charset used to convert Strings to byte arrays and back. */
private static final Charset CHARSET = Charset.forName("ISO-8859-1");
/** The single column family used by this class. */
private static final byte[] FAMILY = "name".getBytes(CHARSET);
/** A hexadecimal representation of the UID this metadata is associated with */
private String uid = "";
/** The type of UID this metadata represents */
@JsonDeserialize(using = JSON.UniqueIdTypeDeserializer.class)
private UniqueIdType type = null;
/**
* This is the identical name of what is stored in the UID table
* It cannot be overridden
*/
private String name = "";
/**
* An optional, user supplied name used for display purposes only
* If this field is empty, the {@link name} field should be used
*/
private String display_name = "";
/** A short description of what this object represents */
private String description = "";
/** Optional, detailed notes about what the object represents */
private String notes = "";
/** A timestamp of when this UID was first recorded by OpenTSDB in seconds */
private long created = 0;
/** Optional user supplied key/values */
private HashMap<String, String> custom = null;
/** Tracks fields that have changed by the user to avoid overwrites */
private final HashMap<String, Boolean> changed =
new HashMap<String, Boolean>();
/**
* Default constructor
* Initializes the the changed map
*/
public UIDMeta() {
initializeChangedMap();
}
/**
* Constructor used for overwriting. Will not reset the name or created values
* in storage.
* @param type Type of UID object
* @param uid UID of the object
*/
public UIDMeta(final UniqueIdType type, final String uid) {
this.type = type;
this.uid = uid;
initializeChangedMap();
}
/**
* Constructor used by TSD only to create a new UID with the given data and
* the current system time for {@code createdd}
* @param type Type of UID object
* @param uid UID of the object
* @param name Name of the UID
*/
public UIDMeta(final UniqueIdType type, final byte[] uid, final String name) {
this.type = type;
this.uid = UniqueId.uidToString(uid);
this.name = name;
created = System.currentTimeMillis() / 1000;
initializeChangedMap();
changed.put("created", true);
}
/** @return a string with details about this object */
@Override
public String toString() {
return "'" + type.toString() + ":" + uid + "'";
}
/**
* Attempts a CompareAndSet storage call, loading the object from storage,
* synchronizing changes, and attempting a put.
* <b>Note:</b> If the local object didn't have any fields set by the caller
* then the data will not be written.
* @param tsdb The TSDB to use for storage access
* @param overwrite When the RPC method is PUT, will overwrite all user
* accessible fields
* @return True if the storage call was successful, false if the object was
* modified in storage during the CAS call. If false, retry the call. Other
* failures will result in an exception being thrown.
* @throws HBaseException if there was an issue fetching
* @throws IllegalArgumentException if parsing failed
* @throws NoSuchUniqueId If the UID does not exist
* @throws IllegalStateException if the data hasn't changed. This is OK!
* @throws JSONException if the object could not be serialized
*/
public Deferred<Boolean> syncToStorage(final TSDB tsdb,
final boolean overwrite) {
if (uid == null || uid.isEmpty()) {
throw new IllegalArgumentException("Missing UID");
}
if (type == null) {
throw new IllegalArgumentException("Missing type");
}
boolean has_changes = false;
for (Map.Entry<String, Boolean> entry : changed.entrySet()) {
if (entry.getValue()) {
has_changes = true;
break;
}
}
if (!has_changes) {
LOG.debug(this + " does not have changes, skipping sync to storage");
throw new IllegalStateException("No changes detected in UID meta data");
}
/**
* Callback used to verify that the UID to name mapping exists. Uses the TSD
* for verification so the name may be cached. If the name does not exist
* it will throw a NoSuchUniqueId and the meta data will not be saved to
* storage
*/
final class NameCB implements Callback<Deferred<Boolean>, String> {
private final UIDMeta local_meta;
public NameCB(final UIDMeta meta) {
local_meta = meta;
}
/**
* Nested callback used to merge and store the meta data after verifying
* that the UID mapping exists. It has to access the {@code local_meta}
* object so that's why it's nested within the NameCB class
*/
final class StoreUIDMeta implements Callback<Deferred<Boolean>,
ArrayList<KeyValue>> {
/**
* Executes the CompareAndSet after merging changes
* @return True if the CAS was successful, false if the stored data
* was modified during flight.
*/
@Override
public Deferred<Boolean> call(final ArrayList<KeyValue> row)
throws Exception {
final UIDMeta stored_meta;
if (row == null || row.isEmpty()) {
stored_meta = null;
} else {
stored_meta = JSON.parseToObject(row.get(0).value(), UIDMeta.class);
stored_meta.initializeChangedMap();
}
final byte[] original_meta = row == null || row.isEmpty() ?
new byte[0] : row.get(0).value();
if (stored_meta != null) {
local_meta.syncMeta(stored_meta, overwrite);
}
// verify the name is set locally just to be safe
if (name == null || name.isEmpty()) {
local_meta.name = name;
}
final PutRequest put = new PutRequest(tsdb.uidTable(),
UniqueId.stringToUid(uid), FAMILY,
(type.toString().toLowerCase() + "_meta").getBytes(CHARSET),
local_meta.getStorageJSON());
return tsdb.getClient().compareAndSet(put, original_meta);
}
}
/**
* NameCB method that fetches the object from storage for merging and
* use in the CAS call
* @return The results of the {@link #StoreUIDMeta} callback
*/
@Override
public Deferred<Boolean> call(final String name) throws Exception {
final GetRequest get = new GetRequest(tsdb.uidTable(),
UniqueId.stringToUid(uid));
get.family(FAMILY);
get.qualifier((type.toString().toLowerCase() + "_meta").getBytes(CHARSET));
// #2 deferred
return tsdb.getClient().get(get)
.addCallbackDeferring(new StoreUIDMeta());
}
}
// start the callback chain by veryfing that the UID name mapping exists
return tsdb.getUidName(type, UniqueId.stringToUid(uid))
.addCallbackDeferring(new NameCB(this));
}
/**
* Attempts to store a blank, new UID meta object in the proper location.
* <b>Warning:</b> This should not be called by user accessible methods as it
* will overwrite any data already in the column. This method does not use
* a CAS, instead it uses a PUT to overwrite anything in the column.
* @param tsdb The TSDB to use for calls
* @return A deferred without meaning. The response may be null and should
* only be used to track completion.
* @throws HBaseException if there was an issue writing to storage
* @throws IllegalArgumentException if data was missing
* @throws JSONException if the object could not be serialized
*/
public Deferred<Object> storeNew(final TSDB tsdb) {
if (uid == null || uid.isEmpty()) {
throw new IllegalArgumentException("Missing UID");
}
if (type == null) {
throw new IllegalArgumentException("Missing type");
}
if (name == null || name.isEmpty()) {
throw new IllegalArgumentException("Missing name");
}
final PutRequest put = new PutRequest(tsdb.uidTable(),
UniqueId.stringToUid(uid), FAMILY,
(type.toString().toLowerCase() + "_meta").getBytes(CHARSET),
UIDMeta.this.getStorageJSON());
return tsdb.getClient().put(put);
}
/**
* Attempts to delete the meta object from storage
* @param tsdb The TSDB to use for access to storage
* @return A deferred without meaning. The response may be null and should
* only be used to track completion.
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if data was missing (uid and type)
*/
public Deferred<Object> delete(final TSDB tsdb) {
if (uid == null || uid.isEmpty()) {
throw new IllegalArgumentException("Missing UID");
}
if (type == null) {
throw new IllegalArgumentException("Missing type");
}
final DeleteRequest delete = new DeleteRequest(tsdb.uidTable(),
UniqueId.stringToUid(uid), FAMILY,
(type.toString().toLowerCase() + "_meta").getBytes(CHARSET));
return tsdb.getClient().delete(delete);
}
/**
* Convenience overload of {@code getUIDMeta(TSDB, UniqueIdType, byte[])}
* @param tsdb The TSDB to use for storage access
* @param type The type of UID to fetch
* @param uid The ID of the meta to fetch
* @return A UIDMeta from storage or a default
* @throws HBaseException if there was an issue fetching
* @throws NoSuchUniqueId If the UID does not exist
*/
public static Deferred<UIDMeta> getUIDMeta(final TSDB tsdb,
final UniqueIdType type, final String uid) {
return getUIDMeta(tsdb, type, UniqueId.stringToUid(uid));
}
/**
* Verifies the UID object exists, then attempts to fetch the meta from
* storage and if not found, returns a default object.
* <p>
* The reason for returning a default object (with the type, uid and name set)
* is due to users who may have just enabled meta data or have upgraded; we
* want to return valid data. If they modify the entry, it will write to
* storage. You can tell it's a default if the {@code created} value is 0. If
* the meta was generated at UID assignment or updated by the meta sync CLI
* command, it will have a valid created timestamp.
* @param tsdb The TSDB to use for storage access
* @param type The type of UID to fetch
* @param uid The ID of the meta to fetch
* @return A UIDMeta from storage or a default
* @throws HBaseException if there was an issue fetching
* @throws NoSuchUniqueId If the UID does not exist
*/
public static Deferred<UIDMeta> getUIDMeta(final TSDB tsdb,
final UniqueIdType type, final byte[] uid) {
/**
* Callback used to verify that the UID to name mapping exists. Uses the TSD
* for verification so the name may be cached. If the name does not exist
* it will throw a NoSuchUniqueId and the meta data will not be returned.
* This helps in case the user deletes a UID but the meta data is still
* stored. The fsck utility can be used later to cleanup orphaned objects.
*/
class NameCB implements Callback<Deferred<UIDMeta>, String> {
/**
* Called after verifying that the name mapping exists
* @return The results of {@link #FetchMetaCB}
*/
@Override
public Deferred<UIDMeta> call(final String name) throws Exception {
/**
* Inner class called to retrieve the meta data after verifying that the
* name mapping exists. It requires the name to set the default, hence
* the reason it's nested.
*/
class FetchMetaCB implements Callback<Deferred<UIDMeta>,
ArrayList<KeyValue>> {
/**
* Called to parse the response of our storage GET call after
* verification
* @return The stored UIDMeta or a default object if the meta data
* did not exist
*/
@Override
public Deferred<UIDMeta> call(ArrayList<KeyValue> row)
throws Exception {
if (row == null || row.isEmpty()) {
// return the default
final UIDMeta meta = new UIDMeta();
meta.uid = UniqueId.uidToString(uid);
meta.type = type;
meta.name = name;
return Deferred.fromResult(meta);
}
final UIDMeta meta = JSON.parseToObject(row.get(0).value(),
UIDMeta.class);
// overwrite the name and UID
meta.name = name;
meta.uid = UniqueId.uidToString(uid);
// fix missing types
if (meta.type == null) {
final String qualifier =
new String(row.get(0).qualifier(), CHARSET);
meta.type = UniqueId.stringToUniqueIdType(qualifier.substring(0,
qualifier.indexOf("_meta")));
}
meta.initializeChangedMap();
return Deferred.fromResult(meta);
}
}
final GetRequest get = new GetRequest(tsdb.uidTable(), uid);
get.family(FAMILY);
get.qualifier((type.toString().toLowerCase() + "_meta").getBytes(CHARSET));
return tsdb.getClient().get(get).addCallbackDeferring(new FetchMetaCB());
}
}
// verify that the UID is still in the map before fetching from storage
return tsdb.getUidName(type, uid).addCallbackDeferring(new NameCB());
}
/**
* Syncs the local object with the stored object for atomic writes,
* overwriting the stored data if the user issued a PUT request
* <b>Note:</b> This method also resets the {@code changed} map to false
* for every field
* @param meta The stored object to sync from
* @param overwrite Whether or not all user mutable data in storage should be
* replaced by the local object
*/
private void syncMeta(final UIDMeta meta, final boolean overwrite) {
// copy non-user-accessible data first
if (meta.uid != null && !meta.uid.isEmpty()) {
uid = meta.uid;
}
if (meta.name != null && !meta.name.isEmpty()) {
name = meta.name;
}
if (meta.type != null) {
type = meta.type;
}
if (meta.created > 0 && (meta.created < created || created == 0)) {
created = meta.created;
}
// handle user-accessible stuff
if (!overwrite && !changed.get("display_name")) {
display_name = meta.display_name;
}
if (!overwrite && !changed.get("description")) {
description = meta.description;
}
if (!overwrite && !changed.get("notes")) {
notes = meta.notes;
}
if (!overwrite && !changed.get("custom")) {
custom = meta.custom;
}
// reset changed flags
initializeChangedMap();
}
/**
* Sets or resets the changed map flags
*/
private void initializeChangedMap() {
// set changed flags
changed.put("display_name", false);
changed.put("description", false);
changed.put("notes", false);
changed.put("custom", false);
changed.put("created", false);
}
/**
* Formats the JSON output for writing to storage. It drops objects we don't
* need or want to store (such as the UIDMeta objects or the total dps) to
* save space. It also serializes in order so that we can make a proper CAS
* call. Otherwise the POJO serializer may place the fields in any order
* and CAS calls would fail all the time.
* @return A byte array to write to storage
*/
private byte[] getStorageJSON() {
// 256 bytes is a good starting value, assumes default info
final ByteArrayOutputStream output = new ByteArrayOutputStream(256);
try {
final JsonGenerator json = JSON.getFactory().createGenerator(output);
json.writeStartObject();
json.writeStringField("type", type.toString());
json.writeStringField("displayName", display_name);
json.writeStringField("description", description);
json.writeStringField("notes", notes);
json.writeNumberField("created", created);
if (custom == null) {
json.writeNullField("custom");
} else {
json.writeObjectFieldStart("custom");
for (Map.Entry<String, String> entry : custom.entrySet()) {
json.writeStringField(entry.getKey(), entry.getValue());
}
json.writeEndObject();
}
json.writeEndObject();
json.close();
return output.toByteArray();
} catch (IOException e) {
throw new RuntimeException("Unable to serialize UIDMeta", e);
}
}
// Getters and Setters --------------
/** @return the uid as a hex encoded string */
public String getUID() {
return uid;
}
/** @return the type of UID represented */
public UniqueIdType getType() {
return type;
}
/** @return the name of the UID object */
public String getName() {
return name;
}
/** @return optional display name, use {@code name} if empty */
public String getDisplayName() {
return display_name;
}
/** @return optional description */
public String getDescription() {
return description;
}
/** @return optional notes */
public String getNotes() {
return notes;
}
/** @return when the UID was first assigned, may be 0 if unknown */
public long getCreated() {
return created;
}
/** @return optional map of custom values from the user */
public Map<String, String> getCustom() {
return custom;
}
/** @param display_name an optional descriptive name for the UID */
public void setDisplayName(final String display_name) {
if (!this.display_name.equals(display_name)) {
changed.put("display_name", true);
this.display_name = display_name;
}
}
/** @param description an optional description of the UID */
public void setDescription(final String description) {
if (!this.description.equals(description)) {
changed.put("description", true);
this.description = description;
}
}
/** @param notes optional notes */
public void setNotes(final String notes) {
if (!this.notes.equals(notes)) {
changed.put("notes", true);
this.notes = notes;
}
}
/** @param custom the custom to set */
public void setCustom(final Map<String, String> custom) {
// equivalency of maps is a pain, users have to submit the whole map
// anyway so we'll just mark it as changed every time we have a non-null
// value
if (this.custom != null || custom != null) {
changed.put("custom", true);
this.custom = new HashMap<String, String>(custom);
}
}
/** @param created the created timestamp Unix epoch in seconds */
public final void setCreated(final long created) {
if (this.created != created) {
changed.put("created", true);
this.created = created;
}
}
}