package mil.nga.giat.geowave.analytic; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import mil.nga.giat.geowave.analytic.param.ParameterEnum; import mil.nga.giat.geowave.core.geotime.store.query.SpatialQuery; import mil.nga.giat.geowave.core.index.ByteArrayUtils; import mil.nga.giat.geowave.core.index.Persistable; import mil.nga.giat.geowave.core.index.PersistenceUtils; import mil.nga.giat.geowave.core.index.sfc.data.NumericRange; import mil.nga.giat.geowave.core.store.query.DistributableQuery; import mil.nga.giat.geowave.core.store.query.QueryOptions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.ParseException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.vividsolutions.jts.geom.GeometryFactory; import com.vividsolutions.jts.geom.PrecisionModel; import com.vividsolutions.jts.io.WKTReader; /** * Manage properties used by the Map Reduce environment that are provided * through the API (e.g. command). Allow these arguments to be placed an 'args' * list for 'main' executables (e.g. ToolRunner). * * The class supports some basic conversions. * * Non-serializable objects: {@link Persistable} instances are converted to and * from byte formats. {@link DistributableQuery} is a special case, supporting * WKT String. {@link Path} are converted to a from string representation of the * their URI. * * Serializable objects: {@link NumericRange} supports min,max in string * representation (e.g. "1.0,2.0") * * * NOTE: ConfigutationWrapper implementation is scopeless. * * EXPECTED FUTURE WORK: I am bit unsatisfied with the duality of the parameters * base class. In one case, in is treated a description for a class value and, * in the other case, it is treated as a description for the type of a property * value. The former is really a descriptor of a Class of type class. Generics * do not help due to erasure. The impact of this inconsistency is the inability * to validate on 'store'. Instead, validation occurs on 'gets'. The ultimate * goal is to uniformly provide feedback to parameters from command line * arguments and property files on submission to the manager rather than on * extraction from the manager. * */ public class PropertyManagement implements Serializable { /** * */ private static final long serialVersionUID = -4186468044516636362L; final static Logger LOGGER = LoggerFactory.getLogger(PropertyManagement.class); private final Map<ParameterEnum<?>, Serializable> localProperties = new HashMap<ParameterEnum<?>, Serializable>(); private final List<PropertyConverter<?>> converters = new ArrayList<PropertyConverter<?>>(); private PropertyManagement nestProperties = null; public PropertyManagement() { converters.add(new QueryConverter()); converters.add(new QueryOptionsConverter()); converters.add(new PathConverter()); converters.add(new PersistableConverter()); converters.add(new DoubleConverter()); converters.add(new IntegerConverter()); converters.add(new ByteConverter()); } public PropertyManagement( final PropertyConverter<?>[] converters, final ParameterEnum<?>[] names, final Object[] values ) { this.converters.add(new QueryConverter()); this.converters.add(new QueryOptionsConverter()); this.converters.add(new PathConverter()); this.converters.add(new PersistableConverter()); this.converters.add(new DoubleConverter()); this.converters.add(new IntegerConverter()); this.converters.add(new ByteConverter()); for (final PropertyConverter<?> converter : converters) { addConverter(converter); } storeAll( names, values); } public PropertyManagement( final ParameterEnum<?>[] names, final Object[] values ) { converters.add(new QueryConverter()); converters.add(new QueryOptionsConverter()); converters.add(new PathConverter()); converters.add(new PersistableConverter()); converters.add(new DoubleConverter()); converters.add(new IntegerConverter()); converters.add(new ByteConverter()); storeAll( names, values); } public PropertyManagement( final PropertyManagement pm ) { nestProperties = pm; converters.addAll(pm.converters); } public Serializable get( final ParameterEnum<?> propertyName ) { return getPropertyValue(propertyName); } public synchronized <T> void store( final ParameterEnum<?> property, final T value, final PropertyConverter<T> converter ) { Serializable convertedValue; try { convertedValue = converter.convert(value); } catch (final Exception e) { throw new IllegalArgumentException( String.format( "Cannot store %s with value %s. Expected type = %s; Error message = %s", property.self().toString(), value.toString(), property.getHelper().getBaseClass().toString(), e.getLocalizedMessage()), e); } localProperties.put( property, convertedValue); addConverter(converter); } public synchronized void store( final ParameterEnum<?> property, final Object value ) { if (value != null) { Serializable convertedValue; try { convertedValue = convertIfNecessary( property, value); } catch (final Exception e) { throw new IllegalArgumentException( String.format( "Cannot store %s with value %s:%s", property.self().toString(), value.toString(), e.getLocalizedMessage())); } localProperties.put( property, convertedValue); } } /** * Does not work for non-serializable data (e.g. Path or Persistable) * */ public synchronized Serializable storeIfEmpty( final ParameterEnum<?> propertyEnum, final Serializable value ) { if (!containsPropertyValue(propertyEnum) && value != null) { LOGGER.info( "Setting parameter : {} to {}", propertyEnum.toString(), value.toString()); store( propertyEnum, value); return value; } return getPropertyValue(propertyEnum); } public synchronized void copy( final ParameterEnum<?> propertyNameFrom, final ParameterEnum<?> propertyNameTo ) { if (containsPropertyValue(propertyNameFrom)) { localProperties.put( propertyNameTo, getPropertyValue(propertyNameFrom)); } } public synchronized void storeAll( final ParameterEnum<?>[] names, final Object[] values ) { if (values.length != names.length) { LOGGER.error("The number of values must equal the number of names passed to the store method"); throw new IllegalArgumentException( "The number of values must equal the number of names passed to the store method"); } int i = 0; for (final Object value : values) { store( names[i++], value); } } public void setConfig( final ParameterEnum<?>[] parameters, final Configuration config, final Class<?> scope ) { for (final ParameterEnum param : parameters) { Object value; try { value = getProperty(param); param.getHelper().setValue( config, scope, value); } catch (final Exception e) { LOGGER.error( "Property " + param.self().toString() + " is not available", e); throw new IllegalArgumentException( "Property " + param.self().toString() + " is not available", e); } } } @SuppressWarnings("unchecked") public <T> T getClassInstance( final ParameterEnum<?> property, final Class<T> iface, final Class<?> defaultClass ) throws InstantiationException { final Object o = getPropertyValue(property); try { final Class<?> clazz = o == null ? defaultClass : (o instanceof Class) ? (Class<?>) o : Class.forName(o .toString()); if (!property.getHelper().getBaseClass().isAssignableFrom( clazz)) { LOGGER.error("Class for property " + property.self().toString() + " does not implement " + property.getHelper().getBaseClass().toString()); } return (T) clazz.newInstance(); } catch (final ClassNotFoundException e) { LOGGER.error( "Class for property " + property.self().toString() + " is not found", e); throw new InstantiationException( property.self().toString()); } catch (final InstantiationException e) { LOGGER.error( "Class for property " + property.self().toString() + " is not instiatable", e); throw new InstantiationException( property.self().toString()); } catch (final IllegalAccessException e) { LOGGER.error( "Class for property " + property.self().toString() + " is not accessible", e); throw new InstantiationException( property.self().toString()); } } public synchronized boolean hasProperty( final ParameterEnum<?> property ) { return containsPropertyValue(property); } public String getPropertyAsString( final ParameterEnum<?> property ) { return getPropertyAsString( property, null); } /** * Returns the value as, without conversion from the properties. Throws an * exception if a conversion is required to a specific type * * @param property * @return * @throws Exception * @throws IllegalArgumentException */ public Object getProperty( final ParameterEnum<?> property ) throws Exception { final Serializable value = getPropertyValue(property); if (!Serializable.class.isAssignableFrom(property.getHelper().getBaseClass())) { for (final PropertyConverter converter : converters) { if (converter.baseClass().isAssignableFrom( property.getHelper().getBaseClass())) { return this.validate( property, converter.convert(value)); } } } return this.validate( property, value); } /** * Returns the value after conversion. Throws an exception if a conversion * fails. * * @param property * @return * @throws Exception * @throws IllegalArgumentException */ public <T> T getProperty( final ParameterEnum<?> property, final PropertyConverter<T> converter ) throws Exception { final Serializable value = getPropertyValue(property); return converter.convert(value); } public byte[] getPropertyAsBytes( final ParameterEnum<?> property ) { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof byte[]) { return (byte[]) val; } return ByteArrayUtils.byteArrayFromString(val.toString()); } return null; } public String getPropertyAsString( final ParameterEnum<?> property, final String defaultValue ) { // not using containsKey to avoid synchronization final Object value = getPropertyValue(property); return (String) validate( property, value == null ? defaultValue : value.toString()); } public Boolean getPropertyAsBoolean( final ParameterEnum<?> property, final Boolean defaultValue ) { final Object val = getPropertyValue(property); if (val != null) { return Boolean.valueOf(val.toString()); } LOGGER.warn("Using default value for parameter : " + property.self().toString()); return defaultValue; } public Integer getPropertyAsInt( final ParameterEnum<?> property, final int defaultValue ) { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof Integer) { return (Integer) val; } return (Integer) validate( property, Integer.parseInt(val.toString())); } LOGGER.warn("Using default value for parameter : " + property.self().toString()); return defaultValue; } public Double getPropertyAsDouble( final ParameterEnum<?> property, final double defaultValue ) { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof Double) { return (Double) val; } return Double.parseDouble(val.toString()); } LOGGER.warn("Using default value for parameter : " + property.self().toString()); return defaultValue; } public NumericRange getPropertyAsRange( final ParameterEnum<?> property, final NumericRange defaultValue ) { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof NumericRange) { return (NumericRange) val; } final String p = val.toString(); final String[] parts = p.split(","); try { if (parts.length == 2) { return new NumericRange( Double.parseDouble(parts[0].trim()), Double.parseDouble(parts[1].trim())); } else { return new NumericRange( 0, Double.parseDouble(p)); } } catch (final Exception ex) { LOGGER.error( "Invalid range parameter " + property.self().toString(), ex); return defaultValue; } } LOGGER.warn("Using default value for parameter : " + property.self().toString()); return defaultValue; } public Class<?> getPropertyAsClass( final ParameterEnum<?> property ) { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof Class) { return validate( (Class<?>) val, property.getHelper().getBaseClass()); } try { return validate( (Class<?>) Class.forName(val.toString()), property.getHelper().getBaseClass()); } catch (final ClassNotFoundException e) { LOGGER.error( "Class not found for property " + property, e); } catch (final java.lang.IllegalArgumentException ex) { LOGGER.error( "Invalid class for property" + property, ex); throw new IllegalArgumentException( "Invalid class for property" + property); } } return null; } public <T> Class<T> getPropertyAsClass( final ParameterEnum<?> property, final Class<T> iface ) throws ClassNotFoundException { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof Class) { return validate( (Class<T>) val, property.getHelper().getBaseClass()); } try { return validate( (Class<T>) Class.forName(val.toString()), property.getHelper().getBaseClass()); } catch (final ClassNotFoundException e) { LOGGER.error("Class not found for property " + property.self().toString()); throw e; } catch (final java.lang.IllegalArgumentException ex) { LOGGER.error( "Invalid class for property" + property.self().toString(), ex); throw new IllegalArgumentException( "Invalid class for property" + property); } } else { LOGGER.error("Value not found for property " + property.self().toString()); } throw new ClassNotFoundException( "Value not found for property " + property.self().toString()); } public <T> Class<? extends T> getPropertyAsClass( final ParameterEnum<?> property, final Class<? extends T> iface, final Class<? extends T> defaultClass ) { final Object val = getPropertyValue(property); if (val != null) { if (val instanceof Class) { return validate( (Class<T>) val, property.getHelper().getBaseClass()); } try { return validate( (Class<T>) Class.forName(val.toString()), property.getHelper().getBaseClass()); } catch (final ClassNotFoundException e) { LOGGER.error( "Class not found for property " + property, e); } catch (final java.lang.IllegalArgumentException ex) { LOGGER.error( "Invalid class for property" + property, ex); throw new IllegalArgumentException( "Invalid class for property" + property); } } LOGGER.warn("Using default class for parameter : " + property.self().toString()); return defaultClass; } private <T> Class<T> validate( final Class<T> classToValidate, final Class<?> iface ) throws IllegalArgumentException { if (!iface.isAssignableFrom(classToValidate)) { throw new IllegalArgumentException( classToValidate + "is an invalid subclass of " + iface); } return classToValidate; } public DistributableQuery getPropertyAsQuery( final ParameterEnum<?> property ) throws Exception { final Serializable val = getPropertyValue(property); if (val != null) { return (DistributableQuery) validate( property, new QueryConverter().convert(val)); } return null; } public QueryOptions getPropertyAsQueryOptions( final ParameterEnum property ) throws Exception { final Serializable val = getPropertyValue(property); if (val != null) { return (QueryOptions) validate( property, new QueryOptionsConverter().convert(val)); } return null; } public Path getPropertyAsPath( final ParameterEnum<?> property ) throws Exception { final Serializable val = getPropertyValue(property); if (val != null) { return (Path) validate( property, new PathConverter().convert(val)); } return null; } public Persistable getPropertyAsPersistable( final ParameterEnum<?> property ) throws Exception { final Serializable val = getPropertyValue(property); if (val != null) { return (Persistable) validate( property, new PersistableConverter().convert(val)); } return null; } public void setJobConfiguration( final Configuration configuration, final Class<?> scope ) { for (final ParameterEnum param : localProperties.keySet()) { param.getHelper().setValue( configuration, scope, param.getHelper().getValue( this)); } if ((nestProperties != null) && !nestProperties.localProperties.isEmpty()) { nestProperties.setJobConfiguration( configuration, scope); } } public void dump() { LOGGER.info("Properties : "); for (final Map.Entry<ParameterEnum<?>, Serializable> prop : localProperties.entrySet()) { LOGGER.info( "{} = {}", prop.getKey(), prop.getValue()); } nestProperties.dump(); } /** * Add to the set of converters used to take a String representation of a * value and convert it into another serializable form. * * This is done if the preferred internal representation does not match that * of a string. For example, a query is maintained as bytes even though it * can be provided as a query * * @param converter */ public synchronized void addConverter( final PropertyConverter<?> converter ) { converters.add(converter); } private static byte[] toBytes( final Persistable persistableObject ) throws UnsupportedEncodingException { return PersistenceUtils.toBinary(persistableObject); } private static Persistable fromBytes( final byte[] data, final Class<? extends Persistable> expectedType ) throws InstantiationException, IllegalAccessException, ClassNotFoundException, UnsupportedEncodingException { return PersistenceUtils.fromBinary( data, expectedType); } private Object validate( final ParameterEnum propertyName, final Object value ) { if (value != null) { if (value instanceof Class) { if (((Class<?>) value).isAssignableFrom(propertyName.getHelper().getBaseClass())) { throw new IllegalArgumentException( String.format( "%s does not accept class %s", propertyName.self().toString(), ((Class<?>) value).getName())); } } else if (!propertyName.getHelper().getBaseClass().isInstance( value)) { throw new IllegalArgumentException( String.format( "%s does not accept type %s", propertyName.self().toString(), value.getClass().getName())); } } return value; } @SuppressWarnings("unchecked") private Serializable convertIfNecessary( final ParameterEnum property, final Object value ) throws Exception { if (!(value instanceof Serializable)) { for (@SuppressWarnings("rawtypes") final PropertyConverter converter : converters) { if (converter.baseClass().isAssignableFrom( property.getHelper().getBaseClass())) { return converter.convert(value); } } } if (!property.getHelper().getBaseClass().isInstance( value) && (value instanceof String)) { for (@SuppressWarnings("rawtypes") final PropertyConverter converter : converters) { if (converter.baseClass().isAssignableFrom( property.getHelper().getBaseClass())) { return converter.convert(converter.convert(value.toString())); } } } return (Serializable) value; } public interface PropertyConverter<T> extends Serializable { public Serializable convert( T ob ) throws Exception; public T convert( Serializable ob ) throws Exception; public Class<T> baseClass(); } public interface PropertyGroup<T extends Serializable> extends Serializable { public T convert( CommandLine commandLine ) throws ParseException; public ParameterEnum getParameter(); } public static class QueryConverter implements PropertyConverter<DistributableQuery> { /** * */ private static final long serialVersionUID = 1L; @Override public Serializable convert( final DistributableQuery ob ) { try { return toBytes(ob); } catch (final UnsupportedEncodingException e) { throw new IllegalArgumentException( String.format( "Cannot convert %s to a DistributableQuery: %s", ob.toString(), e.getLocalizedMessage())); } } @Override public DistributableQuery convert( final Serializable ob ) throws Exception { if (ob instanceof byte[]) { return (DistributableQuery) PropertyManagement.fromBytes( (byte[]) ob, DistributableQuery.class); } final PrecisionModel precision = new PrecisionModel(); final GeometryFactory geometryFactory = new GeometryFactory( precision, 4326); final WKTReader wktReader = new WKTReader( geometryFactory); return new SpatialQuery( wktReader.read(ob.toString())); } @Override public Class<DistributableQuery> baseClass() { return DistributableQuery.class; } } public static class QueryOptionsConverter implements PropertyConverter<QueryOptions> { /** * */ private static final long serialVersionUID = 1L; @Override public Serializable convert( final QueryOptions ob ) { try { return toBytes(ob); } catch (final UnsupportedEncodingException e) { throw new IllegalArgumentException( String.format( "Cannot convert %s to a QueryOptions: %s", ob.toString(), e.getLocalizedMessage())); } } @Override public QueryOptions convert( final Serializable ob ) throws Exception { if (ob instanceof byte[]) { return (QueryOptions) PropertyManagement.fromBytes( (byte[]) ob, QueryOptions.class); } else if (ob instanceof QueryOptions) { return (QueryOptions) ob; } return new QueryOptions(); } @Override public Class<QueryOptions> baseClass() { return QueryOptions.class; } } public static class PathConverter implements PropertyConverter<Path> { /** * */ private static final long serialVersionUID = 1L; @Override public Serializable convert( final Path ob ) { return ob.toUri().toString(); } @Override public Path convert( final Serializable ob ) throws Exception { return new Path( ob.toString()); } @Override public Class<Path> baseClass() { return Path.class; } } public static class ByteConverter implements PropertyConverter<byte[]> { private static final long serialVersionUID = 1L; @Override public Serializable convert( final byte[] ob ) { return ByteArrayUtils.byteArrayToString(ob); } @Override public byte[] convert( final Serializable ob ) throws Exception { return ByteArrayUtils.byteArrayFromString(ob.toString()); } @Override public Class<byte[]> baseClass() { return byte[].class; } } public static class IntegerConverter implements PropertyConverter<Integer> { private static final long serialVersionUID = 1L; @Override public Serializable convert( final Integer ob ) { return ob; } @Override public Integer convert( final Serializable ob ) throws Exception { return Integer.parseInt(ob.toString()); } @Override public Class<Integer> baseClass() { return Integer.class; } } public static class DoubleConverter implements PropertyConverter<Double> { /** * */ private static final long serialVersionUID = 1L; @Override public Serializable convert( final Double ob ) { return ob; } @Override public Double convert( final Serializable ob ) throws Exception { return Double.parseDouble(ob.toString()); } @Override public Class<Double> baseClass() { return Double.class; } } public static class PersistableConverter implements PropertyConverter<Persistable> { /** * */ private static final long serialVersionUID = 1L; @Override public Serializable convert( final Persistable ob ) { try { return toBytes(ob); } catch (final UnsupportedEncodingException e) { throw new IllegalArgumentException( String.format( "Cannot convert %s to a Persistable: %s", ob.toString(), e.getLocalizedMessage())); } } @Override public Persistable convert( final Serializable ob ) throws Exception { if (ob instanceof byte[]) { return fromBytes( (byte[]) ob, Persistable.class); } throw new IllegalArgumentException( String.format( "Cannot convert %s to Persistable", ob.toString())); } @Override public Class<Persistable> baseClass() { return Persistable.class; } } private boolean containsPropertyValue( final ParameterEnum<?> property ) { return ((nestProperties != null) && nestProperties.containsPropertyValue(property)) || localProperties.containsKey(property); } private Serializable getPropertyValue( final ParameterEnum<?> property ) { final Serializable val = localProperties != null ? localProperties.get(property) : null; if (val == null) { return nestProperties != null ? nestProperties.getPropertyValue(property) : null; } return val; } }