/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import org.apache.spark.SparkConf;
import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
/**
* A {@link KryoSerializer} that attempts to honor {@link GryoPool#CONFIG_IO_REGISTRY}.
*/
public class IoRegistryAwareKryoSerializer extends KryoSerializer {
private final SparkConf conf;
private static final Logger log = LoggerFactory.getLogger(IoRegistryAwareKryoSerializer.class);
public IoRegistryAwareKryoSerializer(final SparkConf conf) {
super(conf);
// store conf so that we can access its registry (if one is present) in newKryo()
this.conf = conf;
}
@Override
public Kryo newKryo() {
final Kryo kryo = super.newKryo();
return applyIoRegistryIfPresent(kryo);
}
private Kryo applyIoRegistryIfPresent(final Kryo kryo) {
if (!conf.contains(GryoPool.CONFIG_IO_REGISTRY)) {
log.info("SparkConf {} does not contain setting {}, skipping {} handling",
GryoPool.CONFIG_IO_REGISTRY, conf, IoRegistry.class.getCanonicalName());
return kryo;
}
final String registryClassnames = conf.get(GryoPool.CONFIG_IO_REGISTRY);
for (String registryClassname : registryClassnames.split(",")) {
final IoRegistry registry;
try {
registry = (IoRegistry) Class.forName(registryClassname).newInstance();
log.info("Instantiated {}", registryClassname);
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
log.error("Unable to reflectively instantiate the {} implementation named {}",
IoRegistry.class.getCanonicalName(), registryClassname, e);
return kryo;
}
// Left is the class targeted for serialization, right is a mess of potential types, including
// a shaded Serializer impl, unshaded Serializer impl, or Function<shaded.Kryo,shaded.Serializer>
final List<Pair<Class, Object>> serializers = registry.find(GryoIo.class);
if (null == serializers) {
log.info("Invoking find({}.class) returned null on registry {}; ignoring this registry",
GryoIo.class.getCanonicalName(), registry);
return kryo;
}
for (Pair<Class, Object> p : serializers) {
if (null == p.getValue1()) {
// null on the right is fine
log.info("Registering {} with default serializer", p.getValue0());
kryo.register(p.getValue0());
} else if (p.getValue1() instanceof Serializer) {
// unshaded serializer on the right is fine
log.info("Registering {} with serializer {}", p.getValue0(), p.getValue1());
kryo.register(p.getValue0(), (Serializer) p.getValue1());
} else {
// anything else on the right is unsupported with Spark
log.error("Serializer {} found in {} must implement {} " +
"(the shaded interface {} is not supported on Spark). This class will be registered with " +
"the default behavior of Spark's KryoSerializer.",
p.getValue1(), registryClassname, Serializer.class.getCanonicalName(),
org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
kryo.register(p.getValue0());
}
}
}
return kryo;
}
}