/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.numeric;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.core.DoubleFieldMapper;
import org.elasticsearch.index.mapper.core.LongFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.string.SimpleStringMappingTests;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
/**
*/
public class SimpleNumericTests extends ESSingleNodeTestCase {
@Test
public void testNumericDetectionEnabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.field("numeric_detection", true)
.endObject().endObject().string();
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("s_long", "100")
.field("s_double", "100.0")
.endObject()
.bytes());
assertNotNull(doc.dynamicMappingsUpdate());
client().admin().indices().preparePutMapping("test").setType("type").setSource(doc.dynamicMappingsUpdate().toString()).get();
defaultMapper = index.mapperService().documentMapper("type");
FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("s_long");
assertThat(mapper, instanceOf(LongFieldMapper.class));
mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
assertThat(mapper, instanceOf(DoubleFieldMapper.class));
}
@Test
public void testNumericDetectionDefault() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject().string();
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("s_long", "100")
.field("s_double", "100.0")
.endObject()
.bytes());
assertNotNull(doc.dynamicMappingsUpdate());
assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource(doc.dynamicMappingsUpdate().toString()).get());
defaultMapper = index.mapperService().documentMapper("type");
FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("s_long");
assertThat(mapper, instanceOf(StringFieldMapper.class));
mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
assertThat(mapper, instanceOf(StringFieldMapper.class));
}
@Test
public void testIgnoreMalformedOption() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1").field("type", "integer").field("ignore_malformed", true).endObject()
.startObject("field2").field("type", "integer").field("ignore_malformed", false).endObject()
.startObject("field3").field("type", "integer").endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field1", "a")
.field("field2", "1")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field1"), nullValue());
assertThat(doc.rootDoc().getField("field2"), notNullValue());
try {
defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field2", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Verify that the default is false
try {
defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field3", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Unless the global ignore_malformed option is set to true
Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
defaultMapper = createIndex("test2", indexSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field3", "a")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field3"), nullValue());
// This should still throw an exception, since field2 is specifically set to ignore_malformed=false
try {
defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field2", "a")
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
}
@Test
public void testCoerceOption() throws Exception {
String [] nonFractionNumericFieldTypes={"integer","long","short"};
//Test co-ercion policies on all non-fraction numerics
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
for (String nonFractionNumericFieldType : nonFractionNumericFieldTypes) {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("noErrorNoCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true)
.field("coerce", false).endObject()
.startObject("noErrorCoerceField").field("type", nonFractionNumericFieldType).field("ignore_malformed", true)
.field("coerce", true).endObject()
.startObject("errorDefaultCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false).endObject()
.startObject("errorNoCoerce").field("type", nonFractionNumericFieldType).field("ignore_malformed", false)
.field("coerce", false).endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
//Test numbers passed as strings
String invalidJsonNumberAsString="1";
ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", invalidJsonNumberAsString)
.field("noErrorCoerceField", invalidJsonNumberAsString)
.field("errorDefaultCoerce", invalidJsonNumberAsString)
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue());
assertThat(doc.rootDoc().getField("noErrorCoerceField"), notNullValue());
//Default is ignore_malformed=true and coerce=true
assertThat(doc.rootDoc().getField("errorDefaultCoerce"), notNullValue());
//Test valid case of numbers passed as numbers
int validNumber=1;
doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", validNumber)
.field("noErrorCoerceField", validNumber)
.field("errorDefaultCoerce", validNumber)
.endObject()
.bytes());
assertEquals(validNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue());
assertEquals(validNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue());
assertEquals(validNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue());
//Test valid case of negative numbers passed as numbers
int validNegativeNumber=-1;
doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", validNegativeNumber)
.field("noErrorCoerceField", validNegativeNumber)
.field("errorDefaultCoerce", validNegativeNumber)
.endObject()
.bytes());
assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue());
assertEquals(validNegativeNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue());
assertEquals(validNegativeNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue());
try {
defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("errorNoCoerce", invalidJsonNumberAsString)
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
}
//Test questionable case of floats passed to ints
float invalidJsonForInteger=1.9f;
int coercedFloatValue=1; //This is what the JSON parser will do to a float - truncate not round
doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("noErrorNoCoerceField", invalidJsonForInteger)
.field("noErrorCoerceField", invalidJsonForInteger)
.field("errorDefaultCoerce", invalidJsonForInteger)
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue());
assertEquals(coercedFloatValue,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue());
//Default is ignore_malformed=true and coerce=true
assertEquals(coercedFloatValue,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue());
try {
defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("errorNoCoerce", invalidJsonForInteger)
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
}
}
}
public void testDocValues() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("int")
.field("type", "integer")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("double")
.field("type", "double")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
ParsedDocument parsedDoc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("int", "1234")
.field("double", "1234")
.endObject()
.bytes());
final Document doc = parsedDoc.rootDoc();
assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "int"));
assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "double"));
}
public void testDocValuesOnNested() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("nested")
.field("type", "nested")
.startObject("properties")
.startObject("int")
.field("type", "integer")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("double")
.field("type", "double")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
ParsedDocument parsedDoc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.startArray("nested")
.startObject()
.field("int", "1234")
.field("double", "1234")
.endObject()
.startObject()
.field("int", "-1")
.field("double", "-2")
.endObject()
.endArray()
.endObject()
.bytes());
for (Document doc : parsedDoc.docs()) {
if (doc == parsedDoc.rootDoc()) {
continue;
}
assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "nested.int"));
assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "nested.double"));
}
}
/** Test default precision step for autodetected numeric types */
@Test
public void testPrecisionStepDefaultsDetected() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.field("numeric_detection", true)
.field("date_detection", true)
.endObject().endObject().string();
DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("long", "100")
.field("double", "100.0")
.field("date", "2010-01-01")
.endObject()
.bytes());
assertEquals(1, doc.docs().size());
Document luceneDoc = doc.docs().get(0);
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("long"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("double"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("date"));
}
/** Test default precision step for numeric types */
@Test
public void testPrecisionStepDefaultsMapped() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("int")
.field("type", "integer")
.endObject()
.startObject("float")
.field("type", "float")
.endObject()
.startObject("long")
.field("type", "long")
.endObject()
.startObject("double")
.field("type", "double")
.endObject()
.startObject("short")
.field("type", "short")
.endObject()
.startObject("byte")
.field("type", "byte")
.endObject()
.startObject("date")
.field("type", "date")
.endObject()
.startObject("ip")
.field("type", "ip")
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("int", "100")
.field("float", "100.0")
.field("long", "5000")
.field("double", "34.545")
.field("short", "1645")
.field("byte", "50")
.field("date", "2010-01-01")
.field("ip", "255.255.255.255")
.endObject()
.bytes());
assertEquals(1, doc.docs().size());
Document luceneDoc = doc.docs().get(0);
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("long"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("double"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("date"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("ip"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_32_BIT, luceneDoc.getField("int"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_32_BIT, luceneDoc.getField("float"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_16_BIT, luceneDoc.getField("short"));
assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_8_BIT, luceneDoc.getField("byte"));
}
/** Test precision step set to silly explicit values */
@Test
public void testPrecisionStepExplicit() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("int")
.field("type", "integer")
.field("precision_step", "1")
.endObject()
.startObject("float")
.field("type", "float")
.field("precision_step", "2")
.endObject()
.startObject("long")
.field("type", "long")
.field("precision_step", "1")
.endObject()
.startObject("double")
.field("type", "double")
.field("precision_step", "2")
.endObject()
.startObject("short")
.field("type", "short")
.field("precision_step", "1")
.endObject()
.startObject("byte")
.field("type", "byte")
.field("precision_step", "2")
.endObject()
.startObject("date")
.field("type", "date")
.field("precision_step", "1")
.endObject()
.startObject("ip")
.field("type", "ip")
.field("precision_step", "2")
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("int", "100")
.field("float", "100.0")
.field("long", "5000")
.field("double", "34.545")
.field("short", "1645")
.field("byte", "50")
.field("date", "2010-01-01")
.field("ip", "255.255.255.255")
.endObject()
.bytes());
assertEquals(1, doc.docs().size());
Document luceneDoc = doc.docs().get(0);
assertPrecisionStepEquals(1, luceneDoc.getField("int"));
assertPrecisionStepEquals(2, luceneDoc.getField("float"));
assertPrecisionStepEquals(1, luceneDoc.getField("long"));
assertPrecisionStepEquals(2, luceneDoc.getField("double"));
assertPrecisionStepEquals(1, luceneDoc.getField("short"));
assertPrecisionStepEquals(2, luceneDoc.getField("byte"));
assertPrecisionStepEquals(1, luceneDoc.getField("date"));
assertPrecisionStepEquals(2, luceneDoc.getField("ip"));
}
/** checks precisionstep on both the fieldtype and the tokenstream */
private static void assertPrecisionStepEquals(int expected, IndexableField field) throws IOException {
assertNotNull(field);
assertThat(field, instanceOf(Field.class));
// check fieldtype's precisionstep
assertEquals(expected, ((Field)field).fieldType().numericPrecisionStep());
// check the tokenstream actually used by the indexer
TokenStream ts = field.tokenStream(null, null);
assertThat(ts, instanceOf(NumericTokenStream.class));
assertEquals(expected, ((NumericTokenStream)ts).getPrecisionStep());
}
public void testTermVectorsBackCompat() throws Exception {
for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) {
doTestTermVectorsBackCompat(type);
}
}
private void doTestTermVectorsBackCompat(String type) throws Exception {
DocumentMapperParser parser = createIndex("index-" + type).mapperService().documentMapperParser();
String mappingWithTV = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("foo")
.field("type", type)
.field("term_vector", "yes")
.endObject()
.endObject().endObject().endObject().string();
try {
parser.parse("type", new CompressedXContent(mappingWithTV));
fail();
} catch (MapperParsingException e) {
assertThat(e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters: [term_vector : yes]"));
}
Settings oldIndexSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0)
.build();
parser = createIndex("index2-" + type, oldIndexSettings).mapperService().documentMapperParser();
parser.parse("type", new CompressedXContent(mappingWithTV)); // no exception
}
public void testAnalyzerBackCompat() throws Exception {
for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) {
doTestAnalyzerBackCompat(type);
}
}
private void doTestAnalyzerBackCompat(String type) throws Exception {
DocumentMapperParser parser = createIndex("index-" + type).mapperService().documentMapperParser();
String mappingWithTV = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("foo")
.field("type", type)
.field("analyzer", "keyword")
.endObject()
.endObject().endObject().endObject().string();
try {
parser.parse("type", new CompressedXContent(mappingWithTV));
fail();
} catch (MapperParsingException e) {
assertThat(e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters: [analyzer : keyword]"));
}
Settings oldIndexSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_1_0)
.build();
parser = createIndex("index2-" + type, oldIndexSettings).mapperService().documentMapperParser();
parser.parse("type", new CompressedXContent(mappingWithTV)); // no exception
}
}