/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update.processor;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.schema.IndexSchema;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.junit.BeforeClass;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
/**
* Tests for the field mutating update processors
* that parse Dates, Longs, Doubles, and Booleans.
*/
public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
private static final double EPSILON = 1E-15;
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig-parsing-update-processor-chains.xml", "schema12.xml");
}
public void testParseDateRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("date_dt")); // should match "*_dt" dynamic field
String dateString = "2010-11-12T13:14:15.168Z";
SolrInputDocument d = processAdd("parse-date", doc(f("id", "9"), f("date_dt", dateString)));
assertNotNull(d);
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
assertTrue(d.getFieldValue("date_dt") instanceof Date);
assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_dt")).getTime());
assertU(commit());
assertQ(req("id:9"), "//date[@name='date_dt'][.='" + dateString + "']");
}
public void testParseTrieDateRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("date_tdt")); // should match "*_tdt" dynamic field
String dateString = "2010-11-12T13:14:15.168Z";
SolrInputDocument d = processAdd("parse-date", doc(f("id", "39"), f("date_tdt", dateString)));
assertNotNull(d);
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
assertTrue(d.getFieldValue("date_tdt") instanceof Date);
assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_tdt")).getTime());
assertU(commit());
assertQ(req("id:39"), "//date[@name='date_tdt'][.='" + dateString + "']");
}
public void testParseDateFieldNotInSchema() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "2010-11-12T13:14:15.168Z";
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-date-no-run-processor",
doc(f("id", "18"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
d = processAdd("parse-date-no-run-processor",
doc(f("id", "36"), f("not_in_schema", "not a date", dateString)));
assertNotNull(d);
for (Object val : d.getFieldValues("not_in_schema")) {
// check that nothing was mutated, since not all field values are parseable as dates
assertTrue(val instanceof String);
}
d = processAdd("parse-date-no-run-processor",
doc(f("id", "72"), f("not_in_schema", dateString, "not a date")));
assertNotNull(d);
for (Object val : d.getFieldValues("not_in_schema")) {
// check again that nothing was mutated, but with a valid date first this time
assertTrue(val instanceof String);
}
}
public void testParseDateNonUTCdefaultTimeZoneRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("date_dt")); // should match "*_dt" dynamic field
String dateStringNoTimeZone = "2010-11-12T13:14:15.168";
String dateStringUTC = dateStringNoTimeZone + "Z";
// dateStringNoTimeZone interpreted as being in timeZone America/New_York, then printed as UTC
String dateStringUSEasternTimeAsUTC = "2010-11-12T18:14:15.168Z";
SolrInputDocument d = processAdd
("parse-date-non-UTC-defaultTimeZone", doc(f("id", "99"), f("dateUTC_dt", dateStringUTC),
f("dateNoTimeZone_dt", dateStringNoTimeZone)));
assertNotNull(d);
String pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(pattern);
DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(dateStringUTC);
assertTrue(d.getFieldValue("dateUTC_dt") instanceof Date);
assertTrue(d.getFieldValue("dateNoTimeZone_dt") instanceof Date);
assertEquals(dateTimeUTC.getMillis(), ((Date) d.getFieldValue("dateUTC_dt")).getTime());
assertU(commit());
assertQ(req("id:99")
,"//date[@name='dateUTC_dt'][.='" + dateStringUTC + "']"
,"//date[@name='dateNoTimeZone_dt'][.='" + dateStringUSEasternTimeAsUTC + "']");
}
public void testParseDateExplicitNotInSchemaSelector() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "2010-11-12T13:14:15.168Z";
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-date-explicit-not-in-schema-selector-no-run-processor",
doc(f("id", "88"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
}
public void testParseDateExplicitTypeClassSelector() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("date_dt"));
String dateString = "2010-11-12T13:14:15.168Z";
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d;
if (schema.getField("date_dt").getType().isPointField()) {
d = processAdd("parse-date-explicit-typeclass-point-selector-no-run-processor",
doc(f("id", "77"), f("date_dt", dateString)));
} else {
d = processAdd("parse-date-explicit-typeclass-selector-no-run-processor",
doc(f("id", "77"), f("date_dt", dateString)));
}
assertNotNull(d);
assertTrue(d.getFieldValue("date_dt") instanceof Date);
assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("date_dt")).getTime());
}
public void testParseUSPacificDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "8/9/2010"; // Interpreted as 00:00 US Pacific Daylight Time = UTC+07:00
String dateStringUTC = "2010-08-09T07:00:00.000Z";
SolrInputDocument d = processAdd("US-Pacific-parse-date-no-run-processor",
doc(f("id", "288"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
assertEquals(dateStringUTC,
(new DateTime(((Date)d.getFieldValue("not_in_schema")).getTime(),DateTimeZone.UTC)).toString());
}
public void testParseDateFormats() throws Exception {
String[] formatExamples = {
"yyyy-MM-dd'T'HH:mm:ss.SSSZ", "2010-01-15T00:00:00.000Z",
"yyyy-MM-dd'T'HH:mm:ss,SSSZ", "2010-01-15T00:00:00,000Z",
"yyyy-MM-dd'T'HH:mm:ss.SSS", "2010-01-15T00:00:00.000",
"yyyy-MM-dd'T'HH:mm:ss,SSS", "2010-01-15T00:00:00,000",
"yyyy-MM-dd'T'HH:mm:ssZ", "2010-01-15T00:00:00Z",
"yyyy-MM-dd'T'HH:mm:ss", "2010-01-15T00:00:00",
"yyyy-MM-dd'T'HH:mmZ", "2010-01-15T00:00Z",
"yyyy-MM-dd'T'HH:mm", "2010-01-15T00:00",
"yyyy-MM-dd HH:mm:ss.SSSZ", "2010-01-15 00:00:00.000Z",
"yyyy-MM-dd HH:mm:ss,SSSZ", "2010-01-15 00:00:00,000Z",
"yyyy-MM-dd HH:mm:ss.SSS", "2010-01-15 00:00:00.000",
"yyyy-MM-dd HH:mm:ss,SSS", "2010-01-15 00:00:00,000",
"yyyy-MM-dd HH:mm:ssZ", "2010-01-15 00:00:00Z",
"yyyy-MM-dd HH:mm:ss", "2010-01-15 00:00:00",
"yyyy-MM-dd HH:mmZ", "2010-01-15 00:00Z",
"yyyy-MM-dd HH:mm", "2010-01-15 00:00",
"yyyy-MM-dd hh:mm a", "2010-01-15 12:00 AM",
"yyyy-MM-dd hh:mma", "2010-01-15 12:00AM",
"yyyy-MM-dd", "2010-01-15",
"EEE MMM dd HH:mm:ss Z yyyy", "Fri Jan 15 00:00:00 +0000 2010",
"EEE MMM dd HH:mm:ss yyyy Z", "Fri Jan 15 00:00:00 2010 +00:00",
"EEE MMM dd HH:mm:ss yyyy", "Fri Jan 15 00:00:00 2010",
"EEE, dd MMM yyyy HH:mm:ss Z", "Fri, 15 Jan 2010 00:00:00 +00:00",
"EEEE, dd-MMM-yy HH:mm:ss Z", "Friday, 15-Jan-10 00:00:00 +00:00",
"EEEE, MMMM dd, yyyy", "Friday, January 15, 2010",
"MMMM dd, yyyy", "January 15, 2010",
"MMM. dd, yyyy", "Jan. 15, 2010"
};
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("dateUTC_dt")); // should match "*_dt" dynamic field
String dateTimePattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(dateTimePattern);
DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(formatExamples[1]);
for (int i = 0 ; i < formatExamples.length ; i += 2) {
String format = formatExamples[i];
String dateString = formatExamples[i + 1];
String id = "95" + i;
SolrInputDocument d = processAdd("parse-date-UTC-defaultTimeZone-no-run-processor",
doc(f("id", id), f("dateUTC_dt", dateString)));
assertNotNull(d);
assertTrue("date '" + dateString + "' with format '" + format + "' is not mutated to a Date",
d.getFieldValue("dateUTC_dt") instanceof Date);
assertEquals("date '" + dateString + "' with format '" + format + "' mismatched milliseconds",
dateTimeUTC.getMillis(), ((Date)d.getFieldValue("dateUTC_dt")).getTime());
}
}
public void testParseFrenchDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String frenchDateString = "le vendredi 15 janvier 2010";
String dateString = "2010-01-15T00:00:00.000Z";
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-french-date-UTC-defaultTimeZone-no-run-processor",
doc(f("id", "88"), f("not_in_schema", frenchDateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
}
public void testFailedParseMixedDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Object,Object> mixed = new HashMap<>();
String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
mixed.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
}
Double extraDouble = 29.554d;
mixed.put(extraDouble, extraDouble); // Double-typed field value
SolrInputDocument d = processAdd("parse-date-no-run-processor",
doc(f("id", "7201"), f("not_in_schema", mixed.values())));
assertNotNull(d);
boolean foundDouble = false;
for (Object o : d.getFieldValues("not_in_schema")) {
if (extraDouble == o) {
foundDouble = true;
} else {
assertTrue(o instanceof String);
}
mixed.values().remove(o);
}
assertTrue(foundDouble);
assertTrue(mixed.isEmpty());
}
public void testParseIntRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("int1_i")); // should match dynamic field "*_i"
assertNotNull(schema.getFieldOrNull("int2_i")); // should match dynamic field "*_i"
int value = 1089883491;
String intString1 = "1089883491";
String intString2 = "1,089,883,491";
SolrInputDocument d = processAdd("parse-int",
doc(f("id", "113"), f("int1_i", intString1), f("int2_i", intString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("int1_i") instanceof Integer);
assertEquals(value, ((Integer)d.getFieldValue("int1_i")).intValue());
assertTrue(d.getFieldValue("int2_i") instanceof Integer);
assertEquals(value, ((Integer)d.getFieldValue("int2_i")).intValue());
assertU(commit());
assertQ(req("id:113")
,"//int[@name='int1_i'][.='" + value + "']"
,"//int[@name='int2_i'][.='" + value + "']");
}
public void testParseIntNonRootLocale() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("int_i")); // should match dynamic field "*_i"
assertNull(schema.getFieldOrNull("not_in_schema"));
int value = 1089883491;
String intString1 = "1089883491";
String intString2 = "1 089 883 491"; // no-break space U+00A0
SolrInputDocument d = processAdd("parse-int-russian-no-run-processor",
doc(f("id", "113"), f("int_i", intString1), f("not_in_schema", intString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("int_i") instanceof Integer);
assertEquals(value, ((Integer)d.getFieldValue("int_i")).intValue());
assertTrue(d.getFieldValue("not_in_schema") instanceof Integer);
assertEquals(value, ((Integer)d.getFieldValue("not_in_schema")).intValue());
}
public void testParseTrieIntRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("int1_ti")); // should match dynamic field "*_ti"
assertNotNull(schema.getFieldOrNull("int2_ti")); // should match dynamic field "*_ti"
int value = 1089883491;
String intString1 = "1089883491";
String intString2 = "1,089,883,491";
SolrInputDocument d = processAdd("parse-int",
doc(f("id", "113"), f("int1_ti", intString1), f("int2_ti", intString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("int1_ti") instanceof Integer);
assertEquals(value, ((Integer)d.getFieldValue("int1_ti")).intValue());
assertTrue(d.getFieldValue("int2_ti") instanceof Integer);
assertEquals(value, ((Integer)d.getFieldValue("int2_ti")).intValue());
assertU(commit());
assertQ(req("id:113")
,"//int[@name='int1_ti'][.='" + value + "']"
,"//int[@name='int2_ti'][.='" + value + "']");
}
public void testIntOverflow() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema1"));
assertNull(schema.getFieldOrNull("not_in_schema2"));
long longValue1 = (long)Integer.MAX_VALUE + 100L;
long longValue2 = (long)Integer.MIN_VALUE - 100L;
String longString1 = Long.toString(longValue1);
String longString2 = Long.toString(longValue2);
SolrInputDocument d = processAdd("parse-int-no-run-processor",
doc(f("id", "282"), f("not_in_schema1", longString1), f("not_in_schema2", longString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema1") instanceof String);
assertTrue(d.getFieldValue("not_in_schema2") instanceof String);
}
public void testFailedParseMixedInt() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
Map<Object,Object> mixed = new HashMap<>();
Float floatVal = 294423.0f;
mixed.put(85, "85");
mixed.put(floatVal, floatVal); // Float-typed field value
mixed.put(-2894518, "-2,894,518");
mixed.put(1879472193, "1,879,472,193");
SolrInputDocument d = processAdd("parse-int-no-run-processor",
doc(f("id", "7202"), f("not_in_schema", mixed.values())));
assertNotNull(d);
boolean foundFloat = false;
for (Object o : d.getFieldValues("not_in_schema")) {
if (floatVal == o) {
foundFloat = true;
} else {
assertTrue(o instanceof String);
}
mixed.values().remove(o);
}
assertTrue(foundFloat);
assertTrue(mixed.isEmpty());
}
public void testParseLongRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("long1_l")); // should match dynamic field "*_l"
assertNotNull(schema.getFieldOrNull("long2_l")); // should match dynamic field "*_l"
long value = 1089883491L;
String longString1 = "1089883491";
String longString2 = "1,089,883,491";
SolrInputDocument d = processAdd("parse-long",
doc(f("id", "113"), f("long1_l", longString1), f("long2_l", longString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("long1_l") instanceof Long);
assertEquals(value, ((Long) d.getFieldValue("long1_l")).longValue());
assertTrue(d.getFieldValue("long2_l") instanceof Long);
assertEquals(value, ((Long)d.getFieldValue("long2_l")).longValue());
assertU(commit());
assertQ(req("id:113")
,"//long[@name='long1_l'][.='" + value + "']"
,"//long[@name='long2_l'][.='" + value + "']");
}
public void testParseLongNonRootLocale() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("long_l")); // should match dynamic field "*_l"
assertNull(schema.getFieldOrNull("not_in_schema"));
long value = 1089883491L;
String longString1 = "1089883491";
String longString2 = "1 089 883 491"; // no-break space U+00A0
SolrInputDocument d = processAdd("parse-long-russian-no-run-processor",
doc(f("id", "113"), f("long_l", longString1), f("not_in_schema", longString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("long_l") instanceof Long);
assertEquals(value, ((Long)d.getFieldValue("long_l")).longValue());
assertTrue(d.getFieldValue("not_in_schema") instanceof Long);
assertEquals(value, ((Long)d.getFieldValue("not_in_schema")).longValue());
}
public void testParseTrieLongRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("long1_tl")); // should match dynamic field "*_tl"
assertNotNull(schema.getFieldOrNull("long2_tl")); // should match dynamic field "*_tl"
long value = 1089883491L;
String longString1 = "1089883491";
String longString2 = "1,089,883,491";
SolrInputDocument d = processAdd("parse-long",
doc(f("id", "113"), f("long1_tl", longString1), f("long2_tl", longString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("long1_tl") instanceof Long);
assertEquals(value, ((Long)d.getFieldValue("long1_tl")).longValue());
assertTrue(d.getFieldValue("long2_tl") instanceof Long);
assertEquals(value, ((Long)d.getFieldValue("long2_tl")).longValue());
assertU(commit());
assertQ(req("id:113")
,"//long[@name='long1_tl'][.='" + value + "']"
,"//long[@name='long2_tl'][.='" + value + "']");
}
public void testFailedParseMixedLong() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
Map<Object,Object> mixed = new HashMap<>();
Float floatVal = 294423.0f;
mixed.put(85L, "85");
mixed.put(floatVal, floatVal); // Float-typed field value
mixed.put(-2894518L, "-2,894,518");
mixed.put(1879472193L, "1,879,472,193");
SolrInputDocument d = processAdd("parse-long-no-run-processor",
doc(f("id", "7204"), f("not_in_schema", mixed.values())));
assertNotNull(d);
boolean foundFloat = false;
for (Object o : d.getFieldValues("not_in_schema")) {
if (floatVal == o) {
foundFloat = true;
} else {
assertTrue(o instanceof String);
}
mixed.values().remove(o);
}
assertTrue(foundFloat);
assertTrue(mixed.isEmpty());
}
public void testParseFloatRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("float1_f")); // should match dynamic field "*_f"
assertNotNull(schema.getFieldOrNull("float2_f")); // should match dynamic field "*_f"
float value = 10898.83491f;
String floatString1 = "10898.83491";
String floatString2 = "10,898.83491";
SolrInputDocument d = processAdd("parse-float",
doc(f("id", "128"), f("float1_f", floatString1), f("float2_f", floatString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("float1_f") instanceof Float);
assertEquals(value, (Float)d.getFieldValue("float1_f"), EPSILON);
assertTrue(d.getFieldValue("float2_f") instanceof Float);
assertEquals(value, (Float)d.getFieldValue("float2_f"), EPSILON);
assertU(commit());
assertQ(req("id:128")
,"//float[@name='float1_f'][.='" + value + "']"
,"//float[@name='float2_f'][.='" + value + "']");
}
public void testParseFloatNonRootLocale() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("float_f")); // should match dynamic field "*_f"
assertNull(schema.getFieldOrNull("not_in_schema"));
float value = 10898.83491f;
String floatString1 = "10898,83491";
String floatString2 = "10 898,83491"; // no-break space: U+00A0
SolrInputDocument d = processAdd("parse-float-french-no-run-processor",
doc(f("id", "140"), f("float_f", floatString1),
f("not_in_schema", floatString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("float_f") instanceof Float);
assertEquals(value, (Float)d.getFieldValue("float_f"), EPSILON);
assertTrue(d.getFieldValue("not_in_schema") instanceof Float);
assertEquals(value, (Float)d.getFieldValue("not_in_schema"), EPSILON);
}
public void testParseTrieFloatRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("float1_tf")); // should match dynamic field "*_tf"
assertNotNull(schema.getFieldOrNull("float2_tf")); // should match dynamic field "*_tf"
float value = 10898.83491f;
String floatString1 = "10898.83491";
String floatString2 = "10,898.83491";
SolrInputDocument d = processAdd("parse-float",
doc(f("id", "728"), f("float1_tf", floatString1), f("float2_tf", floatString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("float1_tf") instanceof Float);
assertEquals(value, (Float)d.getFieldValue("float1_tf"), EPSILON);
assertTrue(d.getFieldValue("float2_tf") instanceof Float);
assertEquals(value, (Float)d.getFieldValue("float2_tf"), EPSILON);
assertU(commit());
assertQ(req("id:728")
,"//float[@name='float1_tf'][.='" + value + "']"
,"//float[@name='float2_tf'][.='" + value + "']");
}
public void testMixedFloats() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("float_tf")); // should match dynamic field "*_tf"
Map<Float,Object> mixedFloats = new HashMap<>();
mixedFloats.put(85.0f, "85");
mixedFloats.put(2894518.0f, "2,894,518");
mixedFloats.put(2.94423E-9f, 2.94423E-9f); // Float-typed field value
mixedFloats.put(48794721.937f, "48,794,721.937");
SolrInputDocument d = processAdd("parse-float-no-run-processor",
doc(f("id", "342"), f("float_tf", mixedFloats.values())));
assertNotNull(d);
for (Object o : d.getFieldValues("float_tf")) {
assertTrue(o instanceof Float);
mixedFloats.remove(o);
}
assertTrue(mixedFloats.isEmpty());
}
public void testFailedParseMixedFloat() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
Map<Object,Object> mixed = new HashMap<>();
Long longVal = 294423L;
mixed.put(85L, "85");
mixed.put(longVal, longVal); // Float-typed field value
mixed.put(-2894518L, "-2,894,518");
mixed.put(1879472193L, "1,879,472,193");
SolrInputDocument d = processAdd("parse-float-no-run-processor",
doc(f("id", "7205"), f("not_in_schema", mixed.values())));
assertNotNull(d);
boolean foundLong = false;
for (Object o : d.getFieldValues("not_in_schema")) {
if (longVal == o) {
foundLong = true;
} else {
assertTrue(o instanceof String);
}
mixed.values().remove(o);
}
assertTrue(foundLong);
assertTrue(mixed.isEmpty());
}
public void testParseDoubleRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("double1_d")); // should match dynamic field "*_d"
assertNotNull(schema.getFieldOrNull("double2_d")); // should match dynamic field "*_d"
double value = 10898.83491;
String doubleString1 = "10898.83491";
String doubleString2 = "10,898.83491";
SolrInputDocument d = processAdd("parse-double",
doc(f("id", "128"), f("double1_d", doubleString1), f("double2_d", doubleString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("double1_d") instanceof Double);
assertEquals(value, (Double)d.getFieldValue("double1_d"), EPSILON);
assertTrue(d.getFieldValue("double2_d") instanceof Double);
assertEquals(value, (Double)d.getFieldValue("double2_d"), EPSILON);
assertU(commit());
assertQ(req("id:128")
,"//double[@name='double1_d'][.='" + value + "']"
,"//double[@name='double2_d'][.='" + value + "']");
}
public void testParseDoubleNonRootLocale() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("double_d")); // should match dynamic field "*_d"
assertNull(schema.getFieldOrNull("not_in_schema"));
double value = 10898.83491;
String doubleString1 = "10898,83491";
String doubleString2 = "10 898,83491"; // no-break space: U+00A0
SolrInputDocument d = processAdd("parse-double-french-no-run-processor",
doc(f("id", "140"), f("double_d", doubleString1),
f("not_in_schema", doubleString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("double_d") instanceof Double);
assertEquals(value, (Double)d.getFieldValue("double_d"), EPSILON);
assertTrue(d.getFieldValue("not_in_schema") instanceof Double);
assertEquals(value, (Double)d.getFieldValue("not_in_schema"), EPSILON);
}
public void testParseTrieDoubleRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("double1_td")); // should match dynamic field "*_td"
assertNotNull(schema.getFieldOrNull("double2_td")); // should match dynamic field "*_td"
double value = 10898.83491;
String doubleString1 = "10898.83491";
String doubleString2 = "10,898.83491";
SolrInputDocument d = processAdd("parse-double",
doc(f("id", "728"), f("double1_td", doubleString1), f("double2_td", doubleString2)));
assertNotNull(d);
assertTrue(d.getFieldValue("double1_td") instanceof Double);
assertEquals(value, (Double)d.getFieldValue("double1_td"), EPSILON);
assertTrue(d.getFieldValue("double2_td") instanceof Double);
assertEquals(value, (Double)d.getFieldValue("double2_td"), EPSILON);
assertU(commit());
assertQ(req("id:728")
,"//double[@name='double1_td'][.='" + value + "']"
,"//double[@name='double2_td'][.='" + value + "']");
}
public void testFailedParseMixedDouble() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
Map<Object,Object> mixed = new HashMap<>();
Long longVal = 294423L;
mixed.put(85, "85.0");
mixed.put(longVal, longVal); // Float-typed field value
mixed.put(-2894.518, "-2,894.518");
mixed.put(187947.2193, "187,947.2193");
SolrInputDocument d = processAdd("parse-double-no-run-processor",
doc(f("id", "7206"), f("not_in_schema", mixed.values())));
assertNotNull(d);
boolean foundLong = false;
for (Object o : d.getFieldValues("not_in_schema")) {
if (longVal == o) {
foundLong = true;
} else {
assertTrue(o instanceof String);
}
mixed.values().remove(o);
}
assertTrue(foundLong);
assertTrue(mixed.isEmpty());
}
public void testParseBooleanRoundTrip() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("boolean1_b")); // should match dynamic field "*_b"
assertNotNull(schema.getFieldOrNull("boolean2_b")); // should match dynamic field "*_b"
boolean value1 = true;
boolean value2 = false;
SolrInputDocument d = processAdd("parse-boolean",
doc(f("id", "141"), f("boolean1_b", value1), f("boolean2_b", value2)));
assertNotNull(d);
assertTrue(d.getFieldValue("boolean1_b") instanceof Boolean);
assertEquals(value1, d.getFieldValue("boolean1_b"));
assertTrue(d.getFieldValue("boolean2_b") instanceof Boolean);
assertEquals(value2, d.getFieldValue("boolean2_b"));
assertU(commit());
assertQ(req("id:141")
,"//bool[@name='boolean1_b'][.='" + value1 + "']"
,"//bool[@name='boolean2_b'][.='" + value2 + "']");
}
public void testParseAlternateValueBooleans() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("boolean1_b")); // should match dynamic field "*_b"
assertNotNull(schema.getFieldOrNull("boolean2_b")); // should match dynamic field "*_b"
assertNotNull(schema.getFieldOrNull("boolean3_b")); // should match dynamic field "*_b"
assertNotNull(schema.getFieldOrNull("boolean4_b")); // should match dynamic field "*_b"
assertNotNull(schema.getFieldOrNull("boolean5_b")); // should match dynamic field "*_b"
assertNull(schema.getFieldOrNull("not_in_schema"));
boolean[] values = { true, true, true, false, false, false };
String[] stringValues = { "on", "yes", "True", "Off", "no", "FALSE" };
String[] fieldNames = { "boolean1_b", "boolean2_b", "boolean3_b", "boolean4_b", "boolean5_b", "not_in_schema" };
SolrInputDocument d = doc(f("id", "55"));
for (int i = 0 ; i < values.length ; ++i) {
d.addField(fieldNames[i], stringValues[i]);
}
d = processAdd("parse-boolean-alternate-values-no-run-processor", d);
assertNotNull(d);
for (int i = 0 ; i < values.length ; ++i) {
assertTrue(d.getFieldValue(fieldNames[i]) instanceof Boolean);
assertEquals(values[i], d.getFieldValue(fieldNames[i]));
}
}
public void testParseAlternateSingleValuesBooleans() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("boolean1_b")); // should match dynamic field "*_b"
assertNotNull(schema.getFieldOrNull("boolean2_b")); // should match dynamic field "*_b"
boolean[] values = { true, false };
String[] stringValues = { "yup", "nope" };
String[] fieldNames = { "boolean1_b", "boolean2_b" };
SolrInputDocument d = doc(f("id", "59"));
for (int i = 0 ; i < values.length ; ++i) {
d.addField(fieldNames[i], stringValues[i]);
}
d = processAdd("parse-boolean-alternate-single-values-no-run-processor", d);
assertNotNull(d);
for (int i = 0 ; i < values.length ; ++i) {
assertTrue(d.getFieldValue(fieldNames[i]) instanceof Boolean);
assertEquals(values[i], d.getFieldValue(fieldNames[i]));
}
// Standard boolean values should not be mutated, since they're not configured
stringValues = new String[] { "true", "false" };
d = doc(f("id", "593"));
for (int i = 0 ; i < values.length ; ++i) {
d.addField(fieldNames[i], stringValues[i]);
}
d = processAdd("parse-boolean-alternate-single-values-no-run-processor", d);
assertNotNull(d);
for (int i = 0 ; i < values.length ; ++i) {
assertTrue(d.getFieldValue(fieldNames[i]) instanceof String);
}
}
public void testFailedParseMixedBoolean() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
Map<Object,Object> mixed = new HashMap<>();
Long longVal = 294423L;
mixed.put(true, "true");
mixed.put(longVal, longVal); // Float-typed field value
mixed.put(false, "false");
mixed.put(true, "true");
SolrInputDocument d = processAdd("parse-boolean-no-run-processor",
doc(f("id", "7207"), f("not_in_schema", mixed.values())));
assertNotNull(d);
boolean foundLong = false;
for (Object o : d.getFieldValues("not_in_schema")) {
if (longVal == o) {
foundLong = true;
} else {
assertTrue(o instanceof String);
}
mixed.values().remove(o);
}
assertTrue(foundLong);
assertTrue(mixed.isEmpty());
}
public void testCascadingParsers() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
final String fieldName = "not_in_schema";
assertNull(schema.getFieldOrNull(fieldName));
SolrInputDocument d = null;
String chain = "cascading-parsers-no-run-processor";
Map<Boolean,String> booleans = new HashMap<>();
booleans.put(true, "truE");
booleans.put(false, "False");
d = processAdd(chain, doc(f("id", "341"), f(fieldName, booleans.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Boolean);
booleans.remove(o);
}
assertTrue(booleans.isEmpty());
Map<Integer,String> ints = new HashMap<>();
ints.put(2, "2");
ints.put(50928, "50928");
ints.put(86942008, "86,942,008");
d = processAdd(chain, doc(f("id", "333"), f(fieldName, ints.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Integer);
ints.remove(o);
}
assertTrue(ints.isEmpty());
Map<Long,String> longs = new HashMap<>();
longs.put(2L, "2");
longs.put(50928L, "50928");
longs.put(86942008987654L, "86,942,008,987,654");
d = processAdd(chain, doc(f("id", "342"), f(fieldName, longs.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Long);
longs.remove(o);
}
assertTrue(longs.isEmpty());
/*
// Disabling this test because unlike Integer/Long, Float parsing can perform
// rounding to make values fit. See
Map<Float,String> floats = new HashMap<Float,String>();
floats.put(2.0, "2.");
floats.put(509.28, "509.28");
floats.put(86942.008, "86,942.008");
d = processAdd(chain, doc(f("id", "342"), f(fieldName, floats.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof float);
longs.remove(o);
}
*/
Map<Double,String> doubles = new HashMap<>();
doubles.put(2.0, "2.");
doubles.put(509.28, "509.28");
doubles.put(86942.008, "86,942.008");
d = processAdd(chain, doc(f("id", "342"), f(fieldName, doubles.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Double);
longs.remove(o);
}
DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Date,String> dates = new HashMap<>();
String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
dates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
}
d = processAdd(chain, doc(f("id", "343"), f(fieldName, dates.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Date);
dates.remove(o);
}
assertTrue(dates.isEmpty());
Map<Double,String> mixedLongsAndDoubles = new LinkedHashMap<>(); // preserve order
mixedLongsAndDoubles.put(85.0, "85");
mixedLongsAndDoubles.put(2.94423E-9, "2.94423E-9");
mixedLongsAndDoubles.put(2894518.0, "2,894,518");
mixedLongsAndDoubles.put(48794721.937, "48,794,721.937");
d = processAdd(chain, doc(f("id", "344"), f(fieldName, mixedLongsAndDoubles.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Double);
mixedLongsAndDoubles.remove(o);
}
assertTrue(mixedLongsAndDoubles.isEmpty());
Set<String> mixed = new HashSet<>();
mixed.add("true");
mixed.add("1682-07-22T18:33:00.000Z");
mixed.add("2,894,518");
mixed.add("308,393,131,379,900");
mixed.add("48,794,721.937");
d = processAdd(chain, doc(f("id", "345"), f(fieldName, mixed)));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof String);
}
Map<Double,Object> mixedDoubles = new LinkedHashMap<>(); // preserve order
mixedDoubles.put(85.0, "85");
mixedDoubles.put(2.94423E-9, 2.94423E-9); // Double-typed field value
mixedDoubles.put(2894518.0, "2,894,518");
mixedDoubles.put(48794721.937, "48,794,721.937");
d = processAdd(chain, doc(f("id", "3391"), f(fieldName, mixedDoubles.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Double);
mixedDoubles.remove(o);
}
assertTrue(mixedDoubles.isEmpty());
Map<Integer,Object> mixedInts = new LinkedHashMap<>(); // preserve order
mixedInts.put(85, "85");
mixedInts.put(294423, 294423); // Integer-typed field value
mixedInts.put(-2894518, "-2,894,518");
mixedInts.put(1879472193, "1,879,472,193");
d = processAdd(chain, doc(f("id", "3392"), f(fieldName, mixedInts.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Integer);
mixedInts.remove(o);
}
assertTrue(mixedInts.isEmpty());
Map<Long,Object> mixedLongs = new LinkedHashMap<>(); // preserve order
mixedLongs.put(85L, "85");
mixedLongs.put(42944233L, 42944233L); // Long-typed field value
mixedLongs.put(2894518L, "2,894,518");
mixedLongs.put(48794721937L, "48,794,721,937");
d = processAdd(chain, doc(f("id", "3393"), f(fieldName, mixedLongs.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Long);
mixedLongs.remove(o);
}
assertTrue(mixedLongs.isEmpty());
Map<Boolean,Object> mixedBooleans = new LinkedHashMap<>(); // preserve order
mixedBooleans.put(true, "true");
mixedBooleans.put(false, false); // Boolean-typed field value
mixedBooleans.put(false, "false");
mixedBooleans.put(true, "true");
d = processAdd(chain, doc(f("id", "3394"), f(fieldName, mixedBooleans.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Boolean);
mixedBooleans.remove(o);
}
assertTrue(mixedBooleans.isEmpty());
dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Date,Object> mixedDates = new HashMap<>();
dateStrings = new String[] { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
mixedDates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
}
Date extraDate = dateTimeFormatter.parseDateTime("2003-04-24").toDate();
mixedDates.put(extraDate, extraDate); // Date-typed field value
d = processAdd(chain, doc(f("id", "3395"), f(fieldName, mixedDates.values())));
assertNotNull(d);
for (Object o : d.getFieldValues(fieldName)) {
assertTrue(o instanceof Date);
mixedDates.remove(o);
}
assertTrue(mixedDates.isEmpty());
}
}