package org.apache.solr.handler.component;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
/**
* Statistics Component Test
*/
public class StatsComponentTest extends AbstractSolrTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema11.xml");
}
@Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
lrf = h.getRequestFactory("standard", 0, 20);
}
public void testStats() throws Exception {
for (String f : new String[] {
"stats_i","stats_l","stats_f","stats_d",
"stats_ti","stats_tl","stats_tf","stats_td"
}) {
doTestFieldStatisticsResult(f);
doTestFieldStatisticsMissingResult(f);
doTestFacetStatisticsResult(f);
doTestFacetStatisticsMissingResult(f);
}
for (String f : new String[] {"stats_ii", // plain int
"stats_is", // sortable int
"stats_tis","stats_tfs","stats_tls","stats_tds" // trie fields
}) {
doTestMVFieldStatisticsResult(f);
}
}
public void doTestFieldStatisticsResult(String f) throws Exception {
assertU(adoc("id", "1", f, "-10"));
assertU(adoc("id", "2", f, "-20"));
assertU(commit());
assertU(adoc("id", "3", f, "-30"));
assertU(adoc("id", "4", f, "-40"));
assertU(commit());
assertQ("test statistics values", req("q", "*:*", "stats", "true", "stats.field", f, "stats.calcdistinct", "true")
, "//double[@name='min'][.='-40.0']"
, "//double[@name='max'][.='-10.0']"
, "//double[@name='sum'][.='-100.0']"
, "//long[@name='count'][.='4']"
, "//long[@name='missing'][.='0']"
, "//long[@name='countDistinct'][.='4']"
, "count(//arr[@name='distinctValues']/*)='4'"
, "//double[@name='sumOfSquares'][.='3000.0']"
, "//double[@name='mean'][.='-25.0']"
, "//double[@name='stddev'][.='12.909944487358056']"
);
}
public void doTestMVFieldStatisticsResult(String f) throws Exception {
assertU(adoc("id", "1", f, "-10", f, "-100", "active_s", "true"));
assertU(adoc("id", "2", f, "-20", f, "200", "active_s", "true"));
assertU(adoc("id", "3", f, "-30", f, "-1", "active_s", "false"));
assertU(adoc("id", "4", f, "-40", f, "10", "active_s", "false"));
assertU(adoc("id", "5", "active_s", "false"));
assertU(commit());
assertQ("test statistics values", req("q", "*:*", "stats", "true", "stats.field", f, "stats.calcdistinct", "true")
, "//double[@name='min'][.='-100.0']"
, "//double[@name='max'][.='200.0']"
, "//double[@name='sum'][.='9.0']"
, "//long[@name='count'][.='8']"
, "//long[@name='missing'][.='1']"
, "//long[@name='countDistinct'][.='8']"
, "count(//arr[@name='distinctValues']/*)='8'"
, "//double[@name='sumOfSquares'][.='53101.0']"
, "//double[@name='mean'][.='1.125']"
, "//double[@name='stddev'][.='87.08852228787508']"
);
assertQ("test statistics values", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.calcdistinct", "true")
, "//double[@name='min'][.='-100.0']"
, "//double[@name='max'][.='200.0']"
, "//double[@name='sum'][.='9.0']"
, "//long[@name='count'][.='8']"
, "//long[@name='missing'][.='1']"
, "//long[@name='countDistinct'][.='8']"
, "count(//lst[@name='" + f + "']/arr[@name='distinctValues']/*)='8'"
, "//double[@name='sumOfSquares'][.='53101.0']"
, "//double[@name='mean'][.='1.125']"
, "//double[@name='stddev'][.='87.08852228787508']"
);
assertQ("test value for active_s=true", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.calcdistinct", "true")
, "//lst[@name='true']/double[@name='min'][.='-100.0']"
, "//lst[@name='true']/double[@name='max'][.='200.0']"
, "//lst[@name='true']/double[@name='sum'][.='70.0']"
, "//lst[@name='true']/long[@name='count'][.='4']"
, "//lst[@name='true']/long[@name='missing'][.='0']"
, "//lst[@name='true']//long[@name='countDistinct'][.='4']"
, "count(//lst[@name='true']/arr[@name='distinctValues']/*)='4'"
, "//lst[@name='true']/double[@name='sumOfSquares'][.='50500.0']"
, "//lst[@name='true']/double[@name='mean'][.='17.5']"
, "//lst[@name='true']/double[@name='stddev'][.='128.16005617976296']"
);
assertQ("test value for active_s=false", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.calcdistinct", "true", "indent", "true")
, "//lst[@name='false']/double[@name='min'][.='-40.0']"
, "//lst[@name='false']/double[@name='max'][.='10.0']"
, "//lst[@name='false']/double[@name='sum'][.='-61.0']"
, "//lst[@name='false']/long[@name='count'][.='4']"
, "//lst[@name='false']/long[@name='missing'][.='1']"
, "//lst[@name='true']//long[@name='countDistinct'][.='4']"
, "count(//lst[@name='true']/arr[@name='distinctValues']/*)='4'"
, "//lst[@name='false']/double[@name='sumOfSquares'][.='2601.0']"
, "//lst[@name='false']/double[@name='mean'][.='-15.25']"
, "//lst[@name='false']/double[@name='stddev'][.='23.59908190304586']"
);
}
public void testFieldStatisticsResultsStringField() throws Exception {
SolrCore core = h.getCore();
assertU(adoc("id", "1", "active_s", "string1"));
assertU(adoc("id", "2", "active_s", "string2"));
assertU(adoc("id", "3", "active_s", "string3"));
assertU(adoc("id", "4"));
assertU(commit());
Map<String, String> args = new HashMap<>();
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, "active_s");
args.put("f.active_s.stats.calcdistinct","true");
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
assertQ("test string statistics values", req,
"//str[@name='min'][.='string1']",
"//str[@name='max'][.='string3']",
"//long[@name='count'][.='3']",
"//long[@name='missing'][.='1']",
"//long[@name='countDistinct'][.='3']",
"count(//arr[@name='distinctValues']/str)='3'");
}
public void testFieldStatisticsResultsDateField() throws Exception {
SolrCore core = h.getCore();
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT);
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String date1 = dateFormat.format(new Date(123456789)) + "Z";
String date2 = dateFormat.format(new Date(987654321)) + "Z";
assertU(adoc("id", "1", "active_dt", date1));
assertU(adoc("id", "2", "active_dt", date2));
assertU(adoc("id", "3"));
assertU(commit());
Map<String, String> args = new HashMap<>();
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, "active_dt");
args.put("f.active_dt.stats.calcdistinct","true");
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
assertQ("test date statistics values", req,
"//long[@name='count'][.='2']",
"//long[@name='missing'][.='1']",
"//date[@name='min'][.='1970-01-02T10:17:36Z']",
"//date[@name='max'][.='1970-01-12T10:20:54Z']",
"//long[@name='countDistinct'][.='2']",
"count(//arr[@name='distinctValues']/date)='2'"
// "//date[@name='sum'][.='1970-01-13T20:38:30Z']", // sometimes 29.999Z
// "//date[@name='mean'][.='1970-01-07T10:19:15Z']" // sometiems 14.999Z
);
}
public void doTestFieldStatisticsMissingResult(String f) throws Exception {
assertU(adoc("id", "1", f, "-10"));
assertU(adoc("id", "2", f, "-20"));
assertU(commit());
assertU(adoc("id", "3"));
assertU(adoc("id", "4", f, "-40"));
assertU(commit());
assertQ("test statistics values", req("q", "*:*", "stats", "true", "stats.field", f, "stats.calcdistinct", "true")
, "//double[@name='min'][.='-40.0']"
, "//double[@name='max'][.='-10.0']"
, "//double[@name='sum'][.='-70.0']"
, "//long[@name='count'][.='3']"
, "//long[@name='missing'][.='1']"
, "//long[@name='countDistinct'][.='3']"
, "count(//arr[@name='distinctValues']/*)='3'"
, "//double[@name='sumOfSquares'][.='2100.0']"
, "//double[@name='mean'][.='-23.333333333333332']"
, "//double[@name='stddev'][.='15.275252316519467']"
);
}
public void doTestFacetStatisticsResult(String f) throws Exception {
assertU(adoc("id", "1", f, "10", "active_s", "true", "other_s", "foo"));
assertU(adoc("id", "2", f, "20", "active_s", "true", "other_s", "bar"));
assertU(commit());
assertU(adoc("id", "3", f, "30", "active_s", "false", "other_s", "foo"));
assertU(adoc("id", "4", f, "40", "active_s", "false", "other_s", "foo"));
assertU(commit());
final String pre = "//lst[@name='stats_fields']/lst[@name='"+f+"']/lst[@name='facets']/lst[@name='active_s']";
assertQ("test value for active_s=true", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.facet", "other_s", "stats.calcdistinct", "true", "indent", "true")
, "*[count("+pre+")=1]"
, pre+"/lst[@name='true']/double[@name='min'][.='10.0']"
, pre+"/lst[@name='true']/double[@name='max'][.='20.0']"
, pre+"/lst[@name='true']/double[@name='sum'][.='30.0']"
, pre+"/lst[@name='true']/long[@name='count'][.='2']"
, pre+"/lst[@name='true']/long[@name='missing'][.='0']"
, pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']"
, "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)='2'"
, pre+"/lst[@name='true']/double[@name='sumOfSquares'][.='500.0']"
, pre+"/lst[@name='true']/double[@name='mean'][.='15.0']"
, pre+"/lst[@name='true']/double[@name='stddev'][.='7.0710678118654755']"
);
assertQ("test value for active_s=false", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.calcdistinct", "true")
, pre+"/lst[@name='false']/double[@name='min'][.='30.0']"
, pre+"/lst[@name='false']/double[@name='max'][.='40.0']"
, pre+"/lst[@name='false']/double[@name='sum'][.='70.0']"
, pre+"/lst[@name='false']/long[@name='count'][.='2']"
, pre+"/lst[@name='false']/long[@name='missing'][.='0']"
, pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']"
, "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)='2'"
, pre+"/lst[@name='false']/double[@name='sumOfSquares'][.='2500.0']"
, pre+"/lst[@name='false']/double[@name='mean'][.='35.0']"
, pre+"/lst[@name='false']/double[@name='stddev'][.='7.0710678118654755']"
);
}
public void doTestFacetStatisticsMissingResult(String f) throws Exception {
assertU(adoc("id", "1", f, "10", "active_s", "true"));
assertU(adoc("id", "2", f, "20", "active_s", "true"));
assertU(commit());
assertU(adoc("id", "3", "active_s", "false"));
assertU(adoc("id", "4", f, "40", "active_s", "false"));
assertU(commit());
assertQ("test value for active_s=true", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.calcdistinct", "true")
, "//lst[@name='true']/double[@name='min'][.='10.0']"
, "//lst[@name='true']/double[@name='max'][.='20.0']"
, "//lst[@name='true']/double[@name='sum'][.='30.0']"
, "//lst[@name='true']/long[@name='count'][.='2']"
, "//lst[@name='true']/long[@name='missing'][.='0']"
, "//lst[@name='true']/long[@name='countDistinct'][.='2']"
, "count(//lst[@name='true']/arr[@name='distinctValues']/*)='2'"
, "//lst[@name='true']/double[@name='sumOfSquares'][.='500.0']"
, "//lst[@name='true']/double[@name='mean'][.='15.0']"
, "//lst[@name='true']/double[@name='stddev'][.='7.0710678118654755']"
);
assertQ("test value for active_s=false", req("q", "*:*", "stats", "true", "stats.field", f, "stats.facet", "active_s", "stats.calcdistinct", "true")
, "//lst[@name='false']/double[@name='min'][.='40.0']"
, "//lst[@name='false']/double[@name='max'][.='40.0']"
, "//lst[@name='false']/double[@name='sum'][.='40.0']"
, "//lst[@name='false']/long[@name='count'][.='1']"
, "//lst[@name='false']/long[@name='missing'][.='1']"
, "//lst[@name='false']/long[@name='countDistinct'][.='1']"
, "count(//lst[@name='false']/arr[@name='distinctValues']/*)='1'"
, "//lst[@name='false']/double[@name='sumOfSquares'][.='1600.0']"
, "//lst[@name='false']/double[@name='mean'][.='40.0']"
, "//lst[@name='false']/double[@name='stddev'][.='0.0']"
);
}
public void testFieldStatisticsResultsNumericFieldAlwaysMissing() throws Exception {
SolrCore core = h.getCore();
assertU(adoc("id", "1"));
assertU(adoc("id", "2"));
assertU(commit());
assertU(adoc("id", "3"));
assertU(adoc("id", "4"));
assertU(commit());
Map<String, String> args = new HashMap<>();
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, "active_i");
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
assertQ("test string statistics values", req,
"//null[@name='active_i'][.='']");
}
public void testFieldStatisticsResultsStringFieldAlwaysMissing() throws Exception {
SolrCore core = h.getCore();
assertU(adoc("id", "1"));
assertU(adoc("id", "2"));
assertU(commit());
assertU(adoc("id", "3"));
assertU(adoc("id", "4"));
assertU(commit());
Map<String, String> args = new HashMap<>();
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, "active_s");
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
assertQ("test string statistics values", req,
"//null[@name='active_s'][.='']");
}
//SOLR-3160
public void testFieldStatisticsResultsDateFieldAlwaysMissing() throws Exception {
SolrCore core = h.getCore();
assertU(adoc("id", "1"));
assertU(adoc("id", "2"));
assertU(commit());
assertU(adoc("id", "3"));
assertU(commit());
Map<String, String> args = new HashMap<>();
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
args.put(StatsParams.STATS_FIELD, "active_dt");
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
assertQ("test string statistics values", req,
"//null[@name='active_dt'][.='']");
}
public void testStatsFacetMultivaluedErrorHandling() throws Exception {
SolrCore core = h.getCore();
SchemaField foo_ss = core.getLatestSchema().getField("foo_ss");
assertU(adoc("id", "1", "active_i", "1", "foo_ss", "aa" ));
assertU(commit());
assertU(adoc("id", "2", "active_i", "1", "foo_ss", "bb" ));
assertU(adoc("id", "3", "active_i", "5", "foo_ss", "aa" ));
assertU(commit());
assertTrue("schema no longer satisfies test requirements: foo_ss no longer multivalued", foo_ss.multiValued());
assertTrue("schema no longer satisfies test requirements: foo_ss's fieldtype no longer single valued", ! foo_ss.getType().isMultiValued());
assertQEx("no failure trying to get stats facet on foo_ss",
req("q", "*:*",
"stats", "true",
"stats.field", "active_i",
"stats.facet", "foo_ss"),
400);
}
}