/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.index.fielddata; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.TextFieldMapper; import java.util.List; import java.util.Random; import static org.hamcrest.Matchers.equalTo; public class FilterFieldDataTests extends AbstractFieldDataTestCase { @Override protected String getFieldDataType() { return null; } public void testFilterByFrequency() throws Exception { Random random = random(); for (int i = 0; i < 1000; i++) { Document d = new Document(); d.add(new StringField("id", "" + i, Field.Store.NO)); if (i % 100 == 0) { d.add(new StringField("high_freq", "100", Field.Store.NO)); d.add(new StringField("low_freq", "100", Field.Store.NO)); d.add(new StringField("med_freq", "100", Field.Store.NO)); } if (i % 10 == 0) { d.add(new StringField("high_freq", "10", Field.Store.NO)); d.add(new StringField("med_freq", "10", Field.Store.NO)); } if (i % 5 == 0) { d.add(new StringField("high_freq", "5", Field.Store.NO)); } writer.addDocument(d); } writer.forceMerge(1, true); List<LeafReaderContext> contexts = refreshReader(); final BuilderContext builderCtx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); { ifdService.clear(); MappedFieldType ft = new TextFieldMapper.Builder("high_freq") .fielddata(true) .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0) .build(builderCtx).fieldType(); IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); } } { ifdService.clear(); MappedFieldType ft = new TextFieldMapper.Builder("high_freq") .fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, 201, 100) .build(builderCtx).fieldType(); IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(1L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5")); } } { ifdService.clear(); // test # docs with value MappedFieldType ft = new TextFieldMapper.Builder("med_freq") .fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101) .build(builderCtx).fieldType(); IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); } } { ifdService.clear(); MappedFieldType ft = new TextFieldMapper.Builder("med_freq") .fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101) .build(builderCtx).fieldType(); IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); } } } @Override public void testEmpty() throws Exception { assumeTrue("No need to test empty usage here", false); } }