/* * Licensed to Crate under one or more contributor license agreements. * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. Crate licenses this file * to you under the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. You may * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. * * However, if you have executed another commercial license agreement * with Crate these terms will supersede the license and you may use the * software solely pursuant to the terms of the relevant commercial * agreement. */ package io.crate.operation.collect.files; import com.google.common.collect.ImmutableMap; import io.crate.data.BatchIterator; import io.crate.data.Input; import io.crate.metadata.*; import io.crate.operation.InputFactory; import io.crate.operation.reference.file.FileLineReferenceResolver; import io.crate.test.integration.CrateUnitTest; import io.crate.testing.BatchIteratorTester; import io.crate.types.DataTypes; import org.apache.lucene.util.BytesRef; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.function.Supplier; import static io.crate.testing.TestingHelpers.createReference; public class FileReadingIteratorTest extends CrateUnitTest { private InputFactory inputFactory; private Path tempFilePath; @Before public void prepare() throws Exception { Functions functions = new Functions( ImmutableMap.<FunctionIdent, FunctionImplementation>of(), ImmutableMap.<String, FunctionResolver>of() ); inputFactory = new InputFactory(functions); tempFilePath = createTempFile(); File tmpFile = tempFilePath.toFile(); try (OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(tmpFile), StandardCharsets.UTF_8)) { writer.write("{\"name\": \"Arthur\", \"id\": 4, \"details\": {\"age\": 38}}\n"); writer.write("{\"id\": 5, \"name\": \"Trillian\", \"details\": {\"age\": 33}}\n"); } } @Test public void testIteratorContract() throws Exception { String fileUri = tempFilePath.toUri().toString(); Supplier<BatchIterator> batchIteratorSupplier = () -> createBatchIterator( Collections.singletonList(fileUri), null ); byte[] firstLine = "{\"name\": \"Arthur\", \"id\": 4, \"details\": {\"age\": 38}}".getBytes(StandardCharsets.UTF_8); byte[] secondLine = "{\"id\": 5, \"name\": \"Trillian\", \"details\": {\"age\": 33}}".getBytes(StandardCharsets.UTF_8); List<Object[]> expectedResult = Arrays.asList( new Object[]{new BytesRef(firstLine)}, new Object[]{new BytesRef(secondLine)}); BatchIteratorTester tester = new BatchIteratorTester(batchIteratorSupplier); tester.verifyResultAndEdgeCaseBehaviour(expectedResult); } private BatchIterator createBatchIterator(Collection<String> fileUris, String compression) { Reference raw = createReference("_raw", DataTypes.STRING); InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(FileLineReferenceResolver::getImplementation); List<Input<?>> inputs = Collections.singletonList(ctx.add(raw)); return FileReadingIterator.newInstance( fileUris, inputs, ctx.expressions(), compression, ImmutableMap.of( LocalFsFileInputFactory.NAME, new LocalFsFileInputFactory()), false, 1, 0 ); } }