package org.apache.lucene.index;
/**
* Copyright 2004 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
import java.io.IOException;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.IOUtils;
import com.alimama.mdrill.fdtBlockCompress.FdtCompressIndexInput;
import com.alimama.mdrill.fdtBlockCompress.FdtCompressIndexOutput;
public class FieldsWriter {
private static final Log LOG = LogFactory.getLog(FieldsWriter.class);
static final int FIELD_IS_TOKENIZED = 1 << 0;
static final int FIELD_IS_BINARY = 1 << 1;
/** @deprecated Kept for backwards-compatibility with <3.0 indexes; will be removed in 4.0 */
@Deprecated
static final int FIELD_IS_COMPRESSED = 1 << 2;
private static final int _NUMERIC_BIT_SHIFT = 3;
static final int FIELD_IS_NUMERIC_MASK = 0x07 << _NUMERIC_BIT_SHIFT;
static final int FIELD_IS_NUMERIC_INT = 1 << _NUMERIC_BIT_SHIFT;
static final int FIELD_IS_NUMERIC_LONG = 2 << _NUMERIC_BIT_SHIFT;
static final int FIELD_IS_NUMERIC_FLOAT = 3 << _NUMERIC_BIT_SHIFT;
static final int FIELD_IS_NUMERIC_DOUBLE = 4 << _NUMERIC_BIT_SHIFT;
// currently unused: static final int FIELD_IS_NUMERIC_SHORT = 5 << _NUMERIC_BIT_SHIFT;
// currently unused: static final int FIELD_IS_NUMERIC_BYTE = 6 << _NUMERIC_BIT_SHIFT;
// the next possible bits are: 1 << 6; 1 << 7
// Original format
static final int FORMAT = 0;
// Changed strings to UTF8
static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = 1;
// Lucene 3.0: Removal of compressed fields
static final int FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS = 2;
// Lucene 3.2: NumericFields are stored in binary format
static final int FORMAT_LUCENE_3_2_NUMERIC_FIELDS = 3;
// NOTE: if you introduce a new format, make it 1 higher
// than the current one, and always change this if you
// switch to a new format!
static final int FORMAT_CURRENT = FORMAT_LUCENE_3_2_NUMERIC_FIELDS;
private FieldInfos fieldInfos;
// If null - we were supplied with streams, if notnull - we manage them ourselves
private Directory directory;
private String segment;
private IndexOutput fieldsStream;
private IndexOutput indexStream;
FieldsWriter(Directory directory, String segment, FieldInfos fn) throws IOException {
this.directory = directory;
this.segment = segment;
fieldInfos = fn;
boolean success = false;
try {
IndexOutput fdt = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_EXTENSION));
indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION));
if(FieldsWriterCompress.isFdtCompress()&&!(directory instanceof RAMDirectory))
{
indexStream.writeInt(FieldsWriterCompress.FORMAT_CURRENT);
fieldsStream=new FdtCompressIndexOutput(fdt,1024*512);
fieldsStream.writeInt(FieldsWriterCompress.FORMAT_CURRENT);
}else{
fdt.writeInt(FORMAT_CURRENT);
indexStream.writeInt(FORMAT_CURRENT);
fieldsStream=fdt;
fieldsStream.writeInt(FieldsWriterCompress.FORMAT_CURRENT);
}
success = true;
} finally {
if (!success) {
abort();
}
}
}
FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) {
directory = null;
segment = null;
fieldInfos = fn;
fieldsStream = fdt;
indexStream = fdx;
}
void setFieldsStream(IndexOutput stream) {
this.fieldsStream = stream;
}
// Writes the contents of buffer into the fields stream
// and adds a new entry for this document into the index
// stream. This assumes the buffer was already written
// in the correct fields format.
void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException {
long pos=fieldsStream.getFilePointer();
// LOG.info("flushDocument:"+pos+","+numStoredFields);
indexStream.writeLong(pos);
fieldsStream.writeVInt(numStoredFields);
buffer.writeTo(fieldsStream);
}
void skipDocument() throws IOException {
long pos=fieldsStream.getFilePointer();
// LOG.info("skipDocument:"+pos);
indexStream.writeLong(pos);
fieldsStream.writeVInt(0);
}
void close() throws IOException {
if (directory != null) {
try {
IOUtils.close(fieldsStream, indexStream);
} finally {
fieldsStream = indexStream = null;
}
}
}
void abort() {
if (directory != null) {
try {
close();
} catch (IOException ignored) {
}
try {
directory.deleteFile(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_EXTENSION));
} catch (IOException ignored) {
}
try {
directory.deleteFile(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION));
} catch (IOException ignored) {
}
}
}
final void writeField(FieldInfo fi, Fieldable field) throws IOException {
fieldsStream.writeVInt(fi.number);
int bits = 0;
if (field.isTokenized())
bits |= FIELD_IS_TOKENIZED;
if (field.isBinary())
bits |= FIELD_IS_BINARY;
if (field instanceof NumericField) {
switch (((NumericField) field).getDataType()) {
case INT:
bits |= FIELD_IS_NUMERIC_INT; break;
case LONG:
bits |= FIELD_IS_NUMERIC_LONG; break;
case FLOAT:
bits |= FIELD_IS_NUMERIC_FLOAT; break;
case DOUBLE:
bits |= FIELD_IS_NUMERIC_DOUBLE; break;
default:
assert false : "Should never get here";
}
}
fieldsStream.writeByte((byte) bits);
if (field.isBinary()) {
final byte[] data;
final int len;
final int offset;
data = field.getBinaryValue();
len = field.getBinaryLength();
offset = field.getBinaryOffset();
fieldsStream.writeVInt(len);
fieldsStream.writeBytes(data, offset, len);
} else if (field instanceof NumericField) {
final NumericField nf = (NumericField) field;
final Number n = nf.getNumericValue();
switch (nf.getDataType()) {
case INT:
fieldsStream.writeVVInt(n.intValue()); break;
case LONG:
fieldsStream.writeVVLong(n.longValue()); break;
case FLOAT:
fieldsStream.writeVVVInt(Float.floatToIntBits(n.floatValue())); break;
case DOUBLE:
fieldsStream.writeVVVLong(Double.doubleToLongBits(n.doubleValue())); break;
default:
assert false : "Should never get here";
}
} else {
fieldsStream.writeString(field.stringValue());
}
}
final void addRawDocuments(final IndexInput stream, long[] lengthsstart,long[] lengthsend, int numDocs) throws IOException {
if(stream instanceof FdtCompressIndexInput)
{
FdtCompressIndexInput inputstream=(FdtCompressIndexInput)stream;
for(int i=0;i<numDocs;i++) {
long position = fieldsStream.getFilePointer();
indexStream.writeLong(position);
// LOG.info("addRawDocuments 1 "+position+","+lengthsstart[i]+","+lengthsend[i]);
inputstream.writeToPos(fieldsStream,lengthsend[i]);
}
return ;
}
if(fieldsStream instanceof FdtCompressIndexOutput)
{
for(int i=0;i<numDocs;i++) {
long position = fieldsStream.getFilePointer();
indexStream.writeLong(position);
// LOG.info("addRawDocuments 2 "+position+","+lengthsstart[i]+","+lengthsend[i]);
long end=lengthsend[i];
if(end==-1)
{
end=stream.length();
}
fieldsStream.copyBytes(stream, end-lengthsstart[i]);
}
return ;
}
long position = fieldsStream.getFilePointer();
long start = position;
for(int i=0;i<numDocs;i++) {
indexStream.writeLong(position);
// LOG.info("addRawDocuments 3 "+position+","+lengthsstart[i]+","+lengthsend[i]);
long end=lengthsend[i];
if(end==-1)
{
end=stream.length();
}
position += end-lengthsstart[i];
}
fieldsStream.copyBytes(stream, position-start);
}
final void addDocument(Document doc) throws IOException {
long pos=fieldsStream.getFilePointer();
indexStream.writeLong(pos);
// LOG.info("addDocument "+pos);
int storedCount = 0;
List<Fieldable> fields = doc.getFields();
for (Fieldable field : fields) {
if (field.isStored())
storedCount++;
}
fieldsStream.writeVInt(storedCount);
for (Fieldable field : fields) {
if (field.isStored())
{
writeField(fieldInfos.fieldInfo(field.name()), field);
}
}
}
}