/**
* Copyright 2011 LiveRamp
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.liveramp.hank.storage.curly;
import com.liveramp.hank.coordinator.Domain;
import com.liveramp.hank.coordinator.DomainVersion;
import com.liveramp.hank.coordinator.mock.MockDomain;
import com.liveramp.hank.coordinator.mock.MockDomainVersion;
import com.liveramp.hank.storage.LocalPartitionRemoteFileOps;
import com.liveramp.hank.storage.Writer;
import com.liveramp.hank.storage.cueball.CueballFilePath;
import com.liveramp.hank.storage.cueball.ICueballStreamBufferMergeSortFactory;
import com.liveramp.hank.storage.cueball.IKeyFileStreamBufferMergeSort;
import com.liveramp.hank.storage.incremental.IncrementalDomainVersionProperties;
import com.liveramp.hank.storage.incremental.IncrementalPartitionUpdaterTestCase;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TestCurlyCompactor extends IncrementalPartitionUpdaterTestCase {
private final DomainVersion v0 = new MockDomainVersion(0, 0l, new IncrementalDomainVersionProperties.Base());
private final DomainVersion v1 = new MockDomainVersion(1, 0l, new IncrementalDomainVersionProperties.Delta(0));
private final DomainVersion v2 = new MockDomainVersion(2, 0l, new IncrementalDomainVersionProperties.Delta(1));
private final Domain domain = new MockDomain("domain") {
@Override
public DomainVersion getVersion(int versionNumber) {
switch (versionNumber) {
case 0:
return v0;
case 1:
return v1;
case 2:
return v2;
default:
throw new RuntimeException("Unknown version: " + versionNumber);
}
}
};
private CurlyCompactor compactor;
private boolean mergerCalled = false;
private boolean mergeSortBufferCalled = false;
@Before
public void setUp() throws Exception {
ICurlyCompactingMerger merger = new ICurlyCompactingMerger() {
@Override
public void merge(CurlyFilePath curlyBasePath,
List<CurlyFilePath> curlyDeltas,
IKeyFileStreamBufferMergeSort keyFileStreamBufferMergeSort,
ICurlyReaderFactory curlyReaderFactory,
Writer recordFileWriter) throws IOException {
assertEquals("Correct base version", 0, curlyBasePath.getVersion());
assertEquals("Correct number of deltas used by merge", 2, curlyDeltas.size());
assertEquals("Correct delta version", 1, curlyDeltas.get(0).getVersion());
assertEquals("Correct delta version", 2, curlyDeltas.get(1).getVersion());
mergerCalled = true;
}
};
ICueballStreamBufferMergeSortFactory cueballStreamBufferMergeSortFactory =
new ICueballStreamBufferMergeSortFactory() {
@Override
public IKeyFileStreamBufferMergeSort getInstance(CueballFilePath cueballBase,
List<CueballFilePath> cueballDeltas) {
assertEquals("Correct base version", 0, cueballBase.getVersion());
assertEquals("Correct number of deltas used by merge sort", 2, cueballDeltas.size());
assertEquals("Correct delta version", 1, cueballDeltas.get(0).getVersion());
assertEquals("Correct delta version", 2, cueballDeltas.get(1).getVersion());
mergeSortBufferCalled = true;
return null;
}
};
this.compactor = new CurlyCompactor(domain,
new LocalPartitionRemoteFileOps(remotePartitionRoot, 0),
localPartitionRoot,
merger,
cueballStreamBufferMergeSortFactory,
null);
if (!new File(updateWorkRoot).mkdir()) {
throw new IOException("Failed to create update work root");
}
}
@Test
public void testUpdate() throws IOException {
// Success merging with deltas
assertFalse(existsUpdateWorkFile("00002.base.cueball"));
assertFalse(existsUpdateWorkFile("00002.base.curly"));
makeLocalFile("00000.base.cueball");
makeLocalFile("00000.base.curly");
makeLocalCacheFile("00001.delta.cueball");
makeLocalCacheFile("00001.delta.curly");
makeLocalCacheFile("00002.delta.cueball");
makeLocalCacheFile("00002.delta.curly");
// Make sure file exists on remote partition so that the versions are not considered empty
makeRemoteFile("0/00001.delta.cueball");
makeRemoteFile("0/00001.delta.curly");
makeRemoteFile("0/00002.delta.cueball");
makeRemoteFile("0/00002.delta.curly");
compactor.compact(v2, new Writer() {
@Override
public void write(ByteBuffer key, ByteBuffer value) throws IOException {
}
@Override
public void close() throws IOException {
}
@Override
public long getNumBytesWritten() {
return 0;
}
@Override
public long getNumRecordsWritten() {
return 0;
}
});
assertTrue(mergerCalled);
assertTrue(mergeSortBufferCalled);
}
}