+ * This is useful to pre-allocate the output buffer before calling
+ * {@link #headlessCompress(int[], IntWrapper, int, int[], IntWrapper)}.
+ *
+ *
+ * @param compressedPositions
+ * since not all schemes compress every input integer, this parameter
+ * returns how many input integers will actually be compressed.
+ * This is useful when composing multiple schemes.
+ * @param inlength
+ * number of integers to be compressed
+ * @return the maximum number of integers needed in the output array
+ */
+ int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength);
}
diff --git a/src/main/java/me/lemire/integercompression/UncompressibleInputException.java b/src/main/java/me/lemire/integercompression/UncompressibleInputException.java
deleted file mode 100644
index c490946..0000000
--- a/src/main/java/me/lemire/integercompression/UncompressibleInputException.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package me.lemire.integercompression;
-
-/**
- * This exception might be thrown if the input is poorly compressible.
- *
- */
-public class UncompressibleInputException extends RuntimeException {
-
- /**
- * Create new exception
- * @param string explanation for the exception
- */
- public UncompressibleInputException(String string) {
- super(string);
- }
-
- private static final long serialVersionUID = -798583799846489873L;
-
-}
diff --git a/src/main/java/me/lemire/integercompression/Util.java b/src/main/java/me/lemire/integercompression/Util.java
index 346e3b2..63fc918 100644
--- a/src/main/java/me/lemire/integercompression/Util.java
+++ b/src/main/java/me/lemire/integercompression/Util.java
@@ -15,13 +15,13 @@
public final class Util {
-
- // check whether x is small than y as unsigned ints (supported by Java 8 natively);
- protected static final boolean smallerorequalthan(int x, int y) {
- return (x + Integer.MIN_VALUE) <= (y + Integer.MIN_VALUE);
- }
-
- /**
+
+ // check whether x is small than y as unsigned ints (supported by Java 8 natively);
+ protected static final boolean smallerorequalthan(int x, int y) {
+ return (x + Integer.MIN_VALUE) <= (y + Integer.MIN_VALUE);
+ }
+
+ /**
* Compute the maximum of the integer logarithms (ceil(log(x+1)) of a range
* of value
*
diff --git a/src/main/java/me/lemire/integercompression/VariableByte.java b/src/main/java/me/lemire/integercompression/VariableByte.java
index 09e479b..c9b04d0 100644
--- a/src/main/java/me/lemire/integercompression/VariableByte.java
+++ b/src/main/java/me/lemire/integercompression/VariableByte.java
@@ -21,6 +21,8 @@
*/
public class VariableByte implements IntegerCODEC, ByteIntegerCODEC, SkippableIntegerCODEC {
+ private static final int MAX_BYTES_PER_INT = 5;
+
private static byte extract7bits(int i, long val) {
return (byte) ((val >> (7 * i)) & ((1 << 7) - 1));
}
@@ -208,12 +210,23 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] o
inpos.set(p + (s!=0 ? 1 : 0));
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int maxLengthInBytes = inlength * MAX_BYTES_PER_INT;
+ int maxLengthInInts = (maxLengthInBytes + Integer.BYTES - 1) / Integer.BYTES;
+ compressedPositions.add(inlength);
+ return maxLengthInInts;
+ }
+
/**
* Creates a new buffer of the requested size.
*
* In case you need a different way to allocate buffers, you can override this method
* with a custom behavior. The default implementation allocates a new Java direct
* {@link ByteBuffer} on each invocation.
+ *
+ * @param sizeInBytes
+ * @return
*/
protected ByteBuffer makeBuffer(int sizeInBytes) {
return ByteBuffer.allocateDirect(sizeInBytes);
diff --git a/src/main/java/me/lemire/integercompression/benchmarktools/Benchmark.java b/src/main/java/me/lemire/integercompression/benchmarktools/Benchmark.java
index c5fee69..ef4a386 100644
--- a/src/main/java/me/lemire/integercompression/benchmarktools/Benchmark.java
+++ b/src/main/java/me/lemire/integercompression/benchmarktools/Benchmark.java
@@ -308,10 +308,10 @@ private static void testByteCodec(PrintWriter csvLog, int sparsity,
public static void main(String args[]) throws FileNotFoundException {
System.out
.println("# benchmark based on the ClusterData model from:");
- System.out.println("# Vo Ngoc Anh and Alistair Moffat. ");
- System.out.println("# Index compression using 64-bit words.");
+ System.out.println("# Vo Ngoc Anh and Alistair Moffat. ");
+ System.out.println("# Index compression using 64-bit words.");
System.out
- .println("# Softw. Pract. Exper.40, 2 (February 2010), 131-147. ");
+ .println("# Softw. Pract. Exper.40, 2 (February 2010), 131-147. ");
System.out.println();
PrintWriter writer = null;
diff --git a/src/main/java/me/lemire/integercompression/benchmarktools/BenchmarkSkippable.java b/src/main/java/me/lemire/integercompression/benchmarktools/BenchmarkSkippable.java
index 58bbc4a..b930568 100644
--- a/src/main/java/me/lemire/integercompression/benchmarktools/BenchmarkSkippable.java
+++ b/src/main/java/me/lemire/integercompression/benchmarktools/BenchmarkSkippable.java
@@ -83,7 +83,6 @@ private static int decompressFromSkipTable(Object c, int[] compressed,
if (num > length - uncomppos.get())
num = length - uncomppos.get();
int location = metadata[metapos++];
- // System.out.println("location = "+location);
int initvalue = metadata[metapos++];
int outputlocation = uncomppos.get();
if (location != compressedpos.get())
@@ -242,10 +241,10 @@ private static void testCodec(PrintWriter csvLog, int sparsity, Object c,
*/
public static void main(String args[]) throws FileNotFoundException {
System.out.println("# benchmark based on the ClusterData model from:");
- System.out.println("# Vo Ngoc Anh and Alistair Moffat. ");
- System.out.println("# Index compression using 64-bit words.");
+ System.out.println("# Vo Ngoc Anh and Alistair Moffat. ");
+ System.out.println("# Index compression using 64-bit words.");
System.out
- .println("# Softw. Pract. Exper.40, 2 (February 2010), 131-147. ");
+ .println("# Softw. Pract. Exper.40, 2 (February 2010), 131-147. ");
System.out.println();
PrintWriter writer = null;
diff --git a/src/main/java/me/lemire/integercompression/differential/IntegratedBinaryPacking.java b/src/main/java/me/lemire/integercompression/differential/IntegratedBinaryPacking.java
index 7e1c161..f50a367 100644
--- a/src/main/java/me/lemire/integercompression/differential/IntegratedBinaryPacking.java
+++ b/src/main/java/me/lemire/integercompression/differential/IntegratedBinaryPacking.java
@@ -49,7 +49,8 @@
public class IntegratedBinaryPacking implements IntegratedIntegerCODEC,
SkippableIntegratedIntegerCODEC {
- static final int BLOCK_SIZE = 32;
+ public static final int BLOCK_SIZE = 32;
+ private static final int MAX_BIT_WIDTH = Integer.SIZE;
@Override
public void compress(int[] in, IntWrapper inpos, int inlength, int[] out,
@@ -170,4 +171,13 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
initvalue.set(initoffset);
inpos.set(tmpinpos);
}
+
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int blockCount = inlength / BLOCK_SIZE;
+ int headersSizeInInts = blockCount / Integer.BYTES + (blockCount % Integer.BYTES);
+ int blocksSizeInInts = blockCount * MAX_BIT_WIDTH;
+ compressedPositions.add(blockCount * BLOCK_SIZE);
+ return headersSizeInInts + blocksSizeInInts;
+ }
}
diff --git a/src/main/java/me/lemire/integercompression/differential/IntegratedIntCompressor.java b/src/main/java/me/lemire/integercompression/differential/IntegratedIntCompressor.java
index 5808bdd..1d935c4 100644
--- a/src/main/java/me/lemire/integercompression/differential/IntegratedIntCompressor.java
+++ b/src/main/java/me/lemire/integercompression/differential/IntegratedIntCompressor.java
@@ -3,7 +3,6 @@
import java.util.Arrays;
import me.lemire.integercompression.IntWrapper;
-import me.lemire.integercompression.UncompressibleInputException;
/**
* This is a convenience class that wraps a codec to provide
@@ -36,19 +35,14 @@ public IntegratedIntCompressor() {
*
* @param input array to be compressed
* @return compressed array
- * @throws UncompressibleInputException if the data is too poorly compressible
*/
public int[] compress(int[] input) {
- int [] compressed = new int[input.length + input.length / 100 + 1024];
+ int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length);
+ int [] compressed = new int[maxCompressedLength + 1]; // +1 to store the length of the input
compressed[0] = input.length;
IntWrapper outpos = new IntWrapper(1);
IntWrapper initvalue = new IntWrapper(0);
- try {
- codec.headlessCompress(input, new IntWrapper(0), input.length, compressed, outpos, initvalue);
- } catch (IndexOutOfBoundsException ioebe) {
- throw new UncompressibleInputException(
- "Your input is too poorly compressible with the current codec : " + codec);
- }
+ codec.headlessCompress(input, new IntWrapper(0), input.length, compressed, outpos, initvalue);
compressed = Arrays.copyOf(compressed,outpos.intValue());
return compressed;
}
diff --git a/src/main/java/me/lemire/integercompression/differential/IntegratedVariableByte.java b/src/main/java/me/lemire/integercompression/differential/IntegratedVariableByte.java
index 918a900..a577031 100644
--- a/src/main/java/me/lemire/integercompression/differential/IntegratedVariableByte.java
+++ b/src/main/java/me/lemire/integercompression/differential/IntegratedVariableByte.java
@@ -24,6 +24,8 @@
public class IntegratedVariableByte implements IntegratedIntegerCODEC, IntegratedByteIntegerCODEC,
SkippableIntegratedIntegerCODEC {
+ private static final int MAX_BYTES_PER_INT = 5;
+
private static byte extract7bits(int i, long val) {
return (byte)((val >> (7 * i)) & ((1 << 7) - 1));
}
@@ -257,6 +259,14 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
inpos.set(p + (s!=0 ? 1 : 0));
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int maxLengthInBytes = inlength * MAX_BYTES_PER_INT;
+ int maxLengthInInts = (maxLengthInBytes + Integer.BYTES - 1) / Integer.BYTES;
+ compressedPositions.add(inlength);
+ return maxLengthInInts;
+ }
+
/**
* Creates a new buffer of the requested size.
*
diff --git a/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedComposition.java b/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedComposition.java
index 09c4dd8..4786ec5 100644
--- a/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedComposition.java
+++ b/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedComposition.java
@@ -66,14 +66,25 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
if (inlength == 0)
return;
int init = inpos.get();
+ int outposInit = outpos.get();
+
F1.headlessUncompress(in, inpos, inlength, out, outpos,num,initvalue);
if (inpos.get() == init) {
- inpos.increment();
+ inpos.increment();
}
inlength -= inpos.get() - init;
- num -= outpos.get();
+ num -= outpos.get() - outposInit;
F2.headlessUncompress(in, inpos, inlength, out, outpos,num,initvalue);
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int init = compressedPositions.get();
+ int maxLength = F1.maxHeadlessCompressedLength(compressedPositions, inlength);
+ maxLength += 1; // Add +1 for the potential F2 header. Question: is this header actually needed in the headless version?
+ inlength -= compressedPositions.get() - init;
+ maxLength += F2.maxHeadlessCompressedLength(compressedPositions, inlength);
+ return maxLength;
+ }
}
diff --git a/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedIntegerCODEC.java b/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedIntegerCODEC.java
index 8b7fd4b..e2df754 100644
--- a/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedIntegerCODEC.java
+++ b/src/main/java/me/lemire/integercompression/differential/SkippableIntegratedIntegerCODEC.java
@@ -71,4 +71,21 @@ public void headlessCompress(int[] in, IntWrapper inpos, int inlength, int[] out
public void headlessUncompress(int[] in, IntWrapper inpos, int inlength, int[] out,
IntWrapper outpos, int num, IntWrapper initvalue);
+ /**
+ * Compute the maximum number of integers that might be required to store
+ * the compressed form of a given input array segment, without headers.
+ *
+ * This is useful to pre-allocate the output buffer before calling
+ * {@link #headlessCompress(int[], IntWrapper, int, int[], IntWrapper, IntWrapper)}.
+ *
+ *
+ * @param compressedPositions
+ * since not all schemes compress every input integer, this parameter
+ * returns how many input integers will actually be compressed.
+ * This is useful when composing multiple schemes.
+ * @param inlength
+ * number of integers to be compressed
+ * @return the maximum number of integers needed in the output array
+ */
+ int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength);
}
diff --git a/src/main/java/me/lemire/integercompression/synth/UniformDataGenerator.java b/src/main/java/me/lemire/integercompression/synth/UniformDataGenerator.java
index bbd386a..a50497c 100644
--- a/src/main/java/me/lemire/integercompression/synth/UniformDataGenerator.java
+++ b/src/main/java/me/lemire/integercompression/synth/UniformDataGenerator.java
@@ -42,7 +42,7 @@ int[] generateUniformHash(int N, int Max) {
int[] ans = new int[N];
HashSet s = new HashSet();
while (s.size() < N)
- s.add(new Integer(this.rand.nextInt(Max)));
+ s.add(this.rand.nextInt(Max));
Iterator i = s.iterator();
for (int k = 0; k < N; ++k)
ans[k] = i.next().intValue();
diff --git a/src/main/java/me/lemire/integercompression/vector/VectorFastPFOR.java b/src/main/java/me/lemire/integercompression/vector/VectorFastPFOR.java
index 0b6ca17..7374fa5 100644
--- a/src/main/java/me/lemire/integercompression/vector/VectorFastPFOR.java
+++ b/src/main/java/me/lemire/integercompression/vector/VectorFastPFOR.java
@@ -229,6 +229,11 @@ public void headlessUncompress(int[] in, IntWrapper inpos, int inlength,
}
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ throw new UnsupportedOperationException("Calculating the max compressed length is not supported yet.");
+ }
+
private void loadMetaData(int[] in, int inexcept, int bytesize) {
// Arrays.fill(bem, (byte)0);
int len = (bytesize + 3) / 4;
diff --git a/src/main/java/me/lemire/longcompression/ByteLongCODEC.java b/src/main/java/me/lemire/longcompression/ByteLongCODEC.java
index e405370..dbc6864 100644
--- a/src/main/java/me/lemire/longcompression/ByteLongCODEC.java
+++ b/src/main/java/me/lemire/longcompression/ByteLongCODEC.java
@@ -57,6 +57,6 @@ public void compress(long[] in, IntWrapper inpos, int inlength,
* where to write the compressed output in out
*/
public void uncompress(byte[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos);
+ long[] out, IntWrapper outpos);
}
diff --git a/src/main/java/me/lemire/longcompression/LongAs2IntsCodec.java b/src/main/java/me/lemire/longcompression/LongAs2IntsCodec.java
index 3b2bc76..35c1166 100644
--- a/src/main/java/me/lemire/longcompression/LongAs2IntsCodec.java
+++ b/src/main/java/me/lemire/longcompression/LongAs2IntsCodec.java
@@ -16,174 +16,174 @@
*
*/
public class LongAs2IntsCodec implements LongCODEC {
- final IntegerCODEC highPartsCodec;
- final IntegerCODEC lowPartsCodec;
-
- public LongAs2IntsCodec(IntegerCODEC highPartsCodec, IntegerCODEC lowPartsCodec) {
- this.highPartsCodec = highPartsCodec;
- this.lowPartsCodec = lowPartsCodec;
- }
-
- /**
- * By default, we expect longs to be slightly above Integer.MAX_VALUE. Hence highParts to be small and positive
- * integers. For lowParts, we rely on {@link IntCompressor} default IntegerCODEC
- */
- public LongAs2IntsCodec() {
- this(new VariableByte(), new Composition(new BinaryPacking(), new VariableByte()));
- }
-
- @Override
- public void compress(long[] in, IntWrapper inpos, int inlength, long[] out, IntWrapper outpos) {
- if (inlength == 0) {
- return;
- }
-
- int[] highParts = new int[inlength];
- int[] lowParts = new int[inlength];
-
- for (int i = 0; i < inlength; i++) {
- int inPosition = inpos.get() + i;
-
- highParts[i] = RoaringIntPacking.high(in[inPosition]);
- lowParts[i] = RoaringIntPacking.low(in[inPosition]);
- }
-
- // TODO What would be a relevant buffer size?
- int[] buffer = new int[inlength * 16];
-
- int outPosition = outpos.get();
-
- boolean hasLeftover;
- {
- // The first integer is reserved to hold the number of compressed ints
- IntWrapper highPartsOutPosition = new IntWrapper(1);
-
- highPartsCodec.compress(highParts, new IntWrapper(), inlength, buffer, highPartsOutPosition);
-
- // Record the compressedHighparts length
- buffer[0] = highPartsOutPosition.get() - 1;
-
- for (int i = 0; i < highPartsOutPosition.get() / 2; i++) {
- long pack = RoaringIntPacking.pack(buffer[i * 2], buffer[i * 2 + 1]);
- out[outPosition++] = pack;
- }
-
- if (1 == highPartsOutPosition.get() % 2) {
- // Shift the trailing integer as first in the buffer
- hasLeftover = true;
- buffer[0] = buffer[highPartsOutPosition.get() - 1];
- } else {
- hasLeftover = false;
- }
- }
-
- {
- // The first integer is reserved to hold the number of compressed ints
- IntWrapper lowPartsOutPosition = new IntWrapper(1);
- if (hasLeftover) {
- // Keep the trailing int from highParts before the reserved int from lowParts compressed length
- lowPartsOutPosition.set(2);
- }
-
- lowPartsCodec.compress(lowParts, new IntWrapper(0), inlength, buffer, lowPartsOutPosition);
-
- // Record the compressedHighparts length
- buffer[hasLeftover ? 1 : 0] = lowPartsOutPosition.get() - (hasLeftover ? 2 : 1);
-
- for (int i = 0; i < lowPartsOutPosition.get() / 2; i++) {
- long pack = RoaringIntPacking.pack(buffer[i * 2], buffer[i * 2 + 1]);
- out[outPosition++] = pack;
- }
-
- if (1 == lowPartsOutPosition.get() % 2) {
- // The trailing integer is packed with a 0
- long pack = RoaringIntPacking.pack(buffer[lowPartsOutPosition.get() - 1], 0);
- out[outPosition++] = pack;
- }
- }
-
- inpos.add(inlength);
- outpos.set(outPosition);
- }
-
- /**
- * inlength is ignored by this codec. We may rely on it instead of storing the compressedLowPart length
- */
- @Override
- public void uncompress(long[] in, IntWrapper inpos, int inlength, long[] out, IntWrapper outpos) {
- if (inlength == 0) {
- return;
- }
-
- int longIndex = inpos.get();
-
- int nbCompressedHighParts = RoaringIntPacking.high(in[longIndex]);
- int[] compressedHighParts = new int[nbCompressedHighParts];
-
- // !highPart as we just read the highPart for nbCompressedHighParts
- boolean highPart = false;
- for (int i = 0; i < nbCompressedHighParts; i++) {
- int nextInt;
- if (highPart) {
- nextInt = RoaringIntPacking.high(in[longIndex + (i + 1) / 2]);
- } else {
- nextInt = RoaringIntPacking.low(in[longIndex + (i + 1) / 2]);
- }
- compressedHighParts[i] = nextInt;
-
- highPart = !highPart;
- }
-
- // TODO What would be a relevant buffer size?
- int[] buffer = new int[inlength * 16];
-
- IntWrapper highPartsOutPosition = new IntWrapper();
- highPartsCodec.uncompress(compressedHighParts,
- new IntWrapper(),
- compressedHighParts.length,
- buffer,
- highPartsOutPosition);
- int[] highParts = Arrays.copyOf(buffer, highPartsOutPosition.get());
-
- // +1 as we initially read nbCompressedHighParts
- int intIndexNbCompressedLowParts = longIndex * 2 + 1 + nbCompressedHighParts;
- int nbCompressedLowParts;
- if (highPart) {
- nbCompressedLowParts = RoaringIntPacking.high(in[intIndexNbCompressedLowParts / 2]);
- } else {
- nbCompressedLowParts = RoaringIntPacking.low(in[intIndexNbCompressedLowParts / 2]);
- }
- highPart = !highPart;
-
- int[] compressedLowParts = new int[nbCompressedLowParts];
- for (int i = 0; i < nbCompressedLowParts; i++) {
- int nextInt;
- if (highPart) {
- nextInt = RoaringIntPacking.high(in[(intIndexNbCompressedLowParts + 1 + i) / 2]);
- } else {
- nextInt = RoaringIntPacking.low(in[(intIndexNbCompressedLowParts + 1 + i) / 2]);
- }
- compressedLowParts[i] = nextInt;
-
- highPart = !highPart;
- }
-
- IntWrapper lowPartsOutPosition = new IntWrapper();
- lowPartsCodec.uncompress(compressedLowParts,
- new IntWrapper(),
- compressedLowParts.length,
- buffer,
- lowPartsOutPosition);
- int[] lowParts = Arrays.copyOf(buffer, lowPartsOutPosition.get());
- assert highParts.length == lowParts.length;
-
- int outposition = outpos.get();
- for (int i = 0; i < highParts.length; i++) {
- out[outposition++] = RoaringIntPacking.pack(highParts[i], lowParts[i]);
- }
-
- inpos.add(inlength);
- outpos.set(outposition);
- }
+ final IntegerCODEC highPartsCodec;
+ final IntegerCODEC lowPartsCodec;
+
+ public LongAs2IntsCodec(IntegerCODEC highPartsCodec, IntegerCODEC lowPartsCodec) {
+ this.highPartsCodec = highPartsCodec;
+ this.lowPartsCodec = lowPartsCodec;
+ }
+
+ /**
+ * By default, we expect longs to be slightly above Integer.MAX_VALUE. Hence highParts to be small and positive
+ * integers. For lowParts, we rely on {@link IntCompressor} default IntegerCODEC
+ */
+ public LongAs2IntsCodec() {
+ this(new VariableByte(), new Composition(new BinaryPacking(), new VariableByte()));
+ }
+
+ @Override
+ public void compress(long[] in, IntWrapper inpos, int inlength, long[] out, IntWrapper outpos) {
+ if (inlength == 0) {
+ return;
+ }
+
+ int[] highParts = new int[inlength];
+ int[] lowParts = new int[inlength];
+
+ for (int i = 0; i < inlength; i++) {
+ int inPosition = inpos.get() + i;
+
+ highParts[i] = RoaringIntPacking.high(in[inPosition]);
+ lowParts[i] = RoaringIntPacking.low(in[inPosition]);
+ }
+
+ // TODO What would be a relevant buffer size?
+ int[] buffer = new int[inlength * 16];
+
+ int outPosition = outpos.get();
+
+ boolean hasLeftover;
+ {
+ // The first integer is reserved to hold the number of compressed ints
+ IntWrapper highPartsOutPosition = new IntWrapper(1);
+
+ highPartsCodec.compress(highParts, new IntWrapper(), inlength, buffer, highPartsOutPosition);
+
+ // Record the compressedHighparts length
+ buffer[0] = highPartsOutPosition.get() - 1;
+
+ for (int i = 0; i < highPartsOutPosition.get() / 2; i++) {
+ long pack = RoaringIntPacking.pack(buffer[i * 2], buffer[i * 2 + 1]);
+ out[outPosition++] = pack;
+ }
+
+ if (1 == highPartsOutPosition.get() % 2) {
+ // Shift the trailing integer as first in the buffer
+ hasLeftover = true;
+ buffer[0] = buffer[highPartsOutPosition.get() - 1];
+ } else {
+ hasLeftover = false;
+ }
+ }
+
+ {
+ // The first integer is reserved to hold the number of compressed ints
+ IntWrapper lowPartsOutPosition = new IntWrapper(1);
+ if (hasLeftover) {
+ // Keep the trailing int from highParts before the reserved int from lowParts compressed length
+ lowPartsOutPosition.set(2);
+ }
+
+ lowPartsCodec.compress(lowParts, new IntWrapper(0), inlength, buffer, lowPartsOutPosition);
+
+ // Record the compressedHighparts length
+ buffer[hasLeftover ? 1 : 0] = lowPartsOutPosition.get() - (hasLeftover ? 2 : 1);
+
+ for (int i = 0; i < lowPartsOutPosition.get() / 2; i++) {
+ long pack = RoaringIntPacking.pack(buffer[i * 2], buffer[i * 2 + 1]);
+ out[outPosition++] = pack;
+ }
+
+ if (1 == lowPartsOutPosition.get() % 2) {
+ // The trailing integer is packed with a 0
+ long pack = RoaringIntPacking.pack(buffer[lowPartsOutPosition.get() - 1], 0);
+ out[outPosition++] = pack;
+ }
+ }
+
+ inpos.add(inlength);
+ outpos.set(outPosition);
+ }
+
+ /**
+ * inlength is ignored by this codec. We may rely on it instead of storing the compressedLowPart length
+ */
+ @Override
+ public void uncompress(long[] in, IntWrapper inpos, int inlength, long[] out, IntWrapper outpos) {
+ if (inlength == 0) {
+ return;
+ }
+
+ int longIndex = inpos.get();
+
+ int nbCompressedHighParts = RoaringIntPacking.high(in[longIndex]);
+ int[] compressedHighParts = new int[nbCompressedHighParts];
+
+ // !highPart as we just read the highPart for nbCompressedHighParts
+ boolean highPart = false;
+ for (int i = 0; i < nbCompressedHighParts; i++) {
+ int nextInt;
+ if (highPart) {
+ nextInt = RoaringIntPacking.high(in[longIndex + (i + 1) / 2]);
+ } else {
+ nextInt = RoaringIntPacking.low(in[longIndex + (i + 1) / 2]);
+ }
+ compressedHighParts[i] = nextInt;
+
+ highPart = !highPart;
+ }
+
+ // TODO What would be a relevant buffer size?
+ int[] buffer = new int[inlength * 16];
+
+ IntWrapper highPartsOutPosition = new IntWrapper();
+ highPartsCodec.uncompress(compressedHighParts,
+ new IntWrapper(),
+ compressedHighParts.length,
+ buffer,
+ highPartsOutPosition);
+ int[] highParts = Arrays.copyOf(buffer, highPartsOutPosition.get());
+
+ // +1 as we initially read nbCompressedHighParts
+ int intIndexNbCompressedLowParts = longIndex * 2 + 1 + nbCompressedHighParts;
+ int nbCompressedLowParts;
+ if (highPart) {
+ nbCompressedLowParts = RoaringIntPacking.high(in[intIndexNbCompressedLowParts / 2]);
+ } else {
+ nbCompressedLowParts = RoaringIntPacking.low(in[intIndexNbCompressedLowParts / 2]);
+ }
+ highPart = !highPart;
+
+ int[] compressedLowParts = new int[nbCompressedLowParts];
+ for (int i = 0; i < nbCompressedLowParts; i++) {
+ int nextInt;
+ if (highPart) {
+ nextInt = RoaringIntPacking.high(in[(intIndexNbCompressedLowParts + 1 + i) / 2]);
+ } else {
+ nextInt = RoaringIntPacking.low(in[(intIndexNbCompressedLowParts + 1 + i) / 2]);
+ }
+ compressedLowParts[i] = nextInt;
+
+ highPart = !highPart;
+ }
+
+ IntWrapper lowPartsOutPosition = new IntWrapper();
+ lowPartsCodec.uncompress(compressedLowParts,
+ new IntWrapper(),
+ compressedLowParts.length,
+ buffer,
+ lowPartsOutPosition);
+ int[] lowParts = Arrays.copyOf(buffer, lowPartsOutPosition.get());
+ assert highParts.length == lowParts.length;
+
+ int outposition = outpos.get();
+ for (int i = 0; i < highParts.length; i++) {
+ out[outposition++] = RoaringIntPacking.pack(highParts[i], lowParts[i]);
+ }
+
+ inpos.add(inlength);
+ outpos.set(outposition);
+ }
}
diff --git a/src/main/java/me/lemire/longcompression/LongBinaryPacking.java b/src/main/java/me/lemire/longcompression/LongBinaryPacking.java
new file mode 100644
index 0000000..b6ea58f
--- /dev/null
+++ b/src/main/java/me/lemire/longcompression/LongBinaryPacking.java
@@ -0,0 +1,153 @@
+package me.lemire.longcompression;
+
+import me.lemire.integercompression.BinaryPacking;
+import me.lemire.integercompression.IntWrapper;
+import me.lemire.integercompression.Util;
+
+/**
+ * Scheme based on a commonly used idea: can be extremely fast.
+ * It encodes integers in blocks of 64 longs. For arrays containing
+ * an arbitrary number of longs, you should use it in conjunction
+ * with another CODEC:
+ *
+ *
LongCODEC ic =
+ * new Composition(new LongBinaryPacking(), new LongVariableByte()).
+ *
+ * Note that this does not use differential coding: if you are working on sorted
+ * lists, you must compute the deltas separately.
+ *
+ *
+ * For details, please see {@link BinaryPacking}
+ *
+ *
+ * @author Benoit Lacelle
+ */
+public final class LongBinaryPacking implements LongCODEC, SkippableLongCODEC {
+ public final static int BLOCK_SIZE = 64;
+ private static final int MAX_BIT_WIDTH = Long.SIZE;
+
+ @Override
+ public void compress(long[] in, IntWrapper inpos, int inlength,
+ long[] out, IntWrapper outpos) {
+ inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+ if (inlength == 0)
+ return;
+ out[outpos.get()] = inlength;
+ outpos.increment();
+ headlessCompress(in, inpos, inlength, out, outpos);
+ }
+
+ @Override
+ public void headlessCompress(long[] in, IntWrapper inpos, int inlength,
+ long[] out, IntWrapper outpos) {
+ inlength = Util.greatestMultiple(inlength, BLOCK_SIZE);
+ int tmpoutpos = outpos.get();
+ int s = inpos.get();
+ // Compress by block of 8 * 64 longs as much as possible
+ for (; s + BLOCK_SIZE * 8 - 1 < inpos.get() + inlength; s += BLOCK_SIZE * 8) {
+ // maxbits can be anything between 0 and 64 included: expressed within a byte (1 << 6)
+ final int mbits1 = LongUtil.maxbits(in, s + 0 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits2 = LongUtil.maxbits(in, s + 1 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits3 = LongUtil.maxbits(in, s + 2 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits4 = LongUtil.maxbits(in, s + 3 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits5 = LongUtil.maxbits(in, s + 4 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits6 = LongUtil.maxbits(in, s + 5 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits7 = LongUtil.maxbits(in, s + 6 * BLOCK_SIZE, BLOCK_SIZE);
+ final int mbits8 = LongUtil.maxbits(in, s + 7 * BLOCK_SIZE, BLOCK_SIZE);
+ // The first long expressed the maxbits for the 8 buckets
+ out[tmpoutpos++] = ((long) mbits1 << 56) | ((long) mbits2 << 48) | ((long) mbits3 << 40) | ((long) mbits4 << 32) | (mbits5 << 24) | (mbits6 << 16) | (mbits7 << 8) | (mbits8);
+ LongBitPacking.fastpackwithoutmask(in, s + 0 * BLOCK_SIZE, out, tmpoutpos, (int) mbits1);
+ tmpoutpos += mbits1;
+ LongBitPacking.fastpackwithoutmask(in, s + 1 * BLOCK_SIZE, out, tmpoutpos, (int) mbits2);
+ tmpoutpos += mbits2;
+ LongBitPacking.fastpackwithoutmask(in, s + 2 * BLOCK_SIZE, out, tmpoutpos, (int) mbits3);
+ tmpoutpos += mbits3;
+ LongBitPacking.fastpackwithoutmask(in, s + 3 * BLOCK_SIZE, out, tmpoutpos, (int) mbits4);
+ tmpoutpos += mbits4;
+ LongBitPacking.fastpackwithoutmask(in, s + 4 * BLOCK_SIZE, out, tmpoutpos, (int) mbits5);
+ tmpoutpos += mbits5;
+ LongBitPacking.fastpackwithoutmask(in, s + 5 * BLOCK_SIZE, out, tmpoutpos, (int) mbits6);
+ tmpoutpos += mbits6;
+ LongBitPacking.fastpackwithoutmask(in, s + 6 * BLOCK_SIZE, out, tmpoutpos, (int) mbits7);
+ tmpoutpos += mbits7;
+ LongBitPacking.fastpackwithoutmask(in, s + 7 * BLOCK_SIZE, out, tmpoutpos, (int) mbits8);
+ tmpoutpos += mbits8;
+ }
+ // Then we compress up to 7 blocks of 64 longs
+ for (; s < inpos.get() + inlength; s += BLOCK_SIZE ) {
+ final int mbits = LongUtil.maxbits(in, s, BLOCK_SIZE);
+ out[tmpoutpos++] = mbits;
+ LongBitPacking.fastpackwithoutmask(in, s, out, tmpoutpos, mbits);
+ tmpoutpos += mbits;
+ }
+ inpos.add(inlength);
+ outpos.set(tmpoutpos);
+ }
+
+ @Override
+ public void uncompress(long[] in, IntWrapper inpos, int inlength,
+ long[] out, IntWrapper outpos) {
+ if (inlength == 0)
+ return;
+ final int outlength = (int) in[inpos.get()];
+ inpos.increment();
+ headlessUncompress(in,inpos, inlength,out,outpos,outlength);
+ }
+
+ @Override
+ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength,
+ long[] out, IntWrapper outpos, int num) {
+ final int outlength = Util.greatestMultiple(num, BLOCK_SIZE);
+ int tmpinpos = inpos.get();
+ int s = outpos.get();
+ for (; s + BLOCK_SIZE * 8 - 1 < outpos.get() + outlength; s += BLOCK_SIZE * 8) {
+ final int mbits1 = (int) ((in[tmpinpos] >>> 56));
+ final int mbits2 = (int) ((in[tmpinpos] >>> 48) & 0xFF);
+ final int mbits3 = (int) ((in[tmpinpos] >>> 40) & 0xFF);
+ final int mbits4 = (int) ((in[tmpinpos] >>> 32) & 0xFF);
+ final int mbits5 = (int) ((in[tmpinpos] >>> 24) & 0xFF);
+ final int mbits6 = (int) ((in[tmpinpos] >>> 16) & 0xFF);
+ final int mbits7 = (int) ((in[tmpinpos] >>> 8) & 0xFF);
+ final int mbits8 = (int) ((in[tmpinpos]) & 0xFF);
+ ++tmpinpos;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 0 * BLOCK_SIZE, mbits1);
+ tmpinpos += mbits1;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 1 * BLOCK_SIZE, mbits2);
+ tmpinpos += mbits2;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 2 * BLOCK_SIZE, mbits3);
+ tmpinpos += mbits3;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 3 * BLOCK_SIZE, mbits4);
+ tmpinpos += mbits4;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 4 * BLOCK_SIZE, mbits5);
+ tmpinpos += mbits5;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 5 * BLOCK_SIZE, mbits6);
+ tmpinpos += mbits6;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 6 * BLOCK_SIZE, mbits7);
+ tmpinpos += mbits7;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s + 7 * BLOCK_SIZE, mbits8);
+ tmpinpos += mbits8;
+ }
+ for (; s < outpos.get() + outlength; s += BLOCK_SIZE ) {
+ final int mbits = (int) in[tmpinpos];
+ ++tmpinpos;
+ LongBitPacking.fastunpack(in, tmpinpos, out, s, mbits);
+ tmpinpos += mbits;
+ }
+ outpos.add(outlength);
+ inpos.set(tmpinpos);
+ }
+
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int blockCount = inlength / BLOCK_SIZE;
+ int headersSizeInLongs = blockCount / Long.BYTES + (blockCount % Long.BYTES);
+ int blocksSizeInLongs = blockCount * MAX_BIT_WIDTH;
+ compressedPositions.add(blockCount * BLOCK_SIZE);
+ return headersSizeInLongs + blocksSizeInLongs;
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName();
+ }
+}
diff --git a/src/main/java/me/lemire/longcompression/LongBitPacking.java b/src/main/java/me/lemire/longcompression/LongBitPacking.java
new file mode 100644
index 0000000..2d282ec
--- /dev/null
+++ b/src/main/java/me/lemire/longcompression/LongBitPacking.java
@@ -0,0 +1,146 @@
+/**
+ * This code is released under the
+ * Apache License Version 2.0 http://www.apache.org/licenses/.
+ *
+ * (c) Daniel Lemire, http://lemire.me/en/
+ */
+
+package me.lemire.longcompression;
+
+import java.util.Arrays;
+
+/**
+ * Bitpacking routines
+ *
+ *
+ *
+ * @author Benoit Lacelle
+ *
+ */
+public final class LongBitPacking {
+
+ /**
+ * Pack 64 longs
+ *
+ * @param in
+ * source array
+ * @param inpos
+ * position in source array
+ * @param out
+ * output array
+ * @param outpos
+ * position in output array
+ * @param bit
+ * number of bits to use per long
+ */
+ public static void fastpackwithoutmask(final long[] in, final int inpos,
+ final long[] out, final int outpos, final int bit) {
+ if (bit == 0) {
+ fastpackwithoutmask0(in, inpos, out, outpos);
+ } else if (bit == 64) {
+ fastpackwithoutmask64(in, inpos, out, outpos);
+ } else if (bit > 0 && bit < 64) {
+ slowpackwithoutmask(in, inpos, out, outpos, bit);
+ } else {
+ throw new IllegalArgumentException("Unsupported bit width: " + bit);
+ }
+ }
+
+ protected static void fastpackwithoutmask0(final long[] in, int inpos,
+ final long[] out, int outpos) {
+ // nothing
+ }
+
+ protected static void fastpackwithoutmask64(final long[] in, int inpos,
+ final long[] out, int outpos) {
+ System.arraycopy(in, inpos, out, outpos, 64);
+ }
+
+ protected static void slowpackwithoutmask(final long[] in, int inpos,
+ final long[] out, int outpos, final int bit) {
+ int bucket = 0;
+ int shift = 0;
+
+ out[outpos + bucket] = 0L;
+ for (int i = 0 ; i < 64 ; i++) {
+ if (shift >= 64) {
+ bucket++;
+ out[bucket + outpos] = 0L;
+ shift -= 64;
+
+ if (shift > 0) {
+ // There is some leftovers from previous input in the next bucket
+ out[outpos + bucket] |= in[inpos + i - 1] >> (bit - shift);
+ }
+ }
+ out[outpos + bucket] |= in[inpos + i] << shift;
+
+ shift += bit;
+ }
+ }
+
+
+ /**
+ * Unpack the 64 longs
+ *
+ * @param in
+ * source array
+ * @param inpos
+ * starting point in the source array
+ * @param out
+ * output array
+ * @param outpos
+ * starting point in the output array
+ * @param bit
+ * how many bits to use per integer
+ */
+ public static void fastunpack(final long[] in, final int inpos,
+ final long[] out, final int outpos, final int bit) {
+ if (bit == 0) {
+ fastunpack0(in, inpos, out, outpos);
+ } else if (bit == 64) {
+ fastunpack64(in, inpos, out, outpos);
+ } else if (bit > 0 && bit < 64) {
+ slowunpack(in, inpos, out, outpos, bit);
+ } else {
+ throw new IllegalArgumentException("Unsupported bit width: " + bit);
+ }
+ }
+
+
+ protected static void fastunpack0(final long[] in, int inpos,
+ final long[] out, int outpos) {
+ Arrays.fill(out, outpos, outpos + 64, 0);
+ }
+
+ protected static void fastunpack64(final long[] in, int inpos,
+ final long[] out, int outpos) {
+ System.arraycopy(in, inpos, out, outpos, 64);
+ }
+
+ protected static void slowunpack(final long[] in, int inpos,
+ final long[] out, int outpos, final int bit) {
+ int bucket = 0;
+ int shift = 0;
+ for (int i = 0 ; i < 64 ; i++) {
+ if (shift >= 64) {
+ bucket++;
+ shift -= 64;
+
+ if (shift > 0) {
+ // There is some leftovers from previous input in the next bucket
+ out[outpos + i - 1] |= (in[inpos + bucket] << (bit - shift) & ((1L << bit) - 1));
+ }
+ }
+ out[outpos + i] = ((in[inpos + bucket] >>> shift) & ((1L << bit) - 1));
+
+ shift += bit;
+ }
+ }
+}
diff --git a/src/main/java/me/lemire/longcompression/LongCODEC.java b/src/main/java/me/lemire/longcompression/LongCODEC.java
index c0f67b2..0951ffd 100644
--- a/src/main/java/me/lemire/longcompression/LongCODEC.java
+++ b/src/main/java/me/lemire/longcompression/LongCODEC.java
@@ -27,7 +27,7 @@ public interface LongCODEC {
* @param in
* input array
* @param inpos
- * location in the input array
+ * where to start reading in the array
* @param inlength
* how many longs to compress
* @param out
@@ -36,7 +36,7 @@ public interface LongCODEC {
* where to write in the output array
*/
public void compress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos);
+ long[] out, IntWrapper outpos);
/**
* Uncompress data from an array to another array.
@@ -52,11 +52,11 @@ public void compress(long[] in, IntWrapper inpos, int inlength,
* length of the compressed data (ignored by some
* schemes)
* @param out
- * array where to write the compressed output
+ * array where to write the uncompressed output
* @param outpos
- * where to write the compressed output in out
+ * where to start writing the uncompressed output in out
*/
public void uncompress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos);
+ long[] out, IntWrapper outpos);
}
diff --git a/src/main/java/me/lemire/longcompression/LongComposition.java b/src/main/java/me/lemire/longcompression/LongComposition.java
index 1394a78..5111a51 100644
--- a/src/main/java/me/lemire/longcompression/LongComposition.java
+++ b/src/main/java/me/lemire/longcompression/LongComposition.java
@@ -37,7 +37,7 @@ public LongComposition(LongCODEC f1, LongCODEC f2) {
@Override
public void compress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos) {
+ long[] out, IntWrapper outpos) {
if (inlength == 0) {
return;
}
@@ -54,7 +54,7 @@ public void compress(long[] in, IntWrapper inpos, int inlength,
@Override
public void uncompress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos) {
+ long[] out, IntWrapper outpos) {
if (inlength == 0)
return;
final int init = inpos.get();
diff --git a/src/main/java/me/lemire/longcompression/LongCompressor.java b/src/main/java/me/lemire/longcompression/LongCompressor.java
new file mode 100644
index 0000000..246647f
--- /dev/null
+++ b/src/main/java/me/lemire/longcompression/LongCompressor.java
@@ -0,0 +1,68 @@
+package me.lemire.longcompression;
+
+import java.util.Arrays;
+
+import me.lemire.integercompression.IntWrapper;
+
+/**
+ * This is a convenience class that wraps a codec to provide
+ * a "friendly" API.
+ *
+ * @author Benoit Lacelle
+ */
+public class LongCompressor {
+
+ SkippableLongCODEC codec;
+
+ /**
+ * Constructor wrapping a codec.
+ *
+ * @param c the underlying codec
+ */
+ public LongCompressor(SkippableLongCODEC c) {
+ codec = c;
+ }
+
+ /**
+ * Constructor with default codec.
+ */
+ public LongCompressor() {
+ codec = new SkippableLongComposition(new LongBinaryPacking(),
+ new LongVariableByte());
+ }
+
+ /**
+ * Compress an array and returns the compressed result as a new array.
+ *
+ * @param input array to be compressed
+ * @return compressed array
+ */
+ public long[] compress(long[] input) {
+ int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length);
+ long[] compressed = new long[maxCompressedLength + 1]; // +1 to store the length of the input
+ // Store at index=0 the length of the input, hence enabling .headlessCompress
+ compressed[0] = input.length;
+ IntWrapper outpos = new IntWrapper(1);
+ codec.headlessCompress(input, new IntWrapper(0), input.length, compressed, outpos);
+ compressed = Arrays.copyOf(compressed,outpos.intValue());
+ return compressed;
+ }
+
+ /**
+ * Uncompress an array and returns the uncompressed result as a new array.
+ *
+ * @param compressed compressed array
+ * @return uncompressed array
+ */
+ public long[] uncompress(long[] compressed) {
+ // Read at index=0 the length of the input, hence enabling .headlessUncompress
+ long[] decompressed = new long[(int) compressed[0]];
+ IntWrapper inpos = new IntWrapper(1);
+ codec.headlessUncompress(compressed, inpos,
+ compressed.length - inpos.intValue(),
+ decompressed, new IntWrapper(0),
+ decompressed.length);
+ return decompressed;
+ }
+
+}
diff --git a/src/main/java/me/lemire/longcompression/LongJustCopy.java b/src/main/java/me/lemire/longcompression/LongJustCopy.java
index 7a5a67a..95abc1e 100644
--- a/src/main/java/me/lemire/longcompression/LongJustCopy.java
+++ b/src/main/java/me/lemire/longcompression/LongJustCopy.java
@@ -17,7 +17,7 @@ public final class LongJustCopy implements LongCODEC, SkippableLongCODEC {
@Override
public void headlessCompress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos) {
+ long[] out, IntWrapper outpos) {
System.arraycopy(in, inpos.get(), out, outpos.get(), inlength);
inpos.add(inlength);
outpos.add(inlength);
@@ -25,7 +25,7 @@ public void headlessCompress(long[] in, IntWrapper inpos, int inlength,
@Override
public void uncompress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos) {
+ long[] out, IntWrapper outpos) {
headlessUncompress(in,inpos,inlength,out,outpos,inlength);
}
@@ -36,16 +36,22 @@ public String toString() {
@Override
public void headlessUncompress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos, int num) {
+ long[] out, IntWrapper outpos, int num) {
System.arraycopy(in, inpos.get(), out, outpos.get(), num);
inpos.add(num);
outpos.add(num);
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ compressedPositions.add(inlength);
+ return inlength;
+ }
+
@Override
public void compress(long[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos) {
+ long[] out, IntWrapper outpos) {
headlessCompress(in,inpos,inlength,out,outpos);
}
diff --git a/src/main/java/me/lemire/longcompression/LongUtil.java b/src/main/java/me/lemire/longcompression/LongUtil.java
index c06433f..7bdce83 100644
--- a/src/main/java/me/lemire/longcompression/LongUtil.java
+++ b/src/main/java/me/lemire/longcompression/LongUtil.java
@@ -15,8 +15,38 @@
*/
@Deprecated
public class LongUtil {
+
+ /**
+ * Compute the maximum of the integer logarithms (ceil(log(x+1)) of a range
+ * of value
+ *
+ * @param i
+ * source array
+ * @param pos
+ * starting position
+ * @param length
+ * number of integers to consider
+ * @return integer logarithm
+ */
+ public static int maxbits(long[] i, int pos, int length) {
+ long mask = 0;
+ for (int k = pos; k < pos + length; ++k)
+ mask |= i[k];
+ return bits(mask);
+ }
- protected static String longToBinaryWithLeading(long l) {
- return String.format("%64s", Long.toBinaryString(l)).replace(' ', '0');
- }
+ /**
+ * Compute the integer logarithms (ceil(log(x+1)) of a value
+ *
+ * @param i
+ * source value
+ * @return integer logarithm
+ */
+ public static int bits(long i) {
+ return 64 - Long.numberOfLeadingZeros(i);
+ }
+
+ protected static String longToBinaryWithLeading(long l) {
+ return String.format("%64s", Long.toBinaryString(l)).replace(' ', '0');
+ }
}
diff --git a/src/main/java/me/lemire/longcompression/LongVariableByte.java b/src/main/java/me/lemire/longcompression/LongVariableByte.java
index 478db20..63c194b 100644
--- a/src/main/java/me/lemire/longcompression/LongVariableByte.java
+++ b/src/main/java/me/lemire/longcompression/LongVariableByte.java
@@ -22,6 +22,7 @@
* @author Benoit Lacelle
*/
public class LongVariableByte implements LongCODEC, ByteLongCODEC, SkippableLongCODEC {
+ private static final int MAX_BYTES_PER_INT = 10;
private static byte extract7bits(int i, long val) {
return (byte) ((val >>> (7 * i)) & ((1 << 7) - 1));
@@ -46,7 +47,6 @@ public void headlessCompress(long[] in, IntWrapper inpos, int inlength, long[] o
buf.order(ByteOrder.LITTLE_ENDIAN);
for (int k = inpos.get(); k < inpos.get() + inlength; ++k) {
final long val = in[k];
- // System.out.println(LongUtil.longToBinaryWithLeading(val));
if (val >= 0 && val < (1 << 7)) {
buf.put((byte) (val | (1 << 7)));
} else if (val >= 0 && val < (1 << 14)) {
@@ -91,7 +91,7 @@ public void headlessCompress(long[] in, IntWrapper inpos, int inlength, long[] o
buf.put((byte) extract7bits(5, val));
buf.put((byte) extract7bits(6, val));
buf.put((byte) (extract7bitsmaskless(7, (val)) | (1 << 7)));
- } else if (val >= 0 && val < (1L << 63)) {
+ } else if (val >= 0) {
buf.put((byte) extract7bits(0, val));
buf.put((byte) extract7bits(1, val));
buf.put((byte) extract7bits(2, val));
@@ -176,7 +176,7 @@ public void compress(long[] in, IntWrapper inpos, int inlength, byte[] out,
out[outpostmp++] = (byte) extract7bits(5, val);
out[outpostmp++] = (byte) extract7bits(6, val);
out[outpostmp++] = (byte) (extract7bitsmaskless(7, (val)) | (1 << 7));
- } else if (val >= 0 && val < (1L << 63)) {
+ } else if (val >= 0) {
out[outpostmp++] = (byte) extract7bits(0, val);
out[outpostmp++] = (byte) extract7bits(1, val);
out[outpostmp++] = (byte) extract7bits(2, val);
@@ -187,7 +187,6 @@ public void compress(long[] in, IntWrapper inpos, int inlength, byte[] out,
out[outpostmp++] = (byte) extract7bits(7, val);
out[outpostmp++] = (byte) (extract7bitsmaskless(8, (val)) | (1 << 7));
} else {
- // System.out.println(LongUtil.longToBinaryWithLeading(val));
out[outpostmp++] = (byte) extract7bits(0, val);
out[outpostmp++] = (byte) extract7bits(1, val);
out[outpostmp++] = (byte) extract7bits(2, val);
@@ -214,13 +213,12 @@ public void uncompress(long[] in, IntWrapper inpos, int inlength, long[] out,
int tmpoutpos = outpos.get();
for (long v = 0, shift = 0; p < finalp;) {
val = in[p];
- // System.out.println(LongUtil.longToBinaryWithLeading(val));
long c = (byte) (val >>> s);
// Shift to next byte
s += 8;
// Shift to next long if s==64
p += s>>6;
- // cycle from 63 to 0
+ // Cycle from 63 to 0
s = s & 63;
v += ((c & 127) << shift);
if ((c & 128) == 128) {
@@ -237,7 +235,7 @@ public void uncompress(long[] in, IntWrapper inpos, int inlength, long[] out,
@Override
public void uncompress(byte[] in, IntWrapper inpos, int inlength,
- long[] out, IntWrapper outpos) {
+ long[] out, IntWrapper outpos) {
int p = inpos.get();
int finalp = inpos.get() + inlength;
int tmpoutpos = outpos.get();
@@ -309,13 +307,12 @@ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[]
int finaloutpos = num + tmpoutpos;
for (long v = 0, shift = 0; tmpoutpos < finaloutpos;) {
val = in[p];
- // System.out.println(longToBinaryWithLeading(val));
long c = val >>> s;
// Shift to next byte
s += 8;
// Shift to next long if s == 64
p += s>>6;
- // cycle from 63 to 0
+ // Cycle from 63 to 0
s = s & 63;
v += ((c & 127) << shift);
if ((c & 128) == 128) {
@@ -330,6 +327,14 @@ public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[]
inpos.set(p + (s!=0 ? 1 : 0));
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int maxLengthInBytes = inlength * MAX_BYTES_PER_INT;
+ int maxLengthInLongs = (maxLengthInBytes + Long.BYTES - 1) / Long.BYTES;
+ compressedPositions.add(inlength);
+ return maxLengthInLongs;
+ }
+
/**
* Creates a new buffer of the requested size.
*
diff --git a/src/main/java/me/lemire/longcompression/RoaringIntPacking.java b/src/main/java/me/lemire/longcompression/RoaringIntPacking.java
index f109ab3..d6b6baa 100644
--- a/src/main/java/me/lemire/longcompression/RoaringIntPacking.java
+++ b/src/main/java/me/lemire/longcompression/RoaringIntPacking.java
@@ -3,9 +3,6 @@
*/
package me.lemire.longcompression;
-import java.math.BigInteger;
-import java.util.Comparator;
-
/**
* Used to hold the logic packing 2 integers in a long, and separating a long in two integers. It is
* useful in {@link Roaring64NavigableMap} as the implementation split the input long in two
@@ -46,63 +43,4 @@ public static int low(long id) {
public static long pack(int high, int low) {
return (((long) high) << 32) | (low & 0xffffffffL);
}
-
-
- /**
- *
- * @param signedLongs true if long put in a {@link Roaring64NavigableMap} should be considered as
- * signed long.
- * @return the int representing the highest value which can be set as high value in a
- * {@link Roaring64NavigableMap}
- */
- public static int highestHigh(boolean signedLongs) {
- if (signedLongs) {
- return Integer.MAX_VALUE;
- } else {
- return -1;
- }
- }
-
- /**
- * @return A comparator for unsigned longs: a negative long is a long greater than Long.MAX_VALUE
- */
- public static Comparator unsignedComparator() {
- return new Comparator() {
-
- @Override
- public int compare(Integer o1, Integer o2) {
- return compareUnsigned(o1, o2);
- }
- };
- }
-
- /**
- * Compares two {@code int} values numerically treating the values as unsigned.
- *
- * @param x the first {@code int} to compare
- * @param y the second {@code int} to compare
- * @return the value {@code 0} if {@code x == y}; a value less than {@code 0} if {@code x < y} as
- * unsigned values; and a value greater than {@code 0} if {@code x > y} as unsigned values
- * @since 1.8
- */
- // Duplicated from jdk8 Integer.compareUnsigned
- public static int compareUnsigned(int x, int y) {
- return Integer.compare(x + Integer.MIN_VALUE, y + Integer.MIN_VALUE);
- }
-
- /** the constant 2^64 */
- private static final BigInteger TWO_64 = BigInteger.ONE.shiftLeft(64);
-
- /**
- * JDK8 Long.toUnsignedString was too complex to backport. Go for a slow version relying on
- * BigInteger
- */
- // https://stackoverflow.com/questions/7031198/java-signed-long-to-unsigned-long-string
- static String toUnsignedString(long l) {
- BigInteger b = BigInteger.valueOf(l);
- if (b.signum() < 0) {
- b = b.add(TWO_64);
- }
- return b.toString();
- }
}
diff --git a/src/main/java/me/lemire/longcompression/SkippableLongCODEC.java b/src/main/java/me/lemire/longcompression/SkippableLongCODEC.java
index e3e7b84..33fd562 100644
--- a/src/main/java/me/lemire/longcompression/SkippableLongCODEC.java
+++ b/src/main/java/me/lemire/longcompression/SkippableLongCODEC.java
@@ -11,7 +11,8 @@
/**
* Interface describing a standard CODEC to compress longs. This is a
- * variation on the LongCODEC interface meant to be used for head access.
+ * variation on the LongCODEC interface meant to be used for random access
+ * (i.e., given a large array, you can segment it and decode just the subarray you need).
*
* The main difference is that we must specify the number of longs we wish to
* decode. This information should be stored elsewhere.
@@ -33,7 +34,7 @@ public interface SkippableLongCODEC {
* @param in
* input array
* @param inpos
- * location in the input array
+ * where to start reading in the array
* @param inlength
* how many longs to compress
* @param out
@@ -57,13 +58,30 @@ public void headlessCompress(long[] in, IntWrapper inpos, int inlength, long[] o
* @param inlength
* length of the compressed data (ignored by some schemes)
* @param out
- * array where to write the compressed output
+ * array where to write the uncompressed output
* @param outpos
- * where to write the compressed output in out
+ * where to start writing the uncompressed output in out
* @param num
* number of longs we want to decode, the actual number of longs decoded can be less
*/
public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[] out,
IntWrapper outpos, int num);
+ /**
+ * Compute the maximum number of longs that might be required to store
+ * the compressed form of a given input array segment, without headers.
+ *
+ * This is useful to pre-allocate the output buffer before calling
+ * {@link #headlessCompress(long[], IntWrapper, int, long[], IntWrapper)}.
+ *
+ *
+ * @param compressedPositions
+ * since not all schemes compress every input integer, this parameter
+ * returns how many input integers will actually be compressed.
+ * This is useful when composing multiple schemes.
+ * @param inlength
+ * number of longs to be compressed
+ * @return the maximum number of longs needed in the output array
+ */
+ int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength);
}
diff --git a/src/main/java/me/lemire/longcompression/SkippableLongComposition.java b/src/main/java/me/lemire/longcompression/SkippableLongComposition.java
index 5568489..eb03b72 100644
--- a/src/main/java/me/lemire/longcompression/SkippableLongComposition.java
+++ b/src/main/java/me/lemire/longcompression/SkippableLongComposition.java
@@ -53,15 +53,27 @@ public void headlessCompress(long[] in, IntWrapper inpos, int inlength, long[] o
public void headlessUncompress(long[] in, IntWrapper inpos, int inlength, long[] out,
IntWrapper outpos, int num) {
int init = inpos.get();
+ int outposInit = outpos.get();
+
F1.headlessUncompress(in, inpos, inlength, out, outpos, num);
if (inpos.get() == init) {
- inpos.increment();
+ inpos.increment();
}
inlength -= inpos.get() - init;
- num -= outpos.get();
+ num -= outpos.get() - outposInit;
F2.headlessUncompress(in, inpos, inlength, out, outpos, num);
}
+ @Override
+ public int maxHeadlessCompressedLength(IntWrapper compressedPositions, int inlength) {
+ int init = compressedPositions.get();
+ int maxLength = F1.maxHeadlessCompressedLength(compressedPositions, inlength);
+ maxLength += 1; // Add +1 for the potential F2 header. Question: is this header actually needed in the headless version?
+ inlength -= compressedPositions.get() - init;
+ maxLength += F2.maxHeadlessCompressedLength(compressedPositions, inlength);
+ return maxLength;
+ }
+
@Override
public String toString() {
return F1.toString() + "+" + F2.toString();
diff --git a/src/main/java/me/lemire/longcompression/differential/LongDelta.java b/src/main/java/me/lemire/longcompression/differential/LongDelta.java
index 2b0e077..184e53c 100644
--- a/src/main/java/me/lemire/longcompression/differential/LongDelta.java
+++ b/src/main/java/me/lemire/longcompression/differential/LongDelta.java
@@ -66,7 +66,7 @@ public static long delta(long[] data, int start, int length, int init) {
* @return next initial vale
*/
public static long delta(long[] data, int start, int length, int init,
- long[] out) {
+ long[] out) {
for (int i = length - 1; i > 0; --i) {
out[i] = data[start + i] - data[start + i - 1];
}
@@ -98,7 +98,7 @@ public static void fastinverseDelta(long[] data) {
int sz0 = data.length / 4 * 4;
int i = 1;
if (sz0 >= 4) {
- long a = data[0];
+ long a = data[0];
for (; i < sz0 - 4; i += 4) {
a = data[i] += a;
a = data[i + 1] += a;
@@ -132,7 +132,7 @@ public static long fastinverseDelta(long[] data, int start, int length,
int sz0 = length / 4 * 4;
int i = 1;
if (sz0 >= 4) {
- long a = data[start];
+ long a = data[start];
for (; i < sz0 - 4; i += 4) {
a = data[start + i] += a;
a = data[start + i + 1] += a;
diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java
index d2a749c..d254c12 100644
--- a/src/main/java/module-info.java
+++ b/src/main/java/module-info.java
@@ -5,5 +5,6 @@
// This is currently only for advanced users:
// requires jdk.incubator.vector;
exports me.lemire.integercompression;
- exports me.lemire.integercompression.vector;
+ exports me.lemire.longcompression;
+ // exports me.lemire.integercompression.vector;
}
diff --git a/src/test/java/me/lemire/integercompression/AdhocTest.java b/src/test/java/me/lemire/integercompression/AdhocTest.java
index 8fd4049..ee911b3 100644
--- a/src/test/java/me/lemire/integercompression/AdhocTest.java
+++ b/src/test/java/me/lemire/integercompression/AdhocTest.java
@@ -22,13 +22,48 @@
@SuppressWarnings({ "static-method" })
public class AdhocTest {
-
- /**
- *
- */
+ @Test
+ public void testIssue59() {
+ FastPFOR128 fastpfor = new FastPFOR128();
+
+ int N = 9984;
+ int[] data = new int[N];
+ for (var i = 0; i < N; i += 150) {
+ data[i] = i;
+ }
+
+ int[] compressedoutput1 = new int[N + 1024];
+
+ IntWrapper inputoffset1 = new IntWrapper(0);
+ IntWrapper outputoffset1 = new IntWrapper(0);
+
+ fastpfor.compress(data, inputoffset1, N, compressedoutput1, outputoffset1);
+ int compressedsize1 = outputoffset1.get();
+
+ int[] recovered1 = new int[N];
+ inputoffset1 = new IntWrapper(0);
+ outputoffset1 = new IntWrapper(0);
+ fastpfor.uncompress(compressedoutput1, outputoffset1, compressedsize1, recovered1, inputoffset1);
+ Assert.assertArrayEquals(data, recovered1);
+
+ int[] compressedoutput2 = new int[N + 1024];
+
+ IntWrapper inputoffset2 = new IntWrapper(0);
+ IntWrapper outputoffset2 = new IntWrapper(0);
+
+ fastpfor.compress(data, inputoffset2, N, compressedoutput2, outputoffset2);
+ int compressedsize2 = outputoffset2.get();
+
+ int[] recovered2 = new int[N];
+ inputoffset2 = new IntWrapper(0);
+ outputoffset2 = new IntWrapper(0);
+ fastpfor.uncompress(compressedoutput2, outputoffset2, compressedsize2, recovered2, inputoffset2);
+ Assert.assertArrayEquals(data, recovered2);
+ }
+
@Test
public void testIssue29() {
- for(int x = 0; x < 64; x++) {
+ for(int x = 0; x < 64; x++) {
int[] a = {2, 3, 4, 5};
int[] b = new int[90];
int[] c = new int[a.length];
@@ -42,7 +77,7 @@ public void testIssue29() {
IntWrapper cOffset = new IntWrapper(0);
codec.uncompress(b, bOffset, len, c, cOffset);
Assert.assertArrayEquals(a,c);
- }
+ }
}
/**
@@ -50,20 +85,20 @@ public void testIssue29() {
*/
@Test
public void testIssue29b() {
- for(int x = 0; x < 64; x++) {
- int[] a = {2, 3, 4, 5};
- int[] b = new int[90];
- int[] c = new int[a.length];
- SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
- IntWrapper aOffset = new IntWrapper(0);
- IntWrapper bOffset = new IntWrapper(x);
- codec.headlessCompress(a, aOffset, a.length, b, bOffset);
- int len = bOffset.get() - x;
- bOffset.set(x);
- IntWrapper cOffset = new IntWrapper(0);
- codec.headlessUncompress(b, bOffset, len, c, cOffset, a.length);
- Assert.assertArrayEquals(a,c);
- }
+ for(int x = 0; x < 64; x++) {
+ SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
+ int[] a = {2, 3, 4, 5};
+ int[] b = new int[x + codec.maxHeadlessCompressedLength(new IntWrapper(0), a.length)];
+ int[] c = new int[a.length];
+ IntWrapper aOffset = new IntWrapper(0);
+ IntWrapper bOffset = new IntWrapper(x);
+ codec.headlessCompress(a, aOffset, a.length, b, bOffset);
+ int len = bOffset.get() - x;
+ bOffset.set(x);
+ IntWrapper cOffset = new IntWrapper(0);
+ codec.headlessUncompress(b, bOffset, len, c, cOffset, a.length);
+ Assert.assertArrayEquals(a,c);
+ }
}
@@ -71,30 +106,27 @@ public void testIssue29b() {
*
*/
@Test
- public void testIssue41() {
- for (int x = 0; x < 64; x++) {
- int[] a = { 2, 3, 4, 5 };
- int[] b = new int[90];
- int[] c = new int[a.length];
- SkippableIntegratedIntegerCODEC codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
- new IntegratedVariableByte());
- IntWrapper aOffset = new IntWrapper(0);
- IntWrapper bOffset = new IntWrapper(x);
- IntWrapper initValue = new IntWrapper(0);
-
- codec.headlessCompress(a, aOffset, a.length, b, bOffset, initValue);
- int len = bOffset.get() - x;
- bOffset.set(x);
- IntWrapper cOffset = new IntWrapper(0);
- initValue = new IntWrapper(0);
- codec.headlessUncompress(b, bOffset, len, c, cOffset, a.length, initValue);
- Assert.assertArrayEquals(a, c);
- }
- }
+ public void testIssue41() {
+ for (int x = 0; x < 64; x++) {
+ SkippableIntegratedIntegerCODEC codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
+ new IntegratedVariableByte());
+ int[] a = { 2, 3, 4, 5 };
+ int[] b = new int[x + codec.maxHeadlessCompressedLength(new IntWrapper(0), a.length)];
+ int[] c = new int[a.length];
+ IntWrapper aOffset = new IntWrapper(0);
+ IntWrapper bOffset = new IntWrapper(x);
+ IntWrapper initValue = new IntWrapper(0);
+
+ codec.headlessCompress(a, aOffset, a.length, b, bOffset, initValue);
+ int len = bOffset.get() - x;
+ bOffset.set(x);
+ IntWrapper cOffset = new IntWrapper(0);
+ initValue = new IntWrapper(0);
+ codec.headlessUncompress(b, bOffset, len, c, cOffset, a.length, initValue);
+ Assert.assertArrayEquals(a, c);
+ }
+ }
- /**
- * a test
- */
@Test
public void biggerCompressedArray0() {
// No problem: for comparison.
@@ -102,12 +134,8 @@ public void biggerCompressedArray0() {
assertSymmetry(c, 0, 16384);
c = new Composition(new FastPFOR(), new VariableByte());
assertSymmetry(c, 0, 16384);
-
}
- /**
- * a test
- */
@Test
public void biggerCompressedArray1() {
// Compressed array is bigger than original, because of VariableByte.
@@ -115,9 +143,6 @@ public void biggerCompressedArray1() {
assertSymmetry(c, -1);
}
- /**
- * a test
- */
@Test
public void biggerCompressedArray2() {
// Compressed array is bigger than original, because of Composition.
diff --git a/src/test/java/me/lemire/integercompression/BasicTest.java b/src/test/java/me/lemire/integercompression/BasicTest.java
index b5f292e..b29ae0d 100644
--- a/src/test/java/me/lemire/integercompression/BasicTest.java
+++ b/src/test/java/me/lemire/integercompression/BasicTest.java
@@ -48,35 +48,35 @@ public class BasicTest {
new GroupSimple9(),
new Composition(new XorBinaryPacking(), new VariableByte()),
new Composition(new DeltaZigzagBinaryPacking(),
- new DeltaZigzagVariableByte()) };
+ new DeltaZigzagVariableByte()) };
- /**
+ /**
* This tests with a compressed array with various offset
*/
- @Test
- public void saulTest() {
- for (IntegerCODEC C : codecs) {
- for (int x = 0; x < 50; ++x) {
- int[] a = { 2, 3, 4, 5 };
- int[] b = new int[90];
- int[] c = new int[a.length];
-
- IntWrapper aOffset = new IntWrapper(0);
- IntWrapper bOffset = new IntWrapper(x);
- C.compress(a, aOffset, a.length, b, bOffset);
- int len = bOffset.get() - x;
-
- bOffset.set(x);
- IntWrapper cOffset = new IntWrapper(0);
- C.uncompress(b, bOffset, len, c, cOffset);
- if(!Arrays.equals(a, c)) {
- System.out.println("Problem with "+C);
- }
- assertArrayEquals(a, c);
-
- }
- }
- }
+ @Test
+ public void saulTest() {
+ for (IntegerCODEC C : codecs) {
+ for (int x = 0; x < 50; ++x) {
+ int[] a = { 2, 3, 4, 5 };
+ int[] b = new int[90];
+ int[] c = new int[a.length];
+
+ IntWrapper aOffset = new IntWrapper(0);
+ IntWrapper bOffset = new IntWrapper(x);
+ C.compress(a, aOffset, a.length, b, bOffset);
+ int len = bOffset.get() - x;
+
+ bOffset.set(x);
+ IntWrapper cOffset = new IntWrapper(0);
+ C.uncompress(b, bOffset, len, c, cOffset);
+ if(!Arrays.equals(a, c)) {
+ System.out.println("Problem with "+C);
+ }
+ assertArrayEquals(a, c);
+
+ }
+ }
+ }
/**
*
*/
diff --git a/src/test/java/me/lemire/integercompression/ByteBasicTest.java b/src/test/java/me/lemire/integercompression/ByteBasicTest.java
index 93112c3..2b2d4f1 100644
--- a/src/test/java/me/lemire/integercompression/ByteBasicTest.java
+++ b/src/test/java/me/lemire/integercompression/ByteBasicTest.java
@@ -28,32 +28,32 @@ public class ByteBasicTest {
new IntegratedVariableByte(),
};
- /**
+ /**
*
*/
- @Test
- public void saulTest() {
- for (ByteIntegerCODEC C : codecs) {
- for (int x = 0; x < 50 * 4; ++x) {
- int[] a = { 2, 3, 4, 5 };
- byte[] b = new byte[90*4];
- int[] c = new int[a.length];
+ @Test
+ public void saulTest() {
+ for (ByteIntegerCODEC C : codecs) {
+ for (int x = 0; x < 50 * 4; ++x) {
+ int[] a = { 2, 3, 4, 5 };
+ byte[] b = new byte[90*4];
+ int[] c = new int[a.length];
- IntWrapper aOffset = new IntWrapper(0);
- IntWrapper bOffset = new IntWrapper(x);
- C.compress(a, aOffset, a.length, b, bOffset);
- int len = bOffset.get() - x;
+ IntWrapper aOffset = new IntWrapper(0);
+ IntWrapper bOffset = new IntWrapper(x);
+ C.compress(a, aOffset, a.length, b, bOffset);
+ int len = bOffset.get() - x;
- bOffset.set(x);
- IntWrapper cOffset = new IntWrapper(0);
- C.uncompress(b, bOffset, len, c, cOffset);
- if(!Arrays.equals(a, c)) {
- System.out.println("Problem with "+C);
- }
- assertArrayEquals(a, c);
- }
- }
- }
+ bOffset.set(x);
+ IntWrapper cOffset = new IntWrapper(0);
+ C.uncompress(b, bOffset, len, c, cOffset);
+ if(!Arrays.equals(a, c)) {
+ System.out.println("Problem with "+C);
+ }
+ assertArrayEquals(a, c);
+ }
+ }
+ }
/**
*
*/
diff --git a/src/test/java/me/lemire/integercompression/ExampleTest.java b/src/test/java/me/lemire/integercompression/ExampleTest.java
index f6038b8..c63c69b 100644
--- a/src/test/java/me/lemire/integercompression/ExampleTest.java
+++ b/src/test/java/me/lemire/integercompression/ExampleTest.java
@@ -17,305 +17,303 @@
*
*/
public class ExampleTest {
- /**
- *
- */
- @Test
-
- public void superSimpleExample() {
- IntegratedIntCompressor iic = new IntegratedIntCompressor();
- int[] data = new int[2342351];
- for (int k = 0; k < data.length; ++k)
- data[k] = k;
- System.out.println("Compressing " + data.length + " integers using friendly interface");
- int[] compressed = iic.compress(data);
- int[] recov = iic.uncompress(compressed);
- System.out
- .println("compressed from " + data.length * 4 / 1024 + "KB to " + compressed.length * 4 / 1024 + "KB");
- if (!Arrays.equals(recov, data))
- throw new RuntimeException("bug");
- }
-
- /**
- *
- */
- @Test
-
- public void basicExample() {
- int[] data = new int[2342351];
- System.out.println("Compressing " + data.length + " integers in one go");
- // data should be sorted for best
- // results
- for (int k = 0; k < data.length; ++k)
- data[k] = k;
- // Very important: the data is in sorted order!!! If not, you
- // will get very poor compression with IntegratedBinaryPacking,
- // you should use another CODEC.
-
- // next we compose a CODEC. Most of the processing
- // will be done with binary packing, and leftovers will
- // be processed using variable byte
- IntegratedIntegerCODEC codec = new IntegratedComposition(new IntegratedBinaryPacking(),
- new IntegratedVariableByte());
- // output vector should be large enough...
- int[] compressed = new int[data.length + 1024];
- // compressed might not be large enough in some cases
- // if you get java.lang.ArrayIndexOutOfBoundsException, try
- // allocating more memory
-
- /**
- *
- * compressing
- *
- */
- IntWrapper inputoffset = new IntWrapper(0);
- IntWrapper outputoffset = new IntWrapper(0);
- codec.compress(data, inputoffset, data.length, compressed, outputoffset);
- // got it!
- // inputoffset should be at data.length but outputoffset tells
- // us where we are...
- System.out.println(
- "compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
- // we can repack the data: (optional)
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
-
- /**
- *
- * now uncompressing
- *
- * This assumes that we otherwise know how many integers have been
- * compressed. See basicExampleHeadless for a more general case.
- */
- int[] recovered = new int[data.length];
- IntWrapper recoffset = new IntWrapper(0);
- codec.uncompress(compressed, new IntWrapper(0), compressed.length, recovered, recoffset);
- if (Arrays.equals(data, recovered))
- System.out.println("data is recovered without loss");
- else
- throw new RuntimeException("bug"); // could use assert
- System.out.println();
- }
-
- /**
- * Like the basicExample, but we store the input array size manually.
- */
- @Test
- public void basicExampleHeadless() {
- int[] data = new int[2342351];
- System.out.println("Compressing " + data.length + " integers in one go using the headless approach");
- // data should be sorted for best
- // results
- for (int k = 0; k < data.length; ++k)
- data[k] = k;
- // Very important: the data is in sorted order!!! If not, you
- // will get very poor compression with IntegratedBinaryPacking,
- // you should use another CODEC.
-
- // next we compose a CODEC. Most of the processing
- // will be done with binary packing, and leftovers will
- // be processed using variable byte
- SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
- new IntegratedVariableByte());
- // output vector should be large enough...
- int[] compressed = new int[data.length + 1024];
- // compressed might not be large enough in some cases
- // if you get java.lang.ArrayIndexOutOfBoundsException, try
- // allocating more memory
-
- /**
- *
- * compressing
- *
- */
- IntWrapper inputoffset = new IntWrapper(0);
- IntWrapper outputoffset = new IntWrapper(1);
- compressed[0] = data.length; // we manually store how many integers we
- codec.headlessCompress(data, inputoffset, data.length, compressed, outputoffset, new IntWrapper(0));
- // got it!
- // inputoffset should be at data.length but outputoffset tells
- // us where we are...
- System.out.println(
- "compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
- // we can repack the data: (optional)
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
-
- /**
- *
- * now uncompressing
- *
- */
- int howmany = compressed[0];// we manually stored the number of
- // compressed integers
- int[] recovered = new int[howmany];
- IntWrapper recoffset = new IntWrapper(0);
- codec.headlessUncompress(compressed, new IntWrapper(1), compressed.length, recovered, recoffset, howmany, new IntWrapper(0));
- if (Arrays.equals(data, recovered))
- System.out.println("data is recovered without loss");
- else
- throw new RuntimeException("bug"); // could use assert
- System.out.println();
- }
-
- /**
- * This is an example to show you can compress unsorted integers as long as
- * most are small.
- */
- @Test
- public void unsortedExample() {
- final int N = 1333333;
- int[] data = new int[N];
- // initialize the data (most will be small
- for (int k = 0; k < N; k += 1)
- data[k] = 3;
- // throw some larger values
- for (int k = 0; k < N; k += 5)
- data[k] = 100;
- for (int k = 0; k < N; k += 533)
- data[k] = 10000;
- int[] compressed = new int[N + 1024];// could need more
- IntegerCODEC codec = new Composition(new FastPFOR(), new VariableByte());
- // compressing
- IntWrapper inputoffset = new IntWrapper(0);
- IntWrapper outputoffset = new IntWrapper(0);
- codec.compress(data, inputoffset, data.length, compressed, outputoffset);
- System.out.println("compressed unsorted integers from " + data.length * 4 / 1024 + "KB to "
- + outputoffset.intValue() * 4 / 1024 + "KB");
- // we can repack the data: (optional)
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
-
- int[] recovered = new int[N];
- IntWrapper recoffset = new IntWrapper(0);
- codec.uncompress(compressed, new IntWrapper(0), compressed.length, recovered, recoffset);
- if (Arrays.equals(data, recovered))
- System.out.println("data is recovered without loss");
- else
- throw new RuntimeException("bug"); // could use assert
- System.out.println();
-
- }
-
- /**
- * This is like the basic example, but we show how to process larger arrays
- * in chunks.
- *
- * Some of this code was written by Pavel Klinov.
- */
- @Test
- public void advancedExample() {
- int TotalSize = 2342351; // some arbitrary number
- int ChunkSize = 16384; // size of each chunk, choose a multiple of 128
- System.out.println("Compressing " + TotalSize + " integers using chunks of " + ChunkSize + " integers ("
- + ChunkSize * 4 / 1024 + "KB)");
- System.out.println("(It is often better for applications to work in chunks fitting in CPU cache.)");
- int[] data = new int[TotalSize];
- // data should be sorted for best
- // results
- for (int k = 0; k < data.length; ++k)
- data[k] = k;
- // next we compose a CODEC. Most of the processing
- // will be done with binary packing, and leftovers will
- // be processed using variable byte, using variable byte
- // only for the last chunk!
- IntegratedIntegerCODEC regularcodec = new IntegratedBinaryPacking();
- IntegratedVariableByte ivb = new IntegratedVariableByte();
- IntegratedIntegerCODEC lastcodec = new IntegratedComposition(regularcodec, ivb);
- // output vector should be large enough...
- int[] compressed = new int[TotalSize + 1024];
-
- /**
- *
- * compressing
- *
- */
- IntWrapper inputoffset = new IntWrapper(0);
- IntWrapper outputoffset = new IntWrapper(0);
- for (int k = 0; k < TotalSize / ChunkSize; ++k)
- regularcodec.compress(data, inputoffset, ChunkSize, compressed, outputoffset);
- lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset);
- // got it!
- // inputoffset should be at data.length but outputoffset tells
- // us where we are...
- System.out.println(
- "compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
- // we can repack the data:
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
-
- /**
- *
- * now uncompressing
- *
- * We are *not* assuming that the original array length is known,
- * however we assume that the chunk size (ChunkSize) is known.
- *
- */
- int[] recovered = new int[ChunkSize];
- IntWrapper compoff = new IntWrapper(0);
- IntWrapper recoffset;
- int currentpos = 0;
-
- while (compoff.get() < compressed.length) {
- recoffset = new IntWrapper(0);
- regularcodec.uncompress(compressed, compoff, compressed.length - compoff.get(), recovered, recoffset);
-
- if (recoffset.get() < ChunkSize) {// last chunk detected
- ivb.uncompress(compressed, compoff, compressed.length - compoff.get(), recovered, recoffset);
- }
- for (int i = 0; i < recoffset.get(); ++i) {
- if (data[currentpos + i] != recovered[i])
- throw new RuntimeException("bug"); // could use assert
- }
- currentpos += recoffset.get();
- }
- System.out.println("data is recovered without loss");
- System.out.println();
-
- }
-
- /**
- * Demo of the headless approach where we must supply the array length
- */
- @Test
- public void headlessDemo() {
- System.out.println("Compressing arrays with minimal header...");
- int[] uncompressed1 = { 1, 2, 1, 3, 1 };
- int[] uncompressed2 = { 3, 2, 4, 6, 1 };
-
- int[] compressed = new int[uncompressed1.length + uncompressed2.length + 1024];
-
- SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
-
- // compressing
- IntWrapper outPos = new IntWrapper();
-
- IntWrapper previous = new IntWrapper();
-
- codec.headlessCompress(uncompressed1, new IntWrapper(), uncompressed1.length, compressed, outPos);
- int length1 = outPos.get() - previous.get();
- previous = new IntWrapper(outPos.get());
- codec.headlessCompress(uncompressed2, new IntWrapper(), uncompressed2.length, compressed, outPos);
- int length2 = outPos.get() - previous.get();
-
- compressed = Arrays.copyOf(compressed, length1 + length2);
- System.out
- .println("compressed unsorted integers from " + uncompressed1.length * 4 + "B to " + length1 * 4 + "B");
- System.out
- .println("compressed unsorted integers from " + uncompressed2.length * 4 + "B to " + length2 * 4 + "B");
- System.out.println("Total compressed output " + compressed.length);
-
- int[] recovered1 = new int[uncompressed1.length];
- int[] recovered2 = new int[uncompressed1.length];
- IntWrapper inPos = new IntWrapper();
- System.out.println("Decoding first array starting at pos = " + inPos);
- codec.headlessUncompress(compressed, inPos, compressed.length, recovered1, new IntWrapper(0),
- uncompressed1.length);
- System.out.println("Decoding second array starting at pos = " + inPos);
- codec.headlessUncompress(compressed, inPos, compressed.length, recovered2, new IntWrapper(0),
- uncompressed2.length);
- if (!Arrays.equals(uncompressed1, recovered1))
- throw new RuntimeException("First array does not match.");
- if (!Arrays.equals(uncompressed2, recovered2))
- throw new RuntimeException("Second array does not match.");
- System.out.println("The arrays match, your code is probably ok.");
-
- }
+ /**
+ *
+ */
+ @Test
+
+ public void superSimpleExample() {
+ IntegratedIntCompressor iic = new IntegratedIntCompressor();
+ int[] data = new int[2342351];
+ for (int k = 0; k < data.length; ++k)
+ data[k] = k;
+ System.out.println("Compressing " + data.length + " integers using friendly interface");
+ int[] compressed = iic.compress(data);
+ int[] recov = iic.uncompress(compressed);
+ System.out
+ .println("compressed from " + data.length * 4 / 1024 + "KB to " + compressed.length * 4 / 1024 + "KB");
+ if (!Arrays.equals(recov, data))
+ throw new RuntimeException("bug");
+ }
+
+ /**
+ *
+ */
+ @Test
+
+ public void basicExample() {
+ int[] data = new int[2342351];
+ System.out.println("Compressing " + data.length + " integers in one go");
+ // data should be sorted for best
+ // results
+ for (int k = 0; k < data.length; ++k)
+ data[k] = k;
+ // Very important: the data is in sorted order!!! If not, you
+ // will get very poor compression with IntegratedBinaryPacking,
+ // you should use another CODEC.
+
+ // next we compose a CODEC. Most of the processing
+ // will be done with binary packing, and leftovers will
+ // be processed using variable byte
+ IntegratedIntegerCODEC codec = new IntegratedComposition(new IntegratedBinaryPacking(),
+ new IntegratedVariableByte());
+ // output vector should be large enough...
+ int[] compressed = new int[data.length + 1024];
+ // compressed might not be large enough in some cases
+ // if you get java.lang.ArrayIndexOutOfBoundsException, try
+ // allocating more memory
+
+ /**
+ *
+ * compressing
+ *
+ */
+ IntWrapper inputoffset = new IntWrapper(0);
+ IntWrapper outputoffset = new IntWrapper(0);
+ codec.compress(data, inputoffset, data.length, compressed, outputoffset);
+ // got it!
+ // inputoffset should be at data.length but outputoffset tells
+ // us where we are...
+ System.out.println(
+ "compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
+ // we can repack the data: (optional)
+ compressed = Arrays.copyOf(compressed, outputoffset.intValue());
+
+ /**
+ *
+ * now uncompressing
+ *
+ * This assumes that we otherwise know how many integers have been
+ * compressed. See basicExampleHeadless for a more general case.
+ */
+ int[] recovered = new int[data.length];
+ IntWrapper recoffset = new IntWrapper(0);
+ codec.uncompress(compressed, new IntWrapper(0), compressed.length, recovered, recoffset);
+ if (Arrays.equals(data, recovered))
+ System.out.println("data is recovered without loss");
+ else
+ throw new RuntimeException("bug"); // could use assert
+ System.out.println();
+ }
+
+ /**
+ * Like the basicExample, but we store the input array size manually.
+ */
+ @Test
+ public void basicExampleHeadless() {
+ int[] data = new int[2342351];
+ System.out.println("Compressing " + data.length + " integers in one go using the headless approach");
+ // data should be sorted for best
+ // results
+ for (int k = 0; k < data.length; ++k)
+ data[k] = k;
+ // Very important: the data is in sorted order!!! If not, you
+ // will get very poor compression with IntegratedBinaryPacking,
+ // you should use another CODEC.
+
+ // next we compose a CODEC. Most of the processing
+ // will be done with binary packing, and leftovers will
+ // be processed using variable byte
+ SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),
+ new IntegratedVariableByte());
+ int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
+
+ /**
+ *
+ * compressing
+ *
+ */
+ IntWrapper inputoffset = new IntWrapper(0);
+ IntWrapper outputoffset = new IntWrapper(1);
+ compressed[0] = data.length; // we manually store how many integers we
+ codec.headlessCompress(data, inputoffset, data.length, compressed, outputoffset, new IntWrapper(0));
+ // got it!
+ // inputoffset should be at data.length but outputoffset tells
+ // us where we are...
+ System.out.println(
+ "compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
+ // we can repack the data: (optional)
+ compressed = Arrays.copyOf(compressed, outputoffset.intValue());
+
+ /**
+ *
+ * now uncompressing
+ *
+ */
+ int howmany = compressed[0];// we manually stored the number of
+ // compressed integers
+ int[] recovered = new int[howmany];
+ IntWrapper recoffset = new IntWrapper(0);
+ codec.headlessUncompress(compressed, new IntWrapper(1), compressed.length, recovered, recoffset, howmany, new IntWrapper(0));
+ if (Arrays.equals(data, recovered))
+ System.out.println("data is recovered without loss");
+ else
+ throw new RuntimeException("bug"); // could use assert
+ System.out.println();
+ }
+
+ /**
+ * This is an example to show you can compress unsorted integers as long as
+ * most are small.
+ */
+ @Test
+ public void unsortedExample() {
+ final int N = 1333333;
+ int[] data = new int[N];
+ // initialize the data (most will be small
+ for (int k = 0; k < N; k += 1)
+ data[k] = 3;
+ // throw some larger values
+ for (int k = 0; k < N; k += 5)
+ data[k] = 100;
+ for (int k = 0; k < N; k += 533)
+ data[k] = 10000;
+ int[] compressed = new int[N + 1024];// could need more
+ IntegerCODEC codec = new Composition(new FastPFOR(), new VariableByte());
+ // compressing
+ IntWrapper inputoffset = new IntWrapper(0);
+ IntWrapper outputoffset = new IntWrapper(0);
+ codec.compress(data, inputoffset, data.length, compressed, outputoffset);
+ System.out.println("compressed unsorted integers from " + data.length * 4 / 1024 + "KB to "
+ + outputoffset.intValue() * 4 / 1024 + "KB");
+ // we can repack the data: (optional)
+ compressed = Arrays.copyOf(compressed, outputoffset.intValue());
+
+ int[] recovered = new int[N];
+ IntWrapper recoffset = new IntWrapper(0);
+ codec.uncompress(compressed, new IntWrapper(0), compressed.length, recovered, recoffset);
+ if (Arrays.equals(data, recovered))
+ System.out.println("data is recovered without loss");
+ else
+ throw new RuntimeException("bug"); // could use assert
+ System.out.println();
+
+ }
+
+ /**
+ * This is like the basic example, but we show how to process larger arrays
+ * in chunks.
+ *
+ * Some of this code was written by Pavel Klinov.
+ */
+ @Test
+ public void advancedExample() {
+ int TotalSize = 2342351; // some arbitrary number
+ int ChunkSize = 16384; // size of each chunk, choose a multiple of 128
+ System.out.println("Compressing " + TotalSize + " integers using chunks of " + ChunkSize + " integers ("
+ + ChunkSize * 4 / 1024 + "KB)");
+ System.out.println("(It is often better for applications to work in chunks fitting in CPU cache.)");
+ int[] data = new int[TotalSize];
+ // data should be sorted for best
+ // results
+ for (int k = 0; k < data.length; ++k)
+ data[k] = k;
+ // next we compose a CODEC. Most of the processing
+ // will be done with binary packing, and leftovers will
+ // be processed using variable byte, using variable byte
+ // only for the last chunk!
+ IntegratedIntegerCODEC regularcodec = new IntegratedBinaryPacking();
+ IntegratedVariableByte ivb = new IntegratedVariableByte();
+ IntegratedIntegerCODEC lastcodec = new IntegratedComposition(regularcodec, ivb);
+ // output vector should be large enough...
+ int[] compressed = new int[TotalSize + 1024];
+
+ /**
+ *
+ * compressing
+ *
+ */
+ IntWrapper inputoffset = new IntWrapper(0);
+ IntWrapper outputoffset = new IntWrapper(0);
+ for (int k = 0; k < TotalSize / ChunkSize; ++k)
+ regularcodec.compress(data, inputoffset, ChunkSize, compressed, outputoffset);
+ lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset);
+ // got it!
+ // inputoffset should be at data.length but outputoffset tells
+ // us where we are...
+ System.out.println(
+ "compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");
+ // we can repack the data:
+ compressed = Arrays.copyOf(compressed, outputoffset.intValue());
+
+ /**
+ *
+ * now uncompressing
+ *
+ * We are *not* assuming that the original array length is known,
+ * however we assume that the chunk size (ChunkSize) is known.
+ *
+ */
+ int[] recovered = new int[ChunkSize];
+ IntWrapper compoff = new IntWrapper(0);
+ IntWrapper recoffset;
+ int currentpos = 0;
+
+ while (compoff.get() < compressed.length) {
+ recoffset = new IntWrapper(0);
+ regularcodec.uncompress(compressed, compoff, compressed.length - compoff.get(), recovered, recoffset);
+
+ if (recoffset.get() < ChunkSize) {// last chunk detected
+ ivb.uncompress(compressed, compoff, compressed.length - compoff.get(), recovered, recoffset);
+ }
+ for (int i = 0; i < recoffset.get(); ++i) {
+ if (data[currentpos + i] != recovered[i])
+ throw new RuntimeException("bug"); // could use assert
+ }
+ currentpos += recoffset.get();
+ }
+ System.out.println("data is recovered without loss");
+ System.out.println();
+
+ }
+
+ /**
+ * Demo of the headless approach where we must supply the array length
+ */
+ @Test
+ public void headlessDemo() {
+ System.out.println("Compressing arrays with minimal header...");
+ int[] uncompressed1 = { 1, 2, 1, 3, 1 };
+ int[] uncompressed2 = { 3, 2, 4, 6, 1 };
+
+ SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
+
+ int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed1.length)
+ + codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed2.length);
+ int[] compressed = new int[maxCompressedLength];
+
+ // compressing
+ IntWrapper outPos = new IntWrapper();
+
+ IntWrapper previous = new IntWrapper();
+
+ codec.headlessCompress(uncompressed1, new IntWrapper(), uncompressed1.length, compressed, outPos);
+ int length1 = outPos.get() - previous.get();
+ previous = new IntWrapper(outPos.get());
+ codec.headlessCompress(uncompressed2, new IntWrapper(), uncompressed2.length, compressed, outPos);
+ int length2 = outPos.get() - previous.get();
+
+ compressed = Arrays.copyOf(compressed, length1 + length2);
+ System.out
+ .println("compressed unsorted integers from " + uncompressed1.length * 4 + "B to " + length1 * 4 + "B");
+ System.out
+ .println("compressed unsorted integers from " + uncompressed2.length * 4 + "B to " + length2 * 4 + "B");
+ System.out.println("Total compressed output " + compressed.length);
+
+ int[] recovered1 = new int[uncompressed1.length];
+ int[] recovered2 = new int[uncompressed1.length];
+ IntWrapper inPos = new IntWrapper();
+ System.out.println("Decoding first array starting at pos = " + inPos);
+ codec.headlessUncompress(compressed, inPos, compressed.length, recovered1, new IntWrapper(0),
+ uncompressed1.length);
+ System.out.println("Decoding second array starting at pos = " + inPos);
+ codec.headlessUncompress(compressed, inPos, compressed.length, recovered2, new IntWrapper(0),
+ uncompressed2.length);
+ if (!Arrays.equals(uncompressed1, recovered1))
+ throw new RuntimeException("First array does not match.");
+ if (!Arrays.equals(uncompressed2, recovered2))
+ throw new RuntimeException("Second array does not match.");
+ System.out.println("The arrays match, your code is probably ok.");
+
+ }
}
diff --git a/src/test/java/me/lemire/integercompression/ResourcedTest.java b/src/test/java/me/lemire/integercompression/ResourcedTest.java
index 34f1d05..8316129 100644
--- a/src/test/java/me/lemire/integercompression/ResourcedTest.java
+++ b/src/test/java/me/lemire/integercompression/ResourcedTest.java
@@ -24,65 +24,65 @@
*
*/
public class ResourcedTest {
- SkippableIntegerCODEC[] codecs = { new JustCopy(), new VariableByte(),
- new SkippableComposition(new BinaryPacking(), new VariableByte()),
- new SkippableComposition(new NewPFD(), new VariableByte()),
- new SkippableComposition(new NewPFDS9(), new VariableByte()),
- new SkippableComposition(new NewPFDS16(), new VariableByte()),
- new SkippableComposition(new OptPFD(), new VariableByte()),
- new SkippableComposition(new OptPFDS9(), new VariableByte()),
- new SkippableComposition(new OptPFDS16(), new VariableByte()),
- new SkippableComposition(new FastPFOR128(), new VariableByte()),
- new SkippableComposition(new FastPFOR(), new VariableByte()), new Simple9(), new Simple16() };
+ SkippableIntegerCODEC[] codecs = { new JustCopy(), new VariableByte(),
+ new SkippableComposition(new BinaryPacking(), new VariableByte()),
+ new SkippableComposition(new NewPFD(), new VariableByte()),
+ new SkippableComposition(new NewPFDS9(), new VariableByte()),
+ new SkippableComposition(new NewPFDS16(), new VariableByte()),
+ new SkippableComposition(new OptPFD(), new VariableByte()),
+ new SkippableComposition(new OptPFDS9(), new VariableByte()),
+ new SkippableComposition(new OptPFDS16(), new VariableByte()),
+ new SkippableComposition(new FastPFOR128(), new VariableByte()),
+ new SkippableComposition(new FastPFOR(), new VariableByte()), new Simple9(), new Simple16() };
- /**
- * @throws IOException
- * if the resource cannot be accessed (should be considered a
- * bug)
- *
- */
- @Test
- public void IntCompressorTest() throws IOException {
- // next line requires Java8?
- // int[] data =
- // Files.lines(Paths.get("integers.txt")).mapToInt(Integer::parseInt).toArray();
- File f = new File("src/test/resources/integers.txt");
- System.out.println("loading test data from "+ f.getAbsolutePath());
- BufferedReader bfr = new BufferedReader(new FileReader(f));
- String line;
- ArrayList ai = new ArrayList();
- while ((line = bfr.readLine()) != null) {
- ai.add(Integer.parseInt(line));
- }
- bfr.close();
- int[] data = new int[ai.size()];
- for (int k = 0; k < data.length; ++k)
- data[k] = ai.get(k).intValue();
- ai = null;
- // finally!
- {
- IntegratedIntCompressor iic = new IntegratedIntCompressor();
- int[] compressed = iic.compress(data);
- int[] recovered = iic.uncompress(compressed);
- Assert.assertArrayEquals(recovered, data);
- }
- for (SkippableIntegerCODEC C : codecs) {
- IntCompressor iic = new IntCompressor(C);
- int[] compressed = iic.compress(data);
- int[] recovered = iic.uncompress(compressed);
- Assert.assertArrayEquals(recovered, data);
+ /**
+ * @throws IOException
+ * if the resource cannot be accessed (should be considered a
+ * bug)
+ *
+ */
+ @Test
+ public void IntCompressorTest() throws IOException {
+ // next line requires Java8?
+ // int[] data =
+ // Files.lines(Paths.get("integers.txt")).mapToInt(Integer::parseInt).toArray();
+ File f = new File("src/test/resources/integers.txt");
+ System.out.println("loading test data from "+ f.getAbsolutePath());
+ BufferedReader bfr = new BufferedReader(new FileReader(f));
+ String line;
+ ArrayList ai = new ArrayList();
+ while ((line = bfr.readLine()) != null) {
+ ai.add(Integer.parseInt(line));
+ }
+ bfr.close();
+ int[] data = new int[ai.size()];
+ for (int k = 0; k < data.length; ++k)
+ data[k] = ai.get(k).intValue();
+ ai = null;
+ // finally!
+ {
+ IntegratedIntCompressor iic = new IntegratedIntCompressor();
+ int[] compressed = iic.compress(data);
+ int[] recovered = iic.uncompress(compressed);
+ Assert.assertArrayEquals(recovered, data);
+ }
+ for (SkippableIntegerCODEC C : codecs) {
+ IntCompressor iic = new IntCompressor(C);
+ int[] compressed = iic.compress(data);
+ int[] recovered = iic.uncompress(compressed);
+ Assert.assertArrayEquals(recovered, data);
- }
- for (SkippableIntegerCODEC C : codecs) {
- if (C instanceof SkippableIntegratedIntegerCODEC) {
- IntegratedIntCompressor iic = new IntegratedIntCompressor((SkippableIntegratedIntegerCODEC) C);
- int[] compressed = iic.compress(data);
- int[] recovered = iic.uncompress(compressed);
- Assert.assertArrayEquals(recovered, data);
- }
+ }
+ for (SkippableIntegerCODEC C : codecs) {
+ if (C instanceof SkippableIntegratedIntegerCODEC) {
+ IntegratedIntCompressor iic = new IntegratedIntCompressor((SkippableIntegratedIntegerCODEC) C);
+ int[] compressed = iic.compress(data);
+ int[] recovered = iic.uncompress(compressed);
+ Assert.assertArrayEquals(recovered, data);
+ }
- }
+ }
- }
+ }
}
diff --git a/src/test/java/me/lemire/integercompression/SkippableBasicTest.java b/src/test/java/me/lemire/integercompression/SkippableBasicTest.java
index 93c1784..881dada 100644
--- a/src/test/java/me/lemire/integercompression/SkippableBasicTest.java
+++ b/src/test/java/me/lemire/integercompression/SkippableBasicTest.java
@@ -9,8 +9,14 @@
import java.util.Arrays;
+import me.lemire.integercompression.differential.IntegratedBinaryPacking;
+import me.lemire.integercompression.differential.IntegratedVariableByte;
+import me.lemire.integercompression.differential.SkippableIntegratedComposition;
+import me.lemire.integercompression.differential.SkippableIntegratedIntegerCODEC;
import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
/**
* Just some basic sanity tests.
@@ -19,7 +25,7 @@
*/
@SuppressWarnings({ "static-method" })
public class SkippableBasicTest {
- final SkippableIntegerCODEC[] codecs = {
+ final SkippableIntegerCODEC[] codecs = {
new JustCopy(),
new VariableByte(),
new SkippableComposition(new BinaryPacking(), new VariableByte()),
@@ -48,10 +54,11 @@ public void consistentTest() {
for (SkippableIntegerCODEC c : codecs) {
System.out.println("[SkippeableBasicTest.consistentTest] codec = "
+ c);
- int[] outBuf = new int[N + 1024];
for (int n = 0; n <= N; ++n) {
IntWrapper inPos = new IntWrapper();
IntWrapper outPos = new IntWrapper();
+ int[] outBuf = new int[c.maxHeadlessCompressedLength(new IntWrapper(0), n)];
+
c.headlessCompress(data, inPos, n, outBuf, outPos);
IntWrapper inPoso = new IntWrapper();
@@ -147,5 +154,135 @@ public void varyingLengthTest2() {
}
}
+ @Test
+ public void testMaxHeadlessCompressedLength() {
+ testMaxHeadlessCompressedLength(new IntegratedBinaryPacking(), 16 * IntegratedBinaryPacking.BLOCK_SIZE);
+ testMaxHeadlessCompressedLength(new IntegratedVariableByte(), 128);
+ testMaxHeadlessCompressedLength(new SkippableIntegratedComposition(new IntegratedBinaryPacking(), new IntegratedVariableByte()), 16 * IntegratedBinaryPacking.BLOCK_SIZE + 10);
+
+ testMaxHeadlessCompressedLength(new BinaryPacking(), 16 * BinaryPacking.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new VariableByte(), 128, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new BinaryPacking(), new VariableByte()), 16 * BinaryPacking.BLOCK_SIZE + 10, 32);
+ testMaxHeadlessCompressedLength(new JustCopy(), 128, 32);
+ testMaxHeadlessCompressedLength(new Simple9(), 128, 28);
+ testMaxHeadlessCompressedLength(new Simple16(), 128, 28);
+ testMaxHeadlessCompressedLength(new GroupSimple9(), 128, 28);
+ testMaxHeadlessCompressedLength(new OptPFD(), 4 * OptPFD.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new OptPFD(), new VariableByte()), 4 * OptPFD.BLOCK_SIZE + 10, 32);
+ testMaxHeadlessCompressedLength(new OptPFDS9(), 4 * OptPFDS9.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new OptPFDS9(), new VariableByte()), 4 * OptPFDS9.BLOCK_SIZE + 10, 32);
+ testMaxHeadlessCompressedLength(new OptPFDS16(), 4 * OptPFDS16.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new OptPFDS9(), new VariableByte()), 4 * OptPFDS16.BLOCK_SIZE + 10, 32);
+ testMaxHeadlessCompressedLength(new NewPFD(), 4 * NewPFD.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new NewPFD(), new VariableByte()), 4 * NewPFD.BLOCK_SIZE + 10, 32);
+ testMaxHeadlessCompressedLength(new NewPFDS9(), 4 * NewPFDS9.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new NewPFDS9(), new VariableByte()), 4 * NewPFDS9.BLOCK_SIZE + 10, 32);
+ testMaxHeadlessCompressedLength(new NewPFDS16(), 4 * NewPFDS16.BLOCK_SIZE, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new NewPFDS16(), new VariableByte()), 4 * NewPFDS16.BLOCK_SIZE + 10, 32);
+
+ int fastPfor128PageSize = FastPFOR128.BLOCK_SIZE * 4; // smaller page size than the default to speed up the test
+ testMaxHeadlessCompressedLength(new FastPFOR128(fastPfor128PageSize), 2 * fastPfor128PageSize, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new FastPFOR128(fastPfor128PageSize), new VariableByte()), 2 * fastPfor128PageSize + 10, 32);
+ int fastPforPageSize = FastPFOR.BLOCK_SIZE * 4; // smaller page size than the default to speed up the test
+ testMaxHeadlessCompressedLength(new FastPFOR(fastPforPageSize), 2 * fastPforPageSize, 32);
+ testMaxHeadlessCompressedLength(new SkippableComposition(new FastPFOR(fastPforPageSize), new VariableByte()), 2 * fastPforPageSize + 10, 32);
+ }
+
+ private static void testMaxHeadlessCompressedLength(SkippableIntegratedIntegerCODEC codec, int inlengthTo) {
+ // We test the worst-case scenario by making all deltas and the initial value negative.
+ int delta = -1;
+ int value = delta;
+ for (int inlength = 0; inlength < inlengthTo; ++inlength) {
+ int[] input = new int[inlength];
+ for (int i = 0; i < inlength; i++) {
+ input[i] = value;
+ value += delta;
+ }
+
+ int maxOutputLength = codec.maxHeadlessCompressedLength(new IntWrapper(), inlength);
+ int[] output = new int[maxOutputLength];
+ IntWrapper outPos = new IntWrapper();
+
+ codec.headlessCompress(input, new IntWrapper(), inlength, output, outPos, new IntWrapper());
+ // If we reach this point, no exception was thrown, which means the calculated output length was sufficient.
+
+ assertTrue(maxOutputLength <= outPos.get() + 1); // +1 because SkippableIntegratedComposition always adds one extra integer for the potential header
+ }
+ }
+
+ private static void testMaxHeadlessCompressedLength(SkippableIntegerCODEC codec, int inlengthTo, int maxBitWidth) {
+ // Some schemes ignore bit widths between 21 and 31. Therefore, in addition to maxBitWidth - 1, we also test 20.
+ assertTrue(maxBitWidth >= 20);
+ int[] regularValueBitWidths = { 20, maxBitWidth - 1 };
+
+ for (int inlength = 0; inlength < inlengthTo; ++inlength) {
+ int[] input = new int[inlength];
+
+ int maxOutputLength = codec.maxHeadlessCompressedLength(new IntWrapper(), inlength);
+ int[] output = new int[maxOutputLength];
+
+ for (int exceptionCount = 0; exceptionCount < inlength; exceptionCount++) {
+ int exception = maxBitWidth == 32 ? -1 : (1 << maxBitWidth) - 1;
+
+ for (int regularValueBitWidth : regularValueBitWidths) {
+ int regularValue = regularValueBitWidth == 32 ? -1 : (1 << regularValueBitWidth) - 1;
+
+ Arrays.fill(input, 0, exceptionCount, exception);
+ Arrays.fill(input, exceptionCount, input.length, regularValue);
+
+ codec.headlessCompress(input, new IntWrapper(), inlength, output, new IntWrapper());
+ // If we reach this point, no exception was thrown, which means the calculated output length was sufficient.
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testUncompressOutputOffset_SkippableComposition() {
+ for (int offset : new int[] {0, 1, 6}) {
+ SkippableComposition codec = new SkippableComposition(new BinaryPacking(), new VariableByte());
+
+ int[] input = { 2, 3, 4, 5 };
+ int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length)];
+ int[] uncompressed = new int[offset + input.length];
+
+ IntWrapper inputOffset = new IntWrapper(0);
+ IntWrapper compressedOffset = new IntWrapper(0);
+
+ codec.headlessCompress(input, inputOffset, input.length, compressed, compressedOffset);
+
+ int compressedLength = compressedOffset.get();
+ IntWrapper uncompressedOffset = new IntWrapper(offset);
+ compressedOffset = new IntWrapper(0);
+ codec.headlessUncompress(compressed, compressedOffset, compressedLength, uncompressed, uncompressedOffset, input.length);
+
+ assertArrayEquals(input, Arrays.copyOfRange(uncompressed, offset, offset + input.length));
+ }
+ }
+
+ @Test
+ public void testUncompressOutputOffset_SkippableIntegratedComposition() {
+ for (int offset : new int[] {0, 1, 6}) {
+ SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(), new IntegratedVariableByte());
+
+ int[] input = { 2, 3, 4, 5 };
+ int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length)];
+ int[] uncompressed = new int[offset + input.length];
+
+ IntWrapper inputOffset = new IntWrapper(0);
+ IntWrapper compressedOffset = new IntWrapper(0);
+ IntWrapper initValue = new IntWrapper(0);
+
+ codec.headlessCompress(input, inputOffset, input.length, compressed, compressedOffset, initValue);
+
+ int compressedLength = compressedOffset.get();
+ IntWrapper uncompressedOffset = new IntWrapper(offset);
+ compressedOffset = new IntWrapper(0);
+ initValue = new IntWrapper(0);
+ codec.headlessUncompress(compressed, compressedOffset, compressedLength, uncompressed, uncompressedOffset, input.length, initValue);
+
+ assertArrayEquals(input, Arrays.copyOfRange(uncompressed, offset, offset + input.length));
+ }
+ }
}
diff --git a/src/test/java/me/lemire/integercompression/TestUtils.java b/src/test/java/me/lemire/integercompression/TestUtils.java
index 7ce51b3..b3cbff3 100644
--- a/src/test/java/me/lemire/integercompression/TestUtils.java
+++ b/src/test/java/me/lemire/integercompression/TestUtils.java
@@ -165,7 +165,7 @@ protected static int[] uncompress(ByteIntegerCODEC codec, byte[] data, int len)
}
protected static int[] compressHeadless(SkippableIntegerCODEC codec, int[] data) {
- int[] outBuf = new int[data.length * 4];
+ int[] outBuf = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
IntWrapper inPos = new IntWrapper();
IntWrapper outPos = new IntWrapper();
codec.headlessCompress(data, inPos, data.length, outBuf, outPos);
diff --git a/src/test/java/me/lemire/longcompression/ATestLongCODEC.java b/src/test/java/me/lemire/longcompression/ATestLongCODEC.java
new file mode 100644
index 0000000..c61ea69
--- /dev/null
+++ b/src/test/java/me/lemire/longcompression/ATestLongCODEC.java
@@ -0,0 +1,96 @@
+/**
+ * This code is released under the
+ * Apache License Version 2.0 http://www.apache.org/licenses/.
+ *
+ * (c) Daniel Lemire, http://lemire.me/en/
+ */
+
+package me.lemire.longcompression;
+
+import java.util.stream.LongStream;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Edge-cases to be tested on a per-codec basis
+ *
+ * @author Benoit Lacelle
+ */
+public abstract class ATestLongCODEC {
+ protected void checkConsistency(LongCODEC codec, long[] array) {
+ {
+ long[] compressed = LongTestUtils.compress(codec, array);
+ long[] uncompressed = LongTestUtils.uncompress(codec, compressed, array.length);
+
+ Assert.assertArrayEquals(array, uncompressed);
+ }
+
+ if (codec instanceof ByteLongCODEC) {
+ byte[] compressed = LongTestUtils.compress((ByteLongCODEC) codec, array);
+ long[] uncompressed = LongTestUtils.uncompress((ByteLongCODEC) codec, compressed, array.length);
+
+ Assert.assertArrayEquals(array, uncompressed);
+ }
+
+ if (codec instanceof SkippableLongCODEC) {
+ long[] compressed = LongTestUtils.compressHeadless((SkippableLongCODEC) codec, array);
+ long[] uncompressed =
+ LongTestUtils.uncompressHeadless((SkippableLongCODEC) codec, compressed, array.length);
+
+ Assert.assertArrayEquals(array, uncompressed);
+ }
+ }
+
+ public abstract LongCODEC getCodec();
+
+ @Test
+ public void testCodec_Minus1() {
+ checkConsistency(getCodec(), new long[] { -1 });
+ }
+
+ @Test
+ public void testCodec_ZeroTimes8Minus1() {
+ checkConsistency(getCodec(), new long[] { 0, 0, 0, 0, 0, 0, 0, 0, -1 });
+ }
+
+ @Test
+ public void testCodec_ZeroTimes127Minus1() {
+ long[] array = LongStream.concat(LongStream.range(0, 127).map(l -> 0), LongStream.of(-1)).toArray();
+
+ checkConsistency(getCodec(), array);
+ }
+
+ @Test
+ public void testCodec_ZeroTimes128Minus1() {
+ long[] array = LongStream.concat(LongStream.range(0, 128).map(l -> 0), LongStream.of(-1)).toArray();
+
+ checkConsistency(getCodec(), array);
+ }
+
+ @Test
+ public void testCodec_MinValue() {
+ checkConsistency(getCodec(), new long[] { Long.MIN_VALUE });
+ }
+
+ @Test
+ public void testCodec_ZeroMinValue() {
+ checkConsistency(getCodec(), new long[] { 0, Long.MIN_VALUE });
+ }
+
+ @Test
+ public void testCodec_allPowerOfTwo() {
+ checkConsistency(getCodec(), new long[] { 1L << 42 });
+ for (int i = 0; i < 64; i++) {
+ checkConsistency(getCodec(), new long[] { 1L << i });
+ }
+ }
+
+ @Test
+ public void testCodec_ZeroThenAllPowerOfTwo() {
+ for (int i = 0; i < 64; i++) {
+ checkConsistency(getCodec(), new long[] { 0, 1L << i });
+ }
+ }
+
+}
diff --git a/src/test/java/me/lemire/longcompression/LongBasicTest.java b/src/test/java/me/lemire/longcompression/LongBasicTest.java
index 5aa3551..8dc0c9b 100644
--- a/src/test/java/me/lemire/longcompression/LongBasicTest.java
+++ b/src/test/java/me/lemire/longcompression/LongBasicTest.java
@@ -14,24 +14,9 @@
import org.junit.Test;
-import me.lemire.integercompression.BinaryPacking;
-import me.lemire.integercompression.Composition;
import me.lemire.integercompression.FastPFOR;
import me.lemire.integercompression.FastPFOR128;
import me.lemire.integercompression.IntWrapper;
-import me.lemire.integercompression.JustCopy;
-import me.lemire.integercompression.NewPFD;
-import me.lemire.integercompression.NewPFDS16;
-import me.lemire.integercompression.NewPFDS9;
-import me.lemire.integercompression.OptPFD;
-import me.lemire.integercompression.OptPFDS16;
-import me.lemire.integercompression.OptPFDS9;
-import me.lemire.integercompression.Simple9;
-import me.lemire.integercompression.VariableByte;
-import me.lemire.integercompression.differential.Delta;
-import me.lemire.integercompression.differential.IntegratedBinaryPacking;
-import me.lemire.integercompression.differential.IntegratedComposition;
-import me.lemire.integercompression.differential.IntegratedVariableByte;
import me.lemire.longcompression.differential.LongDelta;
import me.lemire.longcompression.synth.LongClusteredDataGenerator;
@@ -45,35 +30,37 @@ public class LongBasicTest {
final LongCODEC[] codecs = {
new LongJustCopy(),
new LongVariableByte(),
- new LongAs2IntsCodec()};
+ new LongAs2IntsCodec(),
+ new LongComposition(new LongBinaryPacking(), new LongVariableByte()),
+ };
- /**
+ /**
* This tests with a compressed array with various offset
*/
- @Test
- public void saulTest() {
- for (LongCODEC C : codecs) {
- for (int x = 0; x < 50; ++x) {
- long[] a = { 2, 3, 4, 5 };
- long[] b = new long[90];
- long[] c = new long[a.length];
-
- IntWrapper aOffset = new IntWrapper(0);
- IntWrapper bOffset = new IntWrapper(x);
- C.compress(a, aOffset, a.length, b, bOffset);
- int len = bOffset.get() - x;
-
- bOffset.set(x);
- IntWrapper cOffset = new IntWrapper(0);
- C.uncompress(b, bOffset, len, c, cOffset);
- if(!Arrays.equals(a, c)) {
- System.out.println("Problem with "+C);
- }
- assertArrayEquals(a, c);
-
- }
- }
- }
+ @Test
+ public void saulTest() {
+ for (LongCODEC C : codecs) {
+ for (int x = 0; x < 50; ++x) {
+ long[] a = { 2, 3, 4, 5 };
+ long[] b = new long[90];
+ long[] c = new long[a.length];
+
+ IntWrapper aOffset = new IntWrapper(0);
+ IntWrapper bOffset = new IntWrapper(x);
+ C.compress(a, aOffset, a.length, b, bOffset);
+ int len = bOffset.get() - x;
+
+ bOffset.set(x);
+ IntWrapper cOffset = new IntWrapper(0);
+ C.uncompress(b, bOffset, len, c, cOffset);
+ if(!Arrays.equals(a, c)) {
+ System.out.println("Problem with "+C);
+ }
+ assertArrayEquals(a, c);
+
+ }
+ }
+ }
/**
*
*/
@@ -89,14 +76,19 @@ public void varyingLengthTest() {
long[] comp = LongTestUtils.compress(c, Arrays.copyOf(data, L));
long[] answer = LongTestUtils.uncompress(c, comp, L);
for (int k = 0; k < L; ++k)
- if (answer[k] != data[k])
- throw new RuntimeException("bug");
+ if (answer[k] != data[k]) {
+ long[] comp2 = LongTestUtils.compress(c, Arrays.copyOf(data, L));
+ long[] answer2 = LongTestUtils.uncompress(c, comp2, L);
+ throw new RuntimeException("bug");
+ }
}
for (int L = 128; L <= N; L *= 2) {
long[] comp = LongTestUtils.compress(c, Arrays.copyOf(data, L));
long[] answer = LongTestUtils.uncompress(c, comp, L);
for (int k = 0; k < L; ++k)
if (answer[k] != data[k]) {
+ long[] comp2 = LongTestUtils.compress(c, Arrays.copyOf(data, L));
+ long[] answer2 = LongTestUtils.uncompress(c, comp2, L);
System.out.println(Arrays.toString(Arrays.copyOf(
answer, L)));
System.out.println(Arrays.toString(Arrays.copyOf(data,
@@ -167,16 +159,16 @@ public void varyingLengthTest2() {
@Test
public void checkVariousCases() {
for (LongCODEC c : codecs) {
- testZeroInZeroOut(c);
- test(c, c, 5, 10);
- test(c, c, 5, 14);
- test(c, c, 2, 18);
- // TODO Unclear which codec should manage an empty output array or not
- // Some IntegerCodec does not output anything if the input is smaller than some block size
- // testSpurious(c);
- testUnsorted(c);
- testUnsorted2(c);
- testUnsorted3(c);
+ testZeroInZeroOut(c);
+ test(c, c, 5, 10);
+ test(c, c, 5, 14);
+ test(c, c, 2, 18);
+ // TODO Unclear which codec should manage an empty output array or not
+ // Some IntegerCodec does not output anything if the input is smaller than some block size
+ // testSpurious(c);
+ testUnsorted(c);
+ testUnsorted2(c);
+ testUnsorted3(c);
}
}
@@ -273,7 +265,7 @@ private static void testCodec(LongCODEC c, LongCODEC co,
buffer[0] = backupdata[0];
co.uncompress(dataout, inpos, thiscompsize - 1, buffer, outpos);
if (!(c instanceof IntegratedLongCODEC))
- LongDelta.fastinverseDelta(buffer);
+ LongDelta.fastinverseDelta(buffer);
// Check assertions.
assertEquals("length is not match", outpos.get(), data[k].length);
@@ -366,9 +358,12 @@ public void fastPforTest() {
long[] comp = LongTestUtils.compress(codec, Arrays.copyOf(data, N));
long[] answer = LongTestUtils.uncompress(codec, comp, N);
for (int k = 0; k < N; ++k)
- if (answer[k] != data[k])
+ if (answer[k] != data[k]) {
+ long[] comp2 = LongTestUtils.compress(codec, Arrays.copyOf(data, N));
+ long[] answer2 = LongTestUtils.uncompress(codec, comp2, N);
throw new RuntimeException("bug " + k + " " + answer[k]
+ " != " + data[k]);
+ }
}
}
@@ -378,19 +373,19 @@ public void fastPforTest() {
@Test
public void fastPfor128Test() {
// proposed by Stefan Ackermann (https://github.com/Stivo)
- for (LongCODEC codec : codecs) {
- int N = FastPFOR128.BLOCK_SIZE;
- long[] data = new long[N];
- for (int i = 0; i < N; i++)
- data[i] = 0;
- data[126] = -1;
- long[] comp = LongTestUtils.compress(codec, Arrays.copyOf(data, N));
- long[] answer = LongTestUtils.uncompress(codec, comp, N);
- for (int k = 0; k < N; ++k)
- if (answer[k] != data[k])
- throw new RuntimeException("bug " + k + " " + answer[k]
- + " != " + data[k]);
- }
+ for (LongCODEC codec : codecs) {
+ int N = FastPFOR128.BLOCK_SIZE;
+ long[] data = new long[N];
+ for (int i = 0; i < N; i++)
+ data[i] = 0;
+ data[126] = -1;
+ long[] comp = LongTestUtils.compress(codec, Arrays.copyOf(data, N));
+ long[] answer = LongTestUtils.uncompress(codec, comp, N);
+ for (int k = 0; k < N; ++k)
+ if (answer[k] != data[k])
+ throw new RuntimeException("bug " + k + " " + answer[k]
+ + " != " + data[k]);
+ }
}
}
diff --git a/src/test/java/me/lemire/longcompression/LongTestUtils.java b/src/test/java/me/lemire/longcompression/LongTestUtils.java
index a44e665..b7d9c63 100644
--- a/src/test/java/me/lemire/longcompression/LongTestUtils.java
+++ b/src/test/java/me/lemire/longcompression/LongTestUtils.java
@@ -111,7 +111,7 @@ protected static long[] uncompress(ByteLongCODEC codec, byte[] data, int len) {
}
protected static long[] compressHeadless(SkippableLongCODEC codec, long[] data) {
- long[] outBuf = new long[data.length * 4];
+ long[] outBuf = new long[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];
IntWrapper inPos = new IntWrapper();
IntWrapper outPos = new IntWrapper();
codec.headlessCompress(data, inPos, data.length, outBuf, outPos);
@@ -127,7 +127,7 @@ protected static long[] uncompressHeadless(SkippableLongCODEC codec, long[] data
return Arrays.copyOf(outBuf, outPos.get());
}
- public static String longToBinaryWithLeading(long l) {
- return String.format("%64s", Long.toBinaryString(l)).replace(' ', '0');
- }
+ public static String longToBinaryWithLeading(long l) {
+ return String.format("%64s", Long.toBinaryString(l)).replace(' ', '0');
+ }
}
diff --git a/src/test/java/me/lemire/longcompression/SkippableLongBasicTest.java b/src/test/java/me/lemire/longcompression/SkippableLongBasicTest.java
index e900c9c..c4b7e01 100644
--- a/src/test/java/me/lemire/longcompression/SkippableLongBasicTest.java
+++ b/src/test/java/me/lemire/longcompression/SkippableLongBasicTest.java
@@ -15,6 +15,8 @@
import me.lemire.integercompression.TestUtils;
import me.lemire.integercompression.VariableByte;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
/**
* Just some basic sanity tests.
@@ -25,7 +27,8 @@
public class SkippableLongBasicTest {
final SkippableLongCODEC[] codecs = {
new LongJustCopy(),
- new LongVariableByte(), };
+ new LongVariableByte(),
+ new SkippableLongComposition(new LongBinaryPacking(), new LongVariableByte()), };
/**
@@ -41,10 +44,11 @@ public void consistentTest() {
for (SkippableLongCODEC c : codecs) {
System.out.println("[SkippeableBasicTest.consistentTest] codec = "
+ c);
- long[] outBuf = new long[N + 1024];
for (int n = 0; n <= N; ++n) {
IntWrapper inPos = new IntWrapper();
IntWrapper outPos = new IntWrapper();
+ long[] outBuf = new long[c.maxHeadlessCompressedLength(new IntWrapper(0), n)];
+
c.headlessCompress(data, inPos, n, outBuf, outPos);
IntWrapper inPoso = new IntWrapper();
@@ -78,15 +82,15 @@ public void varyingLengthTest() {
for (SkippableLongCODEC c : codecs) {
System.out.println("[SkippeableBasicTest.varyingLengthTest] codec = "+c);
for (int L = 1; L <= 128; L++) {
- long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
- long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
+ long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
+ long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
for (int k = 0; k < L; ++k)
if (answer[k] != data[k])
throw new RuntimeException("bug "+c.toString()+" "+k+" "+answer[k]+" "+data[k]);
}
for (int L = 128; L <= N; L *= 2) {
- long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
- long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
+ long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
+ long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
for (int k = 0; k < L; ++k)
if (answer[k] != data[k])
throw new RuntimeException("bug");
@@ -123,16 +127,16 @@ public void varyingLengthTest2() {
e.printStackTrace();
}
for (int L = 1; L <= 128; L++) {
- long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
- long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
+ long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
+ long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
for (int k = 0; k < L; ++k)
if (answer[k] != data[k]) {
- throw new RuntimeException("L=" + L + ": bug at k = "+k+" "+answer[k]+" "+data[k]+" for "+c.toString());
+ throw new RuntimeException("L=" + L + ": bug at k = "+k+" "+answer[k]+" "+data[k]+" for "+c.toString());
}
}
for (int L = 128; L <= N; L *= 2) {
- long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
- long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
+ long[] comp = LongTestUtils.compressHeadless(c, Arrays.copyOf(data, L));
+ long[] answer = LongTestUtils.uncompressHeadless(c, comp, L);
for (int k = 0; k < L; ++k)
if (answer[k] != data[k])
throw new RuntimeException("bug");
@@ -141,5 +145,50 @@ public void varyingLengthTest2() {
}
}
+ @Test
+ public void testMaxHeadlessCompressedLength() {
+ testMaxHeadlessCompressedLength(new LongJustCopy(), 128);
+ testMaxHeadlessCompressedLength(new LongBinaryPacking(), 16 * LongBinaryPacking.BLOCK_SIZE);
+ testMaxHeadlessCompressedLength(new LongVariableByte(), 128);
+ testMaxHeadlessCompressedLength(new SkippableLongComposition(new LongBinaryPacking(), new LongVariableByte()), 16 * LongBinaryPacking.BLOCK_SIZE + 10);
+ }
+
+ private static void testMaxHeadlessCompressedLength(SkippableLongCODEC codec, int inlengthTo) {
+ for (int inlength = 0; inlength < inlengthTo; ++inlength) {
+ long[] input = new long[inlength];
+ Arrays.fill(input, -1L);
+
+ int maxOutputLength = codec.maxHeadlessCompressedLength(new IntWrapper(), inlength);
+ long[] output = new long[maxOutputLength];
+ IntWrapper outPos = new IntWrapper();
+
+ codec.headlessCompress(input, new IntWrapper(), inlength, output, outPos);
+ // If we reach this point, no exception was thrown, which means the calculated output length was sufficient.
+
+ assertTrue(maxOutputLength <= outPos.get() + 1); // +1 because SkippableLongComposition always adds one extra integer for the potential header
+ }
+ }
+ @Test
+ public void testUncompressOutputOffset_SkippableLongComposition() {
+ for (int offset : new int[] {0, 1, 6}) {
+ SkippableLongComposition codec = new SkippableLongComposition(new LongBinaryPacking(), new LongVariableByte());
+
+ long[] input = { 2, 3, 4, 5 };
+ long[] compressed = new long[codec.maxHeadlessCompressedLength(new IntWrapper(0), input.length)];
+ long[] uncompressed = new long[offset + input.length];
+
+ IntWrapper inputOffset = new IntWrapper(0);
+ IntWrapper compressedOffset = new IntWrapper(0);
+
+ codec.headlessCompress(input, inputOffset, input.length, compressed, compressedOffset);
+
+ int compressedLength = compressedOffset.get();
+ IntWrapper uncompressedOffset = new IntWrapper(offset);
+ compressedOffset = new IntWrapper(0);
+ codec.headlessUncompress(compressed, compressedOffset, compressedLength, uncompressed, uncompressedOffset, input.length);
+
+ assertArrayEquals(input, Arrays.copyOfRange(uncompressed, offset, offset + input.length));
+ }
+ }
}
diff --git a/src/test/java/me/lemire/longcompression/TestLongAs2IntsCodec.java b/src/test/java/me/lemire/longcompression/TestLongAs2IntsCodec.java
index 00bb52a..bddff2a 100644
--- a/src/test/java/me/lemire/longcompression/TestLongAs2IntsCodec.java
+++ b/src/test/java/me/lemire/longcompression/TestLongAs2IntsCodec.java
@@ -7,8 +7,6 @@
package me.lemire.longcompression;
-import java.util.stream.LongStream;
-
import org.junit.Assert;
import org.junit.Test;
@@ -17,90 +15,17 @@
*
* @author Benoit Lacelle
*/
-public class TestLongAs2IntsCodec {
+public class TestLongAs2IntsCodec extends ATestLongCODEC {
final LongAs2IntsCodec codec = new LongAs2IntsCodec();
- private void checkConsistency(LongCODEC codec, long[] array) {
- {
- long[] compressed = LongTestUtils.compress(codec, array);
- long[] uncompressed = LongTestUtils.uncompress(codec, compressed, array.length);
-
- Assert.assertArrayEquals(array, uncompressed);
- }
-
- if (codec instanceof ByteLongCODEC) {
- byte[] compressed = LongTestUtils.compress((ByteLongCODEC) codec, array);
- long[] uncompressed = LongTestUtils.uncompress((ByteLongCODEC) codec, compressed, array.length);
-
- Assert.assertArrayEquals(array, uncompressed);
- }
-
- if (codec instanceof SkippableLongCODEC) {
- long[] compressed = LongTestUtils.compressHeadless((SkippableLongCODEC) codec, array);
- long[] uncompressed =
- LongTestUtils.uncompressHeadless((SkippableLongCODEC) codec, compressed, array.length);
-
- Assert.assertArrayEquals(array, uncompressed);
- }
- }
-
- @Test
- public void testCodec_Zero() {
- checkConsistency(codec, new long[] { 0 });
- }
-
- @Test
- public void testCodec_Minus1() {
- checkConsistency(codec, new long[] { -1 });
+ @Override
+ public LongCODEC getCodec() {
+ return codec;
}
- @Test
- public void testCodec_ZeroTimes8Minus1() {
- checkConsistency(codec, new long[] { 0, 0, 0, 0, 0, 0, 0, 0, -1 });
- }
-
- @Test
- public void testCodec_ZeroTimes127Minus1() {
- long[] array = LongStream.concat(LongStream.range(0, 127).map(l -> 0), LongStream.of(-1)).toArray();
-
- checkConsistency(codec, array);
- }
-
- @Test
- public void testCodec_ZeroTimes128Minus1() {
- long[] array = LongStream.concat(LongStream.range(0, 128).map(l -> 0), LongStream.of(-1)).toArray();
-
- checkConsistency(codec, array);
- }
-
- @Test
- public void testCodec_MinValue() {
- checkConsistency(codec, new long[] { Long.MIN_VALUE });
- }
-
- @Test
- public void testCodec_ZeroMinValue() {
- checkConsistency(codec, new long[] { 0, Long.MIN_VALUE });
- }
-
- @Test
- public void testCodec_allPowerOfTwo() {
- checkConsistency(codec, new long[] { 1L << 42 });
- for (int i = 0; i < 64; i++) {
- checkConsistency(codec, new long[] { 1L << i });
- }
- }
-
- @Test
- public void testCodec_ZeroThenAllPowerOfTwo() {
- for (int i = 0; i < 64; i++) {
- checkConsistency(codec, new long[] { 0, 1L << i });
- }
- }
-
- @Test
- public void testCodec_intermediateHighPowerOfTwo() {
- Assert.assertEquals(3, LongTestUtils.compress((LongCODEC) codec, new long[] { 1L << 42 }).length);
- }
+ @Test
+ public void testCodec_intermediateHighPowerOfTwo() {
+ Assert.assertEquals(3, LongTestUtils.compress((LongCODEC) codec, new long[] { 1L << 42 }).length);
+ }
}
diff --git a/src/test/java/me/lemire/longcompression/TestLongBinaryPacking.java b/src/test/java/me/lemire/longcompression/TestLongBinaryPacking.java
new file mode 100644
index 0000000..ecc3f2e
--- /dev/null
+++ b/src/test/java/me/lemire/longcompression/TestLongBinaryPacking.java
@@ -0,0 +1,26 @@
+/**
+ * This code is released under the
+ * Apache License Version 2.0 http://www.apache.org/licenses/.
+ *
+ * (c) Daniel Lemire, http://lemire.me/en/
+ */
+
+package me.lemire.longcompression;
+
+import org.junit.Ignore;
+
+/**
+ * Edge-cases having caused issue specifically with LongBinaryPacking.
+ *
+ * @author Benoit Lacelle
+ */
+@Ignore("Parent class tests are not valid as LongBinaryPacking process by chunks of 64 longs")
+public class TestLongBinaryPacking extends ATestLongCODEC {
+ final LongBinaryPacking codec = new LongBinaryPacking();
+
+ @Override
+ public LongCODEC getCodec() {
+ return codec;
+ }
+
+}
diff --git a/src/test/java/me/lemire/longcompression/TestLongVariableByte.java b/src/test/java/me/lemire/longcompression/TestLongVariableByte.java
index 15613f2..3cb2a49 100644
--- a/src/test/java/me/lemire/longcompression/TestLongVariableByte.java
+++ b/src/test/java/me/lemire/longcompression/TestLongVariableByte.java
@@ -7,8 +7,6 @@
package me.lemire.longcompression;
-import java.util.stream.LongStream;
-
import org.junit.Assert;
import org.junit.Test;
@@ -17,87 +15,26 @@
*
* @author Benoit Lacelle
*/
-public class TestLongVariableByte {
+public class TestLongVariableByte extends ATestLongCODEC {
final LongVariableByte codec = new LongVariableByte();
- private void checkConsistency(LongCODEC codec, long[] array) {
- {
- long[] compressed = LongTestUtils.compress(codec, array);
- long[] uncompressed = LongTestUtils.uncompress(codec, compressed, array.length);
-
- Assert.assertArrayEquals(array, uncompressed);
- }
-
- if (codec instanceof ByteLongCODEC) {
- byte[] compressed = LongTestUtils.compress((ByteLongCODEC) codec, array);
- long[] uncompressed = LongTestUtils.uncompress((ByteLongCODEC) codec, compressed, array.length);
-
- Assert.assertArrayEquals(array, uncompressed);
- }
-
- if (codec instanceof SkippableLongCODEC) {
- long[] compressed = LongTestUtils.compressHeadless((SkippableLongCODEC) codec, array);
- long[] uncompressed =
- LongTestUtils.uncompressHeadless((SkippableLongCODEC) codec, compressed, array.length);
-
- Assert.assertArrayEquals(array, uncompressed);
- }
- }
-
- @Test
- public void testCodec_ZeroMinus1() {
- checkConsistency(codec, new long[] { -1 });
+ @Override
+ public LongCODEC getCodec() {
+ return codec;
}
@Test
- public void testCodec_ZeroTimes8Minus1() {
- checkConsistency(codec, new long[] { 0, 0, 0, 0, 0, 0, 0, 0, -1 });
- }
+ public void testCodec_allBitWidths() {
+ for (int bitWidth = 0; bitWidth <= 64; bitWidth++) {
+ long value = bitWidth == 0 ? 0 : 1L << (bitWidth - 1);
- @Test
- public void testCodec_ZeroTimes127Minus1() {
- long[] array = LongStream.concat(LongStream.range(0, 127).map(l -> 0), LongStream.of(-1)).toArray();
-
- checkConsistency(codec, array);
- }
-
- @Test
- public void testCodec_ZeroTimes128Minus1() {
- long[] array = LongStream.concat(LongStream.range(0, 128).map(l -> 0), LongStream.of(-1)).toArray();
+ int expectedSizeInBytes = Math.max(1, (bitWidth + 6) / 7);
+ int expectedSizeInLongs = (expectedSizeInBytes > 8) ? 2 : 1;
- checkConsistency(codec, array);
- }
-
- @Test
- public void testCodec_MinValue() {
- checkConsistency(codec, new long[] { Long.MIN_VALUE });
- }
-
- @Test
- public void testCodec_ZeroMinValue() {
- checkConsistency(codec, new long[] { 0, Long.MIN_VALUE });
- }
-
- @Test
- public void testCodec_allPowerOfTwo() {
- checkConsistency(codec, new long[] { 1L << 42 });
- for (int i = 0; i < 64; i++) {
- checkConsistency(codec, new long[] { 1L << i });
+ Assert.assertEquals(expectedSizeInLongs, LongTestUtils.compress((LongCODEC) codec, new long[] { value }).length);
+ Assert.assertEquals(expectedSizeInBytes, LongTestUtils.compress((ByteLongCODEC) codec, new long[] { value }).length);
+ Assert.assertEquals(expectedSizeInLongs,
+ LongTestUtils.compressHeadless((SkippableLongCODEC) codec, new long[] { value }).length);
}
}
-
- @Test
- public void testCodec_ZeroThenAllPowerOfTwo() {
- for (int i = 0; i < 64; i++) {
- checkConsistency(codec, new long[] { 0, 1L << i });
- }
- }
-
- @Test
- public void testCodec_intermediateHighPowerOfTwo() {
- Assert.assertEquals(1, LongTestUtils.compress((LongCODEC) codec, new long[] { 1L << 42 }).length);
- Assert.assertEquals(7, LongTestUtils.compress((ByteLongCODEC) codec, new long[] { 1L << 42 }).length);
- Assert.assertEquals(1, LongTestUtils.compressHeadless((SkippableLongCODEC) codec, new long[] { 1L << 42 }).length);
- }
-
}
diff --git a/src/test/java/me/lemire/longcompression/synth/LongClusteredDataGenerator.java b/src/test/java/me/lemire/longcompression/synth/LongClusteredDataGenerator.java
index 5b90ee0..c964f6f 100644
--- a/src/test/java/me/lemire/longcompression/synth/LongClusteredDataGenerator.java
+++ b/src/test/java/me/lemire/longcompression/synth/LongClusteredDataGenerator.java
@@ -29,7 +29,7 @@ public LongClusteredDataGenerator() {
}
void fillUniform(long[] array, int offset, int length, long Min, long Max) {
- long[] v = this.unidg.generateUniform(length, Max - Min);
+ long[] v = this.unidg.generateUniform(length, Max - Min);
for (int k = 0; k < v.length; ++k)
array[k + offset] = Min + v[k];
}
@@ -70,7 +70,7 @@ void fillClustered(long[] array, int offset, int length, long Min, long Max) {
* @return array containing the integers
*/
public long[] generateClustered(int N, long Max) {
- long[] array = new long[N];
+ long[] array = new long[N];
fillClustered(array, 0, N, 0, Max);
return array;
}
@@ -82,7 +82,7 @@ public long[] generateClustered(int N, long Max) {
* arguments are ignored
*/
public static void main(final String[] args) {
- long[] example = (new LongClusteredDataGenerator())
+ long[] example = (new LongClusteredDataGenerator())
.generateClustered(20, 1000);
for (int k = 0; k < example.length; ++k)
System.out.println(example[k]);
diff --git a/src/test/java/me/lemire/longcompression/synth/LongUniformDataGenerator.java b/src/test/java/me/lemire/longcompression/synth/LongUniformDataGenerator.java
index 4d435f2..4aa797b 100644
--- a/src/test/java/me/lemire/longcompression/synth/LongUniformDataGenerator.java
+++ b/src/test/java/me/lemire/longcompression/synth/LongUniformDataGenerator.java
@@ -59,12 +59,12 @@ long[] generateUniformHash(int N, long Max) {
* output all longs from the range [0,Max) that are not in the array
*/
static long[] negate(long[] x, long Max) {
- int newLength = saturatedCast(Max - x.length);
- long[] ans = new long[newLength];
+ int newLength = saturatedCast(Max - x.length);
+ long[] ans = new long[newLength];
int i = 0;
int c = 0;
for (int j = 0; j < x.length; ++j) {
- long v = x[j];
+ long v = x[j];
for (; i < v; ++i)
ans[c++] = i;
++i;
@@ -74,13 +74,13 @@ static long[] negate(long[] x, long Max) {
return ans;
}
- private static int saturatedCast(long toInt) {
- if (toInt > Integer.MAX_VALUE) {
- return Integer.MAX_VALUE;
- } else {
- return (int) toInt;
- }
- }
+ private static int saturatedCast(long toInt) {
+ if (toInt > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ } else {
+ return (int) toInt;
+ }
+ }
/**
* generates randomly N distinct longs from 0 to Max.
@@ -92,7 +92,7 @@ private static int saturatedCast(long toInt) {
* @return an array containing randomly selected longs
*/
public long[] generateUniform(int N, long Max) {
- assert N >= 0;
+ assert N >= 0;
assert Max >= 0;
if (N * 2 > Max) {
return negate(generateUniform(saturatedCast(Max - N), Max), Max);