the-qa-companyGH-528 experimental caching

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/Bitmap375Big.java

@@ -189,8 +189,9 @@ public void updateIndex() {
 	 */
 	@Override
 	public boolean access(long bitIndex) {
-		if (bitIndex < 0)
+		if (bitIndex < 0) {
 			throw new IndexOutOfBoundsException("bitIndex < 0: " + bitIndex);
+		}
 
 		long wordIndex = wordIndex(bitIndex);
 		if (wordIndex >= words.length()) {
@@ -324,34 +325,25 @@ public long select1(long x) {
 			return 0;
 		}
 		// Search superblock (binary Search)
-		long superBlockIndex = binarySearch(superBlocks, x);
+		long superBlockIndex = binarySearchNew(superBlocks, x);
 
 		// If there is a run of many zeros, two correlative superblocks may have
 		// the same value,
 		// We need to position at the first of them.
 
-		while (superBlockIndex > 0 && (superBlocks.get(superBlockIndex) >= x)) {
-			superBlockIndex--;
+//		long oldSuperBlockIndex = superBlockIndex;
+		superBlockIndex = temp1(x, superBlockIndex);
 
-		}
+//		if(oldSuperBlockIndex != superBlockIndex){
+//			long binarySearch = binarySearch(superBlocks, x);
+//			long binarySearchNew = binarySearchNew(superBlocks, x);
+//			System.out.println();
+//		}
 
 		long countdown = x - superBlocks.get(superBlockIndex);
 
-		long blockIdx = superBlockIndex * BLOCKS_PER_SUPER;
-
 		// Search block
-		while (true) {
-			if (blockIdx >= (superBlockIndex + 1) * BLOCKS_PER_SUPER || blockIdx >= blocks.length()) {
-				blockIdx--;
-				break;
-			}
-			if ((0xFF & blocks.get(blockIdx)) >= countdown) {
-				// We found it!
-				blockIdx--;
-				break;
-			}
-			blockIdx++;
-		}
+		long blockIdx = searchBlock(superBlockIndex * BLOCKS_PER_SUPER, superBlockIndex, countdown);
 		if (blockIdx < 0) {
 			blockIdx = 0;
 		}
@@ -363,6 +355,25 @@ public long select1(long x) {
 		return blockIdx * W + bitpos - 1;
 	}
 
+	private long temp1(long x, long superBlockIndex) {
+		while (superBlockIndex > 0 && (superBlocks.get(superBlockIndex) >= x)) {
+			superBlockIndex--;
+		}
+		return superBlockIndex;
+	}
+
+	private long searchBlock(long blockIdx, long superBlockIndex, long countdown) {
+		long limit = (superBlockIndex + 1) * BLOCKS_PER_SUPER;
+		long blocksLength = blocks.length();
+		while (blockIdx < limit && blockIdx < blocksLength) {
+			if ((blocks.get(blockIdx) & 0xFF) >= countdown) {
+				return blockIdx - 1;
+			}
+			blockIdx++;
+		}
+		return blockIdx - 1;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see hdt.compact.bitmap.Bitmap#countOnes()
@@ -444,6 +455,7 @@ public static long binarySearch0(LongArray arr, long fromIndex, long toIndex, lo
 	 * @param val val
 	 * @return index
 	 */
+
 	public static long binarySearch(LongArray arr, long val) {
 		long min = 0, max = arr.length(), mid;
 
@@ -460,11 +472,67 @@ public static long binarySearch(LongArray arr, long val) {
 		return min;
 	}
 
+	public static long binarySearchNew(LongArray arr, long val) {
+		long min = 0;
+		long max = arr.length();
+		long mid;
+
+		long[] prevFound = arr.getPrevFound();
+
+		int index = (int) (val / 65536 + 1);
+
+		if (index > prevFound.length) {
+			throw new IllegalArgumentException("Index out of bounds: " + index);
+		}
+
+		if (index + 1 < prevFound.length) {
+			long t = prevFound[index + 1];
+			if (t > 0) {
+				max = Math.min(max, t);
+			}
+		}
+
+		if (index - 1 >= 0) {
+			long t = prevFound[index - 1];
+			if (t > 0) {
+				min = t;
+			}
+		}
+
+		long t = prevFound[index];
+		if (t > min && t < max) {
+			mid = t;
+		} else {
+			mid = (min + max) / 2;
+		}
+
+		while (min + 1 < max) {
+
+			long l = arr.get(mid);
+
+			if (l >= val) {
+				max = mid;
+			} else {
+				min = mid;
+			}
+			mid = (min + max) / 2;
+		}
+
+		prevFound[index] = min;
+
+		return min;
+	}
+
 	public CloseSuppressPath getBlocksPath() {
 		return blocksPath;
 	}
 
 	public CloseSuppressPath getSuperBlocksPath() {
 		return superBlocksPath;
 	}
+
+	@Override
+	public String toString() {
+		return "Bitmap375Big{}";
+	}
 }

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt32.java

@@ -40,6 +40,7 @@ public class SequenceInt32 implements DynamicSequence {
 	/** The array that holds the objects **/
 	int[] data;
 	int numelements;
+	private final long[] prevFound = new long[16384];
 
 	/**
 	 * Basic constructor
@@ -67,6 +68,11 @@ public void clear() {
 		Arrays.fill(data, 0);
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	private void resizeArray(int size) {
 		int[] newData = new int[size];
 		System.arraycopy(data, 0, newData, 0, Math.min(newData.length, data.length));

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt64.java

@@ -39,6 +39,7 @@ public class SequenceInt64 implements DynamicSequence {
 	/** The array that holds the objects **/
 	long[] data;
 	long numelements;
+	private final long[] prevFound = new long[16384];
 
 	/**
 	 * Basic constructor
@@ -68,6 +69,11 @@ public void clear() {
 		Arrays.fill(data, 0);
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	private void resizeArray(int size) {
 		long[] newData = new long[size];
 		System.arraycopy(data, 0, newData, 0, Math.min(newData.length, data.length));

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64.java

@@ -49,6 +49,7 @@ public class SequenceLog64 implements DynamicSequence {
 	protected int numbits;
 	protected long numentries;
 	protected long maxvalue;
+	private final long[] prevFound = new long[16384];
 
 	public SequenceLog64() {
 		this(W);
@@ -304,6 +305,11 @@ public void clear() {
 		Arrays.fill(data, 0);
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see hdt.triples.array.Stream#getNumberOfElements()

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64Big.java

@@ -47,6 +47,7 @@ public class SequenceLog64Big implements DynamicSequence {
 	private int numbits;
 	private long numentries;
 	private long maxvalue;
+	private final long[] prevFound = new long[16384];
 
 	public SequenceLog64Big() {
 		this(W);
@@ -282,6 +283,11 @@ public void clear() {
 		data.clear();
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see hdt.triples.array.Stream#getNumberOfElements()

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64BigDisk.java

@@ -40,6 +40,8 @@ public class SequenceLog64BigDisk implements DynamicSequence, Closeable {
 	private long numentries;
 	private long maxvalue;
 
+	private final long[] prevFound = new long[16384];
+
 	public SequenceLog64BigDisk(String location) {
 		this(location, W);
 	}
@@ -292,6 +294,11 @@ public void clear() {
 		data.clear();
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see hdt.triples.array.Stream#getNumberOfElements()

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java

@@ -119,11 +119,19 @@ public long getNpredicates() {
 		return predicates.getNumberOfElements();
 	}
 
+	long cachedNobjects = -1;
+
 	@Override
 	public long getNobjects() {
-		return getNshared() + nonTyped.getNumberOfElements()
+		if (cachedNobjects != -1) {
+			return cachedNobjects;
+		}
+
+		long l = getNshared() + nonTyped.getNumberOfElements()
 				+ languages.values().stream().mapToLong(DictionarySection::getNumberOfElements).sum()
 				+ typed.values().stream().mapToLong(DictionarySection::getNumberOfElements).sum();
+		cachedNobjects = l;
+		return l;
 	}
 
 	@Override

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LargeLongArray.java

@@ -12,6 +12,7 @@
  */
 public class LargeLongArray implements LongArray {
 	private UnsafeLongArray array;
+	private final long[] prevFound = new long[16384];
 
 	/**
 	 * @param array large array
@@ -55,4 +56,9 @@ public void resize(long newSize) throws IOException {
 	public void clear() {
 		array.clear();
 	}
+
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
 }

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArray.java

@@ -208,4 +208,6 @@ public Long next() {
 			}
 		};
 	}
+
+	long[] getPrevFound();
 }

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArrayDisk.java

@@ -41,6 +41,7 @@ public class LongArrayDisk implements Closeable, LongArray {
 	private long size;
 	private final long startByte;
 	private final Path location;
+	private final long[] prevFound = new long[16384];
 
 	public LongArrayDisk(String location, long size) {
 		this(location, size, true);
@@ -289,6 +290,11 @@ public void clear() {
 		set0(0, length());
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	/**
 	 * @return the location of the array disk
 	 */

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleLongArray.java

@@ -4,6 +4,7 @@
 import java.util.Arrays;
 
 public class SimpleLongArray implements LongArray {
+	private final long[] prevFound = new long[16384];
 
 	public static LongArray of(int size) {
 		return wrapper(new long[size]);
@@ -60,4 +61,9 @@ public void resize(long newSize) throws IOException {
 	public void clear() {
 		Arrays.fill(array, 0);
 	}
+
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
 }

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleSplitLongArray.java

@@ -16,6 +16,7 @@ public class SimpleSplitLongArray implements LongArray, Closeable {
 	private final long max;
 	private final int indexMask;
 	private final int numbits;
+	private final long[] prevFound = new long[16384];
 
 	private long size;
 
@@ -113,6 +114,11 @@ public void clear() {
 		array.clear();
 	}
 
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+	}
+
 	@Override
 	public void close() throws IOException {
 		IOUtil.closeObject(array);

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SyncLongArray.java

@@ -8,6 +8,9 @@
  * @author Antoine Willerval
  */
 public class SyncLongArray implements LongArray {
+
+	private final long[] prevFound = new long[16384];
+
 	/**
 	 * Sync a long array
 	 *
@@ -57,4 +60,10 @@ public synchronized void resize(long newSize) throws IOException {
 	public synchronized void clear() {
 		array.clear();
 	}
+
+	@Override
+	public long[] getPrevFound() {
+		return prevFound;
+
+	}
 }