From f7e8e5eef3408f53af69755f46ef33ad370ef692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=CC=8Avard=20Ottestad?= Date: Wed, 27 Nov 2024 12:30:19 +0100 Subject: [PATCH] GH-528 experimental caching --- .../core/compact/bitmap/Bitmap375Big.java | 63 ++++++++++++++++++- .../core/compact/sequence/SequenceInt32.java | 6 ++ .../core/compact/sequence/SequenceInt64.java | 6 ++ .../core/compact/sequence/SequenceLog64.java | 6 ++ .../compact/sequence/SequenceLog64Big.java | 6 ++ .../sequence/SequenceLog64BigDisk.java | 7 +++ .../impl/MultipleLangBaseDictionary.java | 10 ++- .../core/util/disk/LargeLongArray.java | 6 ++ .../qendpoint/core/util/disk/LongArray.java | 2 + .../core/util/disk/LongArrayDisk.java | 6 ++ .../core/util/disk/SimpleLongArray.java | 6 ++ .../core/util/disk/SimpleSplitLongArray.java | 6 ++ .../core/util/disk/SyncLongArray.java | 9 +++ .../io/compress/WriteLongArrayBuffer.java | 6 ++ 14 files changed, 141 insertions(+), 4 deletions(-) diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/Bitmap375Big.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/Bitmap375Big.java index 113501c2e..8c5d0d5dc 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/Bitmap375Big.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/bitmap/Bitmap375Big.java @@ -189,8 +189,9 @@ public void updateIndex() { */ @Override public boolean access(long bitIndex) { - if (bitIndex < 0) + if (bitIndex < 0) { throw new IndexOutOfBoundsException("bitIndex < 0: " + bitIndex); + } long wordIndex = wordIndex(bitIndex); if (wordIndex >= words.length()) { @@ -324,7 +325,7 @@ public long select1(long x) { return 0; } // Search superblock (binary Search) - long superBlockIndex = binarySearch(superBlocks, x); + long superBlockIndex = binarySearchNew(superBlocks, x); // If there is a run of many zeros, two correlative superblocks may have // the same value, @@ -332,7 +333,6 @@ public long select1(long x) { while (superBlockIndex > 0 && (superBlocks.get(superBlockIndex) >= x)) { superBlockIndex--; - } long countdown = x - superBlocks.get(superBlockIndex); @@ -444,6 +444,7 @@ public static long binarySearch0(LongArray arr, long fromIndex, long toIndex, lo * @param val val * @return index */ + public static long binarySearch(LongArray arr, long val) { long min = 0, max = arr.length(), mid; @@ -460,6 +461,57 @@ public static long binarySearch(LongArray arr, long val) { return min; } + public static long binarySearchNew(LongArray arr, long val) { + long min = 0; + long max = arr.length(); + long mid; + + long[] prevFound = arr.getPrevFound(); + + int index = (int) (val / 65536 + 1); + + if (index > prevFound.length) { + throw new IllegalArgumentException("Index out of bounds: " + index); + } + + if (index + 1 < prevFound.length) { + long t = prevFound[index + 1]; + if (t > 0) { + max = Math.min(max, t); + } + } + + if (index - 1 >= 0) { + long t = prevFound[index - 1]; + if (t > 0) { + min = t; + } + } + + long t = prevFound[index]; + if (t > min && t < max) { + mid = t; + } else { + mid = (min + max) / 2; + } + + while (min + 1 < max) { + + long l = arr.get(mid); + + if (l >= val) { + max = mid; + } else { + min = mid; + } + mid = (min + max) / 2; + } + + prevFound[index] = min; + + return min; + } + public CloseSuppressPath getBlocksPath() { return blocksPath; } @@ -467,4 +519,9 @@ public CloseSuppressPath getBlocksPath() { public CloseSuppressPath getSuperBlocksPath() { return superBlocksPath; } + + @Override + public String toString() { + return "Bitmap375Big{}"; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt32.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt32.java index 4e92295c5..d5da64f5f 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt32.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt32.java @@ -40,6 +40,7 @@ public class SequenceInt32 implements DynamicSequence { /** The array that holds the objects **/ int[] data; int numelements; + private final long[] prevFound = new long[16384]; /** * Basic constructor @@ -67,6 +68,11 @@ public void clear() { Arrays.fill(data, 0); } + @Override + public long[] getPrevFound() { + return prevFound; + } + private void resizeArray(int size) { int[] newData = new int[size]; System.arraycopy(data, 0, newData, 0, Math.min(newData.length, data.length)); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt64.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt64.java index ef76c34f5..d1d084e6c 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt64.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceInt64.java @@ -39,6 +39,7 @@ public class SequenceInt64 implements DynamicSequence { /** The array that holds the objects **/ long[] data; long numelements; + private final long[] prevFound = new long[16384]; /** * Basic constructor @@ -68,6 +69,11 @@ public void clear() { Arrays.fill(data, 0); } + @Override + public long[] getPrevFound() { + return prevFound; + } + private void resizeArray(int size) { long[] newData = new long[size]; System.arraycopy(data, 0, newData, 0, Math.min(newData.length, data.length)); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64.java index ac8a3d12b..be793ba0d 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64.java @@ -49,6 +49,7 @@ public class SequenceLog64 implements DynamicSequence { protected int numbits; protected long numentries; protected long maxvalue; + private final long[] prevFound = new long[16384]; public SequenceLog64() { this(W); @@ -304,6 +305,11 @@ public void clear() { Arrays.fill(data, 0); } + @Override + public long[] getPrevFound() { + return prevFound; + } + /* * (non-Javadoc) * @see hdt.triples.array.Stream#getNumberOfElements() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64Big.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64Big.java index c328cd05b..c7ac6e084 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64Big.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64Big.java @@ -47,6 +47,7 @@ public class SequenceLog64Big implements DynamicSequence { private int numbits; private long numentries; private long maxvalue; + private final long[] prevFound = new long[16384]; public SequenceLog64Big() { this(W); @@ -282,6 +283,11 @@ public void clear() { data.clear(); } + @Override + public long[] getPrevFound() { + return prevFound; + } + /* * (non-Javadoc) * @see hdt.triples.array.Stream#getNumberOfElements() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64BigDisk.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64BigDisk.java index ab3b3018c..ed396afc1 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64BigDisk.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64BigDisk.java @@ -40,6 +40,8 @@ public class SequenceLog64BigDisk implements DynamicSequence, Closeable { private long numentries; private long maxvalue; + private final long[] prevFound = new long[16384]; + public SequenceLog64BigDisk(String location) { this(location, W); } @@ -292,6 +294,11 @@ public void clear() { data.clear(); } + @Override + public long[] getPrevFound() { + return prevFound; + } + /* * (non-Javadoc) * @see hdt.triples.array.Stream#getNumberOfElements() diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java index 25853366b..266d4cab6 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/MultipleLangBaseDictionary.java @@ -119,11 +119,19 @@ public long getNpredicates() { return predicates.getNumberOfElements(); } + long cachedNobjects = -1; + @Override public long getNobjects() { - return getNshared() + nonTyped.getNumberOfElements() + if (cachedNobjects != -1) { + return cachedNobjects; + } + + long l = getNshared() + nonTyped.getNumberOfElements() + languages.values().stream().mapToLong(DictionarySection::getNumberOfElements).sum() + typed.values().stream().mapToLong(DictionarySection::getNumberOfElements).sum(); + cachedNobjects = l; + return l; } @Override diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LargeLongArray.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LargeLongArray.java index f0b0e874b..9a9e90642 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LargeLongArray.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LargeLongArray.java @@ -12,6 +12,7 @@ */ public class LargeLongArray implements LongArray { private UnsafeLongArray array; + private final long[] prevFound = new long[16384]; /** * @param array large array @@ -55,4 +56,9 @@ public void resize(long newSize) throws IOException { public void clear() { array.clear(); } + + @Override + public long[] getPrevFound() { + return prevFound; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArray.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArray.java index 77b0add59..980484889 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArray.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArray.java @@ -208,4 +208,6 @@ public Long next() { } }; } + + long[] getPrevFound(); } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArrayDisk.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArrayDisk.java index 1b10eaed3..7bd9344ef 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArrayDisk.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/LongArrayDisk.java @@ -41,6 +41,7 @@ public class LongArrayDisk implements Closeable, LongArray { private long size; private final long startByte; private final Path location; + private final long[] prevFound = new long[16384]; public LongArrayDisk(String location, long size) { this(location, size, true); @@ -289,6 +290,11 @@ public void clear() { set0(0, length()); } + @Override + public long[] getPrevFound() { + return prevFound; + } + /** * @return the location of the array disk */ diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleLongArray.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleLongArray.java index b818cc9a1..5430a799c 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleLongArray.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleLongArray.java @@ -4,6 +4,7 @@ import java.util.Arrays; public class SimpleLongArray implements LongArray { + private final long[] prevFound = new long[16384]; public static LongArray of(int size) { return wrapper(new long[size]); @@ -60,4 +61,9 @@ public void resize(long newSize) throws IOException { public void clear() { Arrays.fill(array, 0); } + + @Override + public long[] getPrevFound() { + return prevFound; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleSplitLongArray.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleSplitLongArray.java index b9e2ab8fa..e27d2a629 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleSplitLongArray.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SimpleSplitLongArray.java @@ -16,6 +16,7 @@ public class SimpleSplitLongArray implements LongArray, Closeable { private final long max; private final int indexMask; private final int numbits; + private final long[] prevFound = new long[16384]; private long size; @@ -113,6 +114,11 @@ public void clear() { array.clear(); } + @Override + public long[] getPrevFound() { + return prevFound; + } + @Override public void close() throws IOException { IOUtil.closeObject(array); diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SyncLongArray.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SyncLongArray.java index 9d2ccf536..40e40baac 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SyncLongArray.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/disk/SyncLongArray.java @@ -8,6 +8,9 @@ * @author Antoine Willerval */ public class SyncLongArray implements LongArray { + + private final long[] prevFound = new long[16384]; + /** * Sync a long array * @@ -57,4 +60,10 @@ public synchronized void resize(long newSize) throws IOException { public synchronized void clear() { array.clear(); } + + @Override + public long[] getPrevFound() { + return prevFound; + + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/WriteLongArrayBuffer.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/WriteLongArrayBuffer.java index 6502676f3..325673070 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/WriteLongArrayBuffer.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/util/io/compress/WriteLongArrayBuffer.java @@ -21,6 +21,7 @@ public class WriteLongArrayBuffer implements LongArray, Closeable { private ArrayElementInt[] bufferInt; private int index = 0; private boolean lastOrder; + private final long[] prevFound = new long[16384]; /** * create the buffer @@ -54,6 +55,11 @@ public void clear() { index = 0; } + @Override + public long[] getPrevFound() { + return prevFound; + } + public void free() { flush(); bufferInt = null;