Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce IOContext.LOAD #11930

Merged
merged 12 commits into from
Nov 15, 2022
Merged

Introduce IOContext.LOAD #11930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
efb6297
Automatically preload index files that are both tiny and very hot.
jpountz Nov 10, 2022
1386a16
Make preload a predicate.
jpountz Nov 14, 2022
f6ebdf4
Restore getter and add more docs.
jpountz Nov 14, 2022
ed5eb24
Remove `PRELOAD_` prefix.
jpountz Nov 14, 2022
6a5ef89
Rename.
jpountz Nov 14, 2022
abf7f6f
Remove automatic preloading.
jpountz Nov 14, 2022
408aae5
Remove MMapDirectory changes.
jpountz Nov 14, 2022
76c8dd7
Merge branch 'main' into add_iocontext_load
jpountz Nov 14, 2022
8244cac
iter
jpountz Nov 14, 2022
7952bea
Improve docs.
jpountz Nov 14, 2022
711116b
Merge remote-tracking branch 'origin/main' into add_iocontext_load
jpountz Nov 15, 2022
5847693
Add back BASED_ON_LOAD_IO_CONTEXT.
jpountz Nov 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -82,7 +83,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
merging = false;

// read in the entries from the metadata file.
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, IOContext.READONCE)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering about this earlier, but it is really nice you fixed this,too!

Throwable priorE = null;

try {
Expand Down
3 changes: 2 additions & 1 deletion lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90NormsProducer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -60,7 +61,7 @@ final class Lucene90NormsProducer extends NormsProducer implements Cloneable {
int version = -1;

// read in the entries from the metadata file.
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, IOContext.READONCE)) {
Throwable priorE = null;
try {
version =
Expand Down
5 changes: 3 additions & 2 deletions lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.bkd.BKDReader;
Expand Down Expand Up @@ -59,7 +60,7 @@ public Lucene90PointsReader(SegmentReadState readState) throws IOException {

boolean success = false;
try {
indexIn = readState.directory.openInput(indexFileName, readState.context);
indexIn = readState.directory.openInput(indexFileName, IOContext.LOAD);
CodecUtil.checkIndexHeader(
indexIn,
Lucene90PointsFormat.INDEX_CODEC_NAME,
Expand All @@ -81,7 +82,7 @@ public Lucene90PointsReader(SegmentReadState readState) throws IOException {

long indexLength = -1, dataLength = -1;
try (ChecksumIndexInput metaIn =
readState.directory.openChecksumInput(metaFileName, readState.context)) {
readState.directory.openChecksumInput(metaFileName, IOContext.READONCE)) {
Throwable priorE = null;
try {
CodecUtil.checkIndexHeader(
Expand Down
6 changes: 4 additions & 2 deletions ...re/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -134,7 +135,7 @@ public Lucene90BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentRe

String indexName =
IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexIn = state.directory.openInput(indexName, state.context);
indexIn = state.directory.openInput(indexName, IOContext.LOAD);
CodecUtil.checkIndexHeader(
indexIn,
TERMS_INDEX_CODEC_NAME,
Expand All @@ -149,7 +150,8 @@ public Lucene90BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentRe
Map<String, FieldReader> fieldMap = null;
Throwable priorE = null;
long indexLength = -1, termsLength = -1;
try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaName, state.context)) {
try (ChecksumIndexInput metaIn =
state.directory.openChecksumInput(metaName, IOContext.READONCE)) {
try {
CodecUtil.checkIndexHeader(
metaIn,
Expand Down
2 changes: 1 addition & 1 deletion lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/FieldsIndexReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ final class FieldsIndexReader extends FieldsIndex {
maxPointer = metaIn.readLong();

indexInput =
dir.openInput(IndexFileNames.segmentFileName(name, suffix, extension), IOContext.READ);
dir.openInput(IndexFileNames.segmentFileName(name, suffix, extension), IOContext.LOAD);
boolean success = false;
try {
CodecUtil.checkIndexHeader(
Expand Down
4 changes: 3 additions & 1 deletion lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -89,7 +90,8 @@ private int readMetadata(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene94HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName, state.context)) {
try (ChecksumIndexInput meta =
state.directory.openChecksumInput(metaFileName, IOContext.READONCE)) {
Throwable priorE = null;
try {
versionMeta =
Expand Down
23 changes: 19 additions & 4 deletions lucene/core/src/java/org/apache/lucene/store/IOContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,34 +41,47 @@ public enum Context {

public final FlushInfo flushInfo;

/** This flag indicates that the file will be opened, then fully read sequentially then closed. */
public final boolean readOnce;

/**
* This flag is used for files that are a small fraction of the total index size and are expected
* to be heavily accessed in random-access fashion. Some {@link Directory} implementations may
* choose to load such files into physical memory (e.g. Java heap) as a way to provide stronger
* guarantees on query latency.
*/
public final boolean load;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This description looks good.


public static final IOContext DEFAULT = new IOContext(Context.DEFAULT);

public static final IOContext READONCE = new IOContext(true);
public static final IOContext READONCE = new IOContext(true, false);

public static final IOContext READ = new IOContext(false, false);

public static final IOContext READ = new IOContext(false);
public static final IOContext LOAD = new IOContext(false, true);

public IOContext() {
this(false);
this(false, false);
}

public IOContext(FlushInfo flushInfo) {
assert flushInfo != null;
this.context = Context.FLUSH;
this.mergeInfo = null;
this.readOnce = false;
this.load = false;
this.flushInfo = flushInfo;
}

public IOContext(Context context) {
this(context, null);
}

private IOContext(boolean readOnce) {
private IOContext(boolean readOnce, boolean load) {
this.context = Context.READ;
this.mergeInfo = null;
this.readOnce = readOnce;
this.load = load;
this.flushInfo = null;
}

Expand All @@ -82,6 +95,7 @@ private IOContext(Context context, MergeInfo mergeInfo) {
assert context != Context.FLUSH : "Use IOContext(FlushInfo) to create a FLUSH IOContext";
this.context = context;
this.readOnce = false;
this.load = false;
this.mergeInfo = mergeInfo;
this.flushInfo = null;
}
Expand All @@ -99,6 +113,7 @@ public IOContext(IOContext ctxt, boolean readOnce) {
this.mergeInfo = ctxt.mergeInfo;
this.flushInfo = ctxt.flushInfo;
this.readOnce = readOnce;
this.load = false;
}

@Override
Expand Down
7 changes: 7 additions & 0 deletions lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,13 @@ public class MMapDirectory extends FSDirectory {
*/
public static final BiPredicate<String, IOContext> NO_FILES = (filename, context) -> false;

/**
* Argument for {@link #setPreload(BiPredicate)} that configures files to be preloaded upon
* opening them if they use the {@link IOContext#LOAD} I/O context.
*/
public static final BiPredicate<String, IOContext> BASED_ON_LOAD_IO_CONTEXT =
(filename, context) -> context.load;

private boolean useUnmapHack = UNMAP_SUPPORTED;
private BiPredicate<String, IOContext> preload = NO_FILES;

Expand Down