[fix](index compaction)Skip writing terms with a doc frequency of 0 (#…

…248)
apache · Oct 31, 2024 · 7cf6cf4 · 7cf6cf4
1 parent 9882657
commit 7cf6cf4
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/src/core/CLucene/index/IndexWriter.cpp b/src/core/CLucene/index/IndexWriter.cpp
@@ -1831,6 +1831,17 @@ void IndexWriter::mergeTerms(bool hasProx, IndexVersion indexVersion) {
         }
 
         for (int i = 0; i < numDestIndexes; ++i) {
+            if (dfs[i] == 0) {
+                if (infoStream != nullptr) {
+                    std::string name = lucene_wcstoutf8string(smallestTerm->text(), smallestTerm->textLength());
+                    std::string field = lucene_wcstoutf8string(smallestTerm->field(), wcslen(smallestTerm->field()));
+                    std::stringstream ss;
+                    ss << "term: " << name << ", field: " << field << ", doc frequency is zero[" << dfs[i] << "], skip it." << "\n";
+                    message(ss.str());
+                }
+                // if doc frequency is 0, it means the term is deleted. So we should not write it.
+                continue;
+            }
             DefaultSkipListWriter *skipListWriter = skipListWriterList[i];
             CL_NS(store)::IndexOutput *freqOutput = freqOutputList[i];
             CL_NS(store)::IndexOutput *proxOutput = proxOutputList[i];