From 645e5604035ecc7d48e4312622704e44bbe0614d Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 11 Dec 2023 19:08:16 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BC=98=E5=8C=96segment=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E6=A3=80=E6=B5=8B=EF=BC=8C=E5=8C=85=E5=90=AB?= =?UTF-8?q?version=E6=96=87=E4=BB=B6=E7=9A=84=E7=9B=AE=E5=BD=95=E6=97=A0?= =?UTF-8?q?=E9=9C=80=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/havenask/engine/MetaDataSyncer.java | 3 +- .../generator/RuntimeSegmentGenerator.java | 25 ++++++---- .../RuntimeSegmentGeneratorTests.java | 48 ++++++++++++++++--- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/MetaDataSyncer.java b/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/MetaDataSyncer.java index 3f12cea7..e6ee61d3 100644 --- a/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/MetaDataSyncer.java +++ b/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/MetaDataSyncer.java @@ -39,7 +39,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.havenask.cluster.ClusterChangedEvent; @@ -70,6 +69,8 @@ import org.havenask.index.Index; import org.havenask.threadpool.ThreadPool; +import com.carrotsearch.hppc.cursors.ObjectCursor; + public class MetaDataSyncer extends AbstractLifecycleComponent implements ClusterStateApplier { private static final Logger LOGGER = LogManager.getLogger(MetaDataSyncer.class); private static final int MAX_SYNC_TIMES = 30; diff --git a/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/index/config/generator/RuntimeSegmentGenerator.java b/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/index/config/generator/RuntimeSegmentGenerator.java index 4048b0a0..888e24de 100644 --- a/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/index/config/generator/RuntimeSegmentGenerator.java +++ b/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/index/config/generator/RuntimeSegmentGenerator.java @@ -14,13 +14,6 @@ package org.havenask.engine.index.config.generator; -import org.havenask.common.Nullable; -import org.havenask.common.settings.Settings; -import org.havenask.engine.index.config.Schema; -import org.havenask.engine.util.RangeUtil; -import org.havenask.index.mapper.MapperService; -import org.havenask.index.shard.ShardId; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -28,7 +21,19 @@ import java.nio.file.StandardOpenOption; import java.util.Locale; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.havenask.common.Nullable; +import org.havenask.common.settings.Settings; +import org.havenask.engine.index.config.Schema; +import org.havenask.engine.util.RangeUtil; +import org.havenask.engine.util.Utils; +import org.havenask.index.mapper.MapperService; +import org.havenask.index.shard.ShardId; + public class RuntimeSegmentGenerator { + private static final Logger LOGGER = LogManager.getLogger(RuntimeSegmentGenerator.class); + public static final String VERSION_FILE_NAME = "version.0"; public static final String VERSION_FILE_CONTENT = "{\n" + "\"description\":\n" @@ -198,10 +203,14 @@ public void generate() throws IOException { Files.createDirectories(dataPath); } - if (Files.exists(dataPath.resolve(VERSION_FILE_NAME))) { + String versionFile = Utils.getIndexMaxVersion(dataPath); + + if (versionFile != null && false == versionFile.isEmpty()) { return; } + LOGGER.info("generate runtime segment for index [{}], partition [{}]", indexName, partitionName); + Files.write( dataPath.resolve(VERSION_FILE_NAME), VERSION_FILE_CONTENT.getBytes(StandardCharsets.UTF_8), diff --git a/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java b/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java index fdf9bc74..00fd4a89 100644 --- a/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java +++ b/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java @@ -14,20 +14,21 @@ package org.havenask.engine.index.config.generator; -import org.havenask.common.settings.Settings; -import org.havenask.engine.HavenaskEnginePlugin; -import org.havenask.index.mapper.MapperService; -import org.havenask.index.mapper.MapperServiceTestCase; -import org.havenask.index.shard.ShardId; -import org.havenask.plugins.Plugin; +import static java.util.Collections.singletonList; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.Locale; -import static java.util.Collections.singletonList; +import org.havenask.common.settings.Settings; +import org.havenask.engine.HavenaskEnginePlugin; +import org.havenask.index.mapper.MapperService; +import org.havenask.index.mapper.MapperServiceTestCase; +import org.havenask.index.shard.ShardId; +import org.havenask.plugins.Plugin; public class RuntimeSegmentGeneratorTests extends MapperServiceTestCase { @Override @@ -146,4 +147,37 @@ public void testBasic() throws IOException { ); } + public void testExists() throws IOException { + String indexName = randomAlphaOfLength(5); + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "keyword"))); + Path runtimePath = createTempDir(); + ShardId shardId = new ShardId(indexName, "_na_", 0); + RuntimeSegmentGenerator runtimeSegmentGenerator = new RuntimeSegmentGenerator( + shardId, + 1, + Settings.EMPTY, + mapperService, + runtimePath + ); + + Path dataPath = runtimePath.resolve(indexName).resolve("generation_0").resolve("partition_0_65535"); + Files.createDirectories(dataPath); + + Files.write( + dataPath.resolve("version.1"), + RuntimeSegmentGenerator.VERSION_FILE_CONTENT.getBytes(), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + runtimeSegmentGenerator.generate(); + + assertFalse(Files.exists(dataPath.resolve(RuntimeSegmentGenerator.VERSION_FILE_NAME))); + assertFalse(Files.exists(dataPath.resolve(RuntimeSegmentGenerator.INDEX_FORMAT_VERSION_FILE_NAME))); + assertFalse(Files.exists(dataPath.resolve(RuntimeSegmentGenerator.INDEX_PARTITION_META_FILE_NAME))); + assertFalse(Files.exists(dataPath.resolve(RuntimeSegmentGenerator.SCHEMA_FILE_NAME))); + assertFalse(Files.exists(dataPath.resolve(RuntimeSegmentGenerator.DEPLOY_META_FILE_NAME))); + assertFalse(Files.exists(dataPath.resolve(RuntimeSegmentGenerator.ENTRY_TABLE_FILE_NAME))); + } + } From a301a46b61b954a9d469ab5725013b81411b1cc1 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 11 Dec 2023 19:11:25 +0800 Subject: [PATCH 2/2] fixup --- .../index/config/generator/RuntimeSegmentGeneratorTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java b/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java index 00fd4a89..63131c57 100644 --- a/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java +++ b/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/index/config/generator/RuntimeSegmentGeneratorTests.java @@ -17,6 +17,7 @@ import static java.util.Collections.singletonList; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -165,7 +166,7 @@ public void testExists() throws IOException { Files.write( dataPath.resolve("version.1"), - RuntimeSegmentGenerator.VERSION_FILE_CONTENT.getBytes(), + RuntimeSegmentGenerator.VERSION_FILE_CONTENT.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING );