diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 558db9d2..97e81dba 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,10 +4,12 @@ on: branches: - main - opendistro-* + - 7.10.2-no-workbench pull_request: branches: - main - opendistro-* + - 7.10.2-no-workbench jobs: Build-ad: diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java index 97a20e90..e0563e1f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java @@ -22,7 +22,6 @@ import java.time.Clock; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -51,12 +50,10 @@ import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.monitor.jvm.JvmService; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -189,7 +186,7 @@ /** * Entry point of AD plugin. */ -public class AnomalyDetectorPlugin extends Plugin implements ActionPlugin, ScriptPlugin, JobSchedulerExtension, SystemIndexPlugin { +public class AnomalyDetectorPlugin extends Plugin implements ActionPlugin, ScriptPlugin, JobSchedulerExtension { private static final Logger LOG = LogManager.getLogger(AnomalyDetectorPlugin.class); @@ -726,19 +723,4 @@ public ScheduledJobParser getJobParser() { return AnomalyDetectorJob.parse(parser); }; } - - @Override - public Collection getSystemIndexDescriptors(Settings settings) { - return Collections - .unmodifiableList( - Arrays - .asList( - new SystemIndexDescriptor(AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN, "anomaly result"), - new SystemIndexDescriptor(AnomalyDetector.ANOMALY_DETECTORS_INDEX, "detector definition"), - new SystemIndexDescriptor(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, "detector job"), - new SystemIndexDescriptor(CommonName.CHECKPOINT_INDEX_NAME, "model checkpoint"), - new SystemIndexDescriptor(CommonName.DETECTION_STATE_INDEX, "detector information like total rcf updates") - ) - ); - } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/indices/AnomalyDetectionIndices.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/indices/AnomalyDetectionIndices.java index 4ef32149..9bf641ac 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/indices/AnomalyDetectionIndices.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/indices/AnomalyDetectionIndices.java @@ -113,6 +113,8 @@ public class AnomalyDetectionIndices implements LocalNodeMasterListener { private boolean allUpdated; // we only want one update at a time private final AtomicBoolean updateRunning; + // AD index settings + private final Settings setting; class IndexState { // keep track of whether the mapping version is up-to-date @@ -169,6 +171,8 @@ public AnomalyDetectionIndices( .addSettingsUpdateConsumer(AD_RESULT_HISTORY_RETENTION_PERIOD, it -> { historyRetentionPeriod = it; }); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_PRIMARY_SHARDS, it -> maxPrimaryShards = it); + + this.setting = Settings.builder().put("index.hidden", true).build(); } /** @@ -328,7 +332,8 @@ public void initAnomalyDetectorIndexIfAbsent(ActionListener */ public void initAnomalyDetectorIndex(ActionListener actionListener) throws IOException { CreateIndexRequest request = new CreateIndexRequest(AnomalyDetector.ANOMALY_DETECTORS_INDEX) - .mapping(AnomalyDetector.TYPE, getAnomalyDetectorMappings(), XContentType.JSON); + .mapping(AnomalyDetector.TYPE, getAnomalyDetectorMappings(), XContentType.JSON) + .settings(setting); adminClient.indices().create(request, markMappingUpToDate(ADIndex.CONFIG, actionListener)); } @@ -357,6 +362,7 @@ private void choosePrimaryShards(CreateIndexRequest request) { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, Math.min(nodeFilter.getNumberOfEligibleDataNodes(), maxPrimaryShards)) // 1 replica for better search performance and fail-over .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.hidden", true) ); } @@ -400,7 +406,8 @@ public void initAnomalyDetectorJobIndex(ActionListener acti public void initDetectionStateIndex(ActionListener actionListener) { try { CreateIndexRequest request = new CreateIndexRequest(CommonName.DETECTION_STATE_INDEX) - .mapping(AnomalyDetector.TYPE, getDetectionStateMappings(), XContentType.JSON); + .mapping(AnomalyDetector.TYPE, getDetectionStateMappings(), XContentType.JSON) + .settings(setting); adminClient.indices().create(request, markMappingUpToDate(ADIndex.STATE, actionListener)); } catch (IOException e) { logger.error("Fail to init AD detection state index", e); @@ -469,7 +476,7 @@ void rolloverAndDeleteHistoryIndex() { } // We have to pass null for newIndexName in order to get Elastic to increment the index count. - RolloverRequest request = new RolloverRequest(CommonName.ANOMALY_RESULT_INDEX_ALIAS, null); + RolloverRequest rollOverRequest = new RolloverRequest(CommonName.ANOMALY_RESULT_INDEX_ALIAS, null); String adResultMapping = null; try { adResultMapping = getAnomalyResultMappings(); @@ -477,12 +484,14 @@ void rolloverAndDeleteHistoryIndex() { logger.error("Fail to roll over AD result index, as can't get AD result index mapping"); return; } - request - .getCreateIndexRequest() - .index(AD_RESULT_HISTORY_INDEX_PATTERN) - .mapping(CommonName.MAPPING_TYPE, adResultMapping, XContentType.JSON); - request.addMaxIndexDocsCondition(historyMaxDocs); - adminClient.indices().rolloverIndex(request, ActionListener.wrap(response -> { + CreateIndexRequest createRequest = rollOverRequest.getCreateIndexRequest(); + + createRequest.index(AD_RESULT_HISTORY_INDEX_PATTERN).mapping(CommonName.MAPPING_TYPE, adResultMapping, XContentType.JSON); + + choosePrimaryShards(createRequest); + + rollOverRequest.addMaxIndexDocsCondition(historyMaxDocs); + adminClient.indices().rolloverIndex(rollOverRequest, ActionListener.wrap(response -> { if (!response.isRolledOver()) { logger .warn("{} not rolled over. Conditions were: {}", CommonName.ANOMALY_RESULT_INDEX_ALIAS, response.getConditionStatus());