From 9db6f095e8e1f04220a10dc3973e6d72608c44b0 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 10 Jan 2019 09:53:36 -0600 Subject: [PATCH] ML: adjusting for backport of #36643 --- .../xpack/core/ml/action/MlUpgradeAction.java | 19 +++++++++-- .../xpack/ml/integration/MlUpgradeIT.java | 11 ++++--- .../xpack/ml/ResultsIndexUpgradeService.java | 2 +- .../ml/action/TransportMlUpgradeAction.java | 7 ++-- .../ml/rest/results/RestUpgradeMlAction.java | 14 ++------ ...{ml.upgrade.json => xpack.ml.upgrade.json} | 6 ++-- .../rest-api-spec/test/ml/ml_upgrade.yml | 6 ++-- x-pack/qa/rolling-upgrade/build.gradle | 11 ++++++- .../mixed_cluster/80_ml_results_upgrade.yml | 4 +-- .../old_cluster/80_ml_results_upgrade.yml | 20 ++++++------ .../80_ml_results_upgrade.yml | 32 +++++++++---------- 11 files changed, 73 insertions(+), 59 deletions(-) rename x-pack/plugin/src/test/resources/rest-api-spec/api/{ml.upgrade.json => xpack.ml.upgrade.json} (79%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlUpgradeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlUpgradeAction.java index 404f15d4f6270..8893080083b97 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlUpgradeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlUpgradeAction.java @@ -28,7 +28,8 @@ import java.util.Objects; -public class MlUpgradeAction extends Action { +public class MlUpgradeAction extends Action { + public static final MlUpgradeAction INSTANCE = new MlUpgradeAction(); public static final String NAME = "cluster:admin/xpack/ml/upgrade"; @@ -41,6 +42,11 @@ public AcknowledgedResponse newResponse() { return new AcknowledgedResponse(); } + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + public static class Request extends MasterNodeReadRequest implements ToXContentObject { private static final ParseField REINDEX_BATCH_SIZE = new ParseField("reindex_batch_size"); @@ -60,6 +66,7 @@ public static class Request extends MasterNodeReadRequest implements To // for serialization public Request() { + super(); } public Request(StreamInput in) throws IOException { @@ -73,6 +80,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(reindexBatchSize); } + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + reindexBatchSize = in.readInt(); + } + public String[] indices() { return new String[]{INDEX}; } @@ -152,8 +165,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { - public RequestBuilder(ElasticsearchClient client) { - super(client, INSTANCE, new Request()); + public RequestBuilder(ElasticsearchClient client, MlUpgradeAction action) { + super(client, action, new Request()); } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlUpgradeIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlUpgradeIT.java index a2a05ea1686fa..20c602a5a7e01 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlUpgradeIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlUpgradeIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.ReindexAction; @@ -88,7 +89,7 @@ public void testMigrationWhenItIsNotNecessary() throws Exception { assertThat(getTotalDocCount(job3Index), equalTo(job3Total)); ClusterState state = admin().cluster().state(new ClusterStateRequest()).actionGet().getState(); - IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); String[] indices = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.strictExpandOpenAndForbidClosed(), AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"); @@ -120,7 +121,7 @@ public void testMigration() throws Exception { long job2Total = getJobResultsCount(job2.getId()); long job3Total = getJobResultsCount(job3.getId()); - IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); ResultsIndexUpgradeService resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver, ThreadPool.Names.SAME, @@ -203,7 +204,7 @@ public void testMigrationWithManuallyCreatedIndexThatNeedsMigrating() throws Exc String manuallyCreatedIndex = job1Index + "-" + Version.CURRENT.major; client().admin().indices().prepareCreate(manuallyCreatedIndex).execute().actionGet(); - IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); ResultsIndexUpgradeService resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver, ThreadPool.Names.SAME, @@ -257,7 +258,7 @@ public void testMigrationWithExistingIndexWithData() throws Exception { String alreadyMigratedWriteIndex = job1Index + "-" + Version.CURRENT.major; client().admin().indices().prepareCreate(alreadyMigratedWriteIndex).execute().actionGet(); - IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); ResultsIndexUpgradeService resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver, ThreadPool.Names.SAME, @@ -315,7 +316,7 @@ private long getTotalDocCount(String indexName) { .setTrackTotalHits(true) .setQuery(QueryBuilders.matchAllQuery()) .execute().actionGet(); - return searchResponse.getHits().getTotalHits().value; + return searchResponse.getHits().getTotalHits(); } private long getJobResultsCount(String jobId) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/ResultsIndexUpgradeService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/ResultsIndexUpgradeService.java index ccbaed13feca0..df14bdab78a78 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/ResultsIndexUpgradeService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/ResultsIndexUpgradeService.java @@ -63,7 +63,7 @@ public class ResultsIndexUpgradeService { // Adjust the following constants as necessary for various versions and backports. private static final int INDEX_VERSION = Version.CURRENT.major; - private static final Version MIN_REQUIRED_VERSION = Version.CURRENT.minimumCompatibilityVersion(); + private static final Version MIN_REQUIRED_VERSION = Version.V_5_6_0; private final IndexNameExpressionResolver indexNameExpressionResolver; private final Predicate shouldUpgrade; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlUpgradeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlUpgradeAction.java index 2b676277aa690..7f090a84e0fb1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlUpgradeAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlUpgradeAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -33,11 +34,11 @@ public class TransportMlUpgradeAction private final ResultsIndexUpgradeService resultsIndexUpgradeService; @Inject - public TransportMlUpgradeAction(TransportService transportService, ClusterService clusterService, + public TransportMlUpgradeAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, Client client, IndexNameExpressionResolver indexNameExpressionResolver) { - super(MlUpgradeAction.NAME, transportService, clusterService, threadPool, - actionFilters, MlUpgradeAction.Request::new, indexNameExpressionResolver); + super(settings, MlUpgradeAction.NAME, transportService, clusterService, threadPool, + actionFilters, indexNameExpressionResolver, MlUpgradeAction.Request::new); this.client = client; this.resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver, executor(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestUpgradeMlAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestUpgradeMlAction.java index cad82ce325c27..405ccad4965fd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestUpgradeMlAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestUpgradeMlAction.java @@ -5,9 +5,7 @@ */ package org.elasticsearch.xpack.ml.rest.results; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -28,18 +26,10 @@ public class RestUpgradeMlAction extends BaseRestHandler { - private static final DeprecationLogger deprecationLogger = - new DeprecationLogger(LogManager.getLogger(RestUpgradeMlAction.class)); - public RestUpgradeMlAction(Settings settings, RestController controller) { super(settings); - controller.registerWithDeprecatedHandler( - POST, - MachineLearning.BASE_PATH + "_upgrade", - this, - POST, - MachineLearning.PRE_V7_BASE_PATH + "_upgrade", - deprecationLogger); + controller.registerHandler(POST, MachineLearning.BASE_PATH + "_upgrade", this); + controller.registerHandler(POST,MachineLearning.V7_BASE_PATH + "_upgrade", this); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.upgrade.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.ml.upgrade.json similarity index 79% rename from x-pack/plugin/src/test/resources/rest-api-spec/api/ml.upgrade.json rename to x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.ml.upgrade.json index b67b125bb692a..863d5ce7dba70 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.upgrade.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.ml.upgrade.json @@ -1,10 +1,10 @@ { - "ml.upgrade": { + "xpack.ml.upgrade": { "documentation": "TODO", "methods": [ "POST" ], "url": { - "path": "/_ml/_upgrade", - "paths": [ "/_ml/_upgrade" ], + "path": "/_xpack/ml/_upgrade", + "paths": [ "/_xpack/ml/_upgrade" ], "params": { "wait_for_completion": { "type": "boolean", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_upgrade.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_upgrade.yml index ee1f9f77f9325..bbdcdab139fcc 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_upgrade.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_upgrade.yml @@ -4,7 +4,7 @@ setup: - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - ml.put_job: + xpack.ml.put_job: job_id: jobs-upgrade-results body: > { @@ -43,7 +43,7 @@ setup: --- "Upgrade results when there is nothing to upgrade": - do: - ml.upgrade: + xpack.ml.upgrade: wait_for_completion: true - match: { acknowledged: true } @@ -56,7 +56,7 @@ setup: --- "Upgrade results when there is nothing to upgrade not waiting for results": - do: - ml.upgrade: + xpack.ml.upgrade: wait_for_completion: false - match: {task: '/.+:\d+/'} diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 170948566b9fb..f93443f95f09f 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -295,8 +295,17 @@ subprojects { systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster' } } + def toBlackList = [] + // If we are upgrading on a minor version, these tests fail + // They only work if we are upgrading between major versions + if (versionParts[0].equals("6")) { + toBlackList << 'upgraded_cluster/80_ml_results_upgrade/Migrate results data to latest index binary version' + } if (version.before('6.1.0') || version.onOrAfter('6.3.0')) { - systemProperty 'tests.rest.blacklist', '/30_ml_jobs_crud/Test model memory limit is updated' + toBlackList << '/30_ml_jobs_crud/Test model memory limit is updated' + } + if (!toBlackList.empty) { + systemProperty 'tests.rest.blacklist', toBlackList.join(',') } } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_ml_results_upgrade.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_ml_results_upgrade.yml index 73478be65597e..45d877a168a06 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_ml_results_upgrade.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_ml_results_upgrade.yml @@ -1,11 +1,11 @@ --- "Verify jobs exist": - do: - ml.get_jobs: + xpack.ml.get_jobs: job_id: old-cluster-job-to-upgrade - match: { count: 1 } - do: - ml.get_jobs: + xpack.ml.get_jobs: job_id: old-cluster-job-to-upgrade-custom - match: { count: 1 } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_ml_results_upgrade.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_ml_results_upgrade.yml index d21b5e6def61d..610724b62a546 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_ml_results_upgrade.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_ml_results_upgrade.yml @@ -2,7 +2,7 @@ "Put job on the old cluster and post some data": - do: - ml.put_job: + xpack.ml.put_job: job_id: old-cluster-job-to-upgrade body: > { @@ -23,11 +23,11 @@ - match: { job_id: old-cluster-job-to-upgrade } - do: - ml.open_job: + xpack.ml.open_job: job_id: old-cluster-job-to-upgrade - do: - ml.post_data: + xpack.ml.post_data: job_id: old-cluster-job-to-upgrade body: - airline: AAL @@ -41,11 +41,11 @@ - match: { processed_record_count: 2 } - do: - ml.close_job: + xpack.ml.close_job: job_id: old-cluster-job-to-upgrade - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: old-cluster-job-to-upgrade - match: { count: 1 } @@ -59,7 +59,7 @@ --- "Put job on the old cluster with a custom index": - do: - ml.put_job: + xpack.ml.put_job: job_id: old-cluster-job-to-upgrade-custom body: > { @@ -81,11 +81,11 @@ - match: { job_id: old-cluster-job-to-upgrade-custom } - do: - ml.open_job: + xpack.ml.open_job: job_id: old-cluster-job-to-upgrade-custom - do: - ml.post_data: + xpack.ml.post_data: job_id: old-cluster-job-to-upgrade-custom body: - airline: AAL @@ -103,11 +103,11 @@ - match: { processed_record_count: 3 } - do: - ml.close_job: + xpack.ml.close_job: job_id: old-cluster-job-to-upgrade-custom - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: old-cluster-job-to-upgrade-custom - match: { count: 3 } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_ml_results_upgrade.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_ml_results_upgrade.yml index f049b9c073ad8..c106f993b36d6 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_ml_results_upgrade.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_ml_results_upgrade.yml @@ -2,12 +2,12 @@ "Migrate results data to latest index binary version": # Verify that all the results are there and the typical indices exist - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: old-cluster-job-to-upgrade - match: { count: 1 } - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: old-cluster-job-to-upgrade-custom - match: { count: 3 } @@ -32,7 +32,7 @@ # Do the upgrade - do: - ml.upgrade: + xpack.ml.upgrade: wait_for_completion: true - match: { acknowledged: true } @@ -56,12 +56,12 @@ indices.refresh: {} - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: old-cluster-job-to-upgrade - match: { count: 1 } - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: old-cluster-job-to-upgrade-custom - match: { count: 3 } @@ -71,20 +71,20 @@ indices.get_settings: index: .ml-anomalies-old-cluster-job-to-upgrade name: index.version.created - - match: { \.ml-anomalies-shared-7.settings.index.version.created: '/7\d+/' } - - match: { \.ml-anomalies-shared-7r.settings.index.version.created: '/7\d+/' } + - match: { \.ml-anomalies-shared-6.settings.index.version.created: '/6\d+/' } + - match: { \.ml-anomalies-shared-6r.settings.index.version.created: '/6\d+/' } - do: indices.get_settings: index: .ml-anomalies-old-cluster-job-to-upgrade-custom name: index.version.created - - match: { \.ml-anomalies-custom-old-cluster-job-to-upgrade-custom-7.settings.index.version.created: '/7\d+/' } - - match: { \.ml-anomalies-custom-old-cluster-job-to-upgrade-custom-7r.settings.index.version.created: '/7\d+/' } + - match: { \.ml-anomalies-custom-old-cluster-job-to-upgrade-custom-6.settings.index.version.created: '/6\d+/' } + - match: { \.ml-anomalies-custom-old-cluster-job-to-upgrade-custom-6r.settings.index.version.created: '/6\d+/' } # Create a new job to verify that the .ml-anomalies-shared index gets created again without issues - do: - ml.put_job: + xpack.ml.put_job: job_id: upgraded-cluster-job-should-not-upgrade body: > { @@ -105,11 +105,11 @@ - match: { job_id: upgraded-cluster-job-should-not-upgrade } - do: - ml.open_job: + xpack.ml.open_job: job_id: upgraded-cluster-job-should-not-upgrade - do: - ml.post_data: + xpack.ml.post_data: job_id: upgraded-cluster-job-should-not-upgrade body: - airline: AAL @@ -123,11 +123,11 @@ - match: { processed_record_count: 2 } - do: - ml.close_job: + xpack.ml.close_job: job_id: upgraded-cluster-job-should-not-upgrade - do: - ml.get_buckets: + xpack.ml.get_buckets: job_id: upgraded-cluster-job-should-not-upgrade - match: { count: 1 } @@ -142,11 +142,11 @@ index: .ml-anomalies-shared name: index.version.created - - match: { \.ml-anomalies-shared.settings.index.version.created: '/7\d+/' } + - match: { \.ml-anomalies-shared.settings.index.version.created: '/6\d+/' } # Do the upgrade Again as nothing needs upgraded now - do: - ml.upgrade: + xpack.ml.upgrade: wait_for_completion: true - match: { acknowledged: true }