Remove Elasticsearch 6.3 logic (#1497)

Rally only supports Elasticsearch 6.8+.

docs/car.rst

@@ -32,7 +32,7 @@ A Rally "car" is a specific configuration of Elasticsearch. You can list the ava
     x-pack-monitoring-local  mixin   X-Pack Monitoring (local exporter)
     x-pack-security          mixin   X-Pack Security
 
-You can specify the car that Rally should use with e.g. ``--car="4gheap"``. It is also possible to specify one or more "mixins" to further customize the configuration. For example, you can specify ``--car="4gheap,ea"`` to run with a 4GB heap and enable Java assertions (they are disabled by default) or ``--car="4gheap,x-pack-security"`` to benchmark Elasticsearch with X-Pack Security enabled (requires Elasticsearch 6.3.0 or better).
+You can specify the car that Rally should use with e.g. ``--car="4gheap"``. It is also possible to specify one or more "mixins" to further customize the configuration. For example, you can specify ``--car="4gheap,ea"`` to run with a 4GB heap and enable Java assertions (they are disabled by default) or ``--car="4gheap,x-pack-security"`` to benchmark Elasticsearch with X-Pack Security enabled.
 
 .. note::
     To benchmark ``x-pack-security`` you need to add the following command line options: ``--client-options="use_ssl:true,verify_certs:false,basic_auth_user:'rally',basic_auth_password:'rally-password'"``

docs/elasticsearch_plugins.rst

@@ -53,19 +53,12 @@ Rally will use several techniques to install and configure plugins:
 * First, Rally checks whether directory ``plugins/PLUGIN_NAME`` in the currently configured team repository exists. If this is the case, then plugin installation and configuration details will be read from this directory.
 * Next, Rally will use the provided plugin name when running the Elasticsearch plugin installer. With this approach we can avoid to create a plugin configuration directory in the team repository for very simple plugins that do not need any configuration.
 
-As mentioned above, Rally also allows you to specify a plugin configuration and you can even combine them. Here are some examples (requires Elasticsearch < 6.3.0 because with 6.3.0 x-pack has turned into a module of Elasticsearch which is treated as a "car" in Rally):
+As mentioned above, Rally also allows you to specify a plugin configuration and you can even combine them. Here are some examples:
 
-* Run a benchmark with the ``x-pack`` plugin in the ``security`` configuration: ``--elasticsearch-plugins=x-pack:security``
-* Run a benchmark with the ``x-pack`` plugin in the ``security`` and the ``graph`` configuration: ``--elasticsearch-plugins=x-pack:security+graph``
+* Run a benchmark with the ``transport-nio`` plugin in the ``http`` configuration: ``--elasticsearch-plugins=transport-nio:http``
+* Run a benchmark with the ``transport-nio`` plugin in the ``http`` and the ``transport`` configuration: ``--elasticsearch-plugins=transport-nio:http+transport``
 
-.. note::
-    To benchmark the ``security`` configuration of ``x-pack`` you need to add the following command line options: ``--client-options="use_ssl:true,verify_certs:false,basic_auth_user:'rally',basic_auth_password:'rally-password'"``
-
-You can also override plugin variables with ``--plugin-params`` which is needed for example if you want to use the ``monitoring-http`` configuration in order to export monitoring data. You can export monitoring data e.g. with the following configuration::
-
-    --elasticsearch-plugins="x-pack:monitoring-http" --plugin-params="monitoring_type:'http',monitoring_host:'some_remote_host',monitoring_port:10200,monitoring_user:'rally',monitoring_password:'m0n1t0r1ng'"
-
-The ``monitoring_user`` and ``monitoring_password`` parameters are optional, the other parameters are mandatory. For more details on the configuration options check the `Monitoring plugin documentation <https://www.elastic.co/guide/en/x-pack/current/monitoring-production.html>`_.
+You can also override plugin variables with ``--plugin-params``.
 
 If you are behind a proxy, set the environment variable ``ES_JAVA_OPTS`` accordingly on each target machine as described in the `Elasticsearch plugin documentation <https://www.elastic.co/guide/en/elasticsearch/plugins/current/_other_command_line_parameters.html#_proxy_settings>`_.
 

esrally/driver/driver.py

@@ -354,7 +354,8 @@ def _after_track_prepared(self):
         self.send(
             self.start_sender,
             PreparationComplete(
-                # older versions (pre 6.3.0) don't expose build_flavor because the only (implicit) flavor was "oss"
+                # manually compiled versions don't expose build_flavor but Rally expects a value in telemetry devices
+                # we should default to trial/basic, but let's default to oss for now to avoid breaking the chart generator
                 cluster_version.get("build_flavor", "oss"),
                 cluster_version.get("number"),
                 cluster_version.get("build_hash"),

esrally/mechanic/provisioner.py

@@ -26,7 +26,7 @@
 
 from esrally import exceptions
 from esrally.mechanic import java_resolver, team
-from esrally.utils import console, convert, io, process, versions
+from esrally.utils import console, convert, io, process
 
 
 def local(cfg, car, plugins, ip, http_port, all_node_ips, all_node_names, target_root, node_name):
@@ -219,17 +219,7 @@ def _provisioner_variables(self):
         plugin_variables = {}
         mandatory_plugins = []
         for installer in self.plugin_installers:
-            # For Elasticsearch < 6.3 more specific plugin names are required for mandatory plugin check
-            # Details in: https://github.com/elastic/elasticsearch/pull/28710
-            # TODO: Remove this section with Elasticsearch <6.3 becomes EOL.
-            try:
-                major, minor, _, _ = versions.components(self.distribution_version)
-                if (major == 6 and minor < 3) or major < 6:
-                    mandatory_plugins.append(installer.sub_plugin_name)
-                else:
-                    mandatory_plugins.append(installer.plugin_name)
-            except (TypeError, exceptions.InvalidSyntax):
-                mandatory_plugins.append(installer.plugin_name)
+            mandatory_plugins.append(installer.plugin_name)
             plugin_variables.update(installer.variables)
 
         cluster_settings = {}

esrally/telemetry.py

@@ -1755,8 +1755,7 @@ def on_benchmark_start(self):
             return
         revision = client_info["version"]["build_hash"]
         distribution_version = client_info["version"]["number"]
-        # older versions (pre 6.3.0) don't expose a build_flavor property because the only (implicit) flavor was "oss".
-        distribution_flavor = client_info["version"].get("build_flavor", "oss")
+        distribution_flavor = client_info["version"]["build_flavor"]
         self.metrics_store.add_meta_info(metrics.MetaInfoScope.cluster, None, "source_revision", revision)
         self.metrics_store.add_meta_info(metrics.MetaInfoScope.cluster, None, "distribution_version", distribution_version)
         self.metrics_store.add_meta_info(metrics.MetaInfoScope.cluster, None, "distribution_flavor", distribution_flavor)

tests/mechanic/provisioner_test.py

@@ -128,101 +128,15 @@ def __repr__(self):
                 r.append("%s = [%s]" % (prop, repr(value)))
             return ", ".join(r)
 
-    @mock.patch("glob.glob", lambda p: ["/opt/elasticsearch-5.0.0"])
-    @mock.patch("esrally.utils.io.decompress")
-    @mock.patch("esrally.utils.io.ensure_dir")
-    @mock.patch("esrally.mechanic.provisioner.PluginInstaller.install")
-    @mock.patch("shutil.rmtree")
-    def test_prepare_distribution_lt_63_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress):
-        """
-        Test that plugin.mandatory is set to the specific plugin name (e.g. `x-pack-security`) and not
-        the meta plugin name (e.g. `x-pack`) for Elasticsearch <6.3
-
-        See: https://github.com/elastic/elasticsearch/pull/28710
-        """
-        apply_config_calls = []
-
-        def null_apply_config(source_root_path, target_root_path, config_vars):
-            apply_config_calls.append((source_root_path, target_root_path, config_vars))
-
-        installer = provisioner.ElasticsearchInstaller(
-            car=team.Car(
-                names="unit-test-car",
-                root_path=None,
-                config_paths=[HOME_DIR + "/.rally/benchmarks/teams/default/my-car"],
-                variables={
-                    "heap": "4g",
-                    "runtime.jdk": "8",
-                    "runtime.jdk.bundled": "true",
-                },
-            ),
-            java_home="/usr/local/javas/java8",
-            node_name="rally-node-0",
-            cluster_name="rally-benchmark",
-            node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
-            all_node_ips=["10.17.22.22", "10.17.22.23"],
-            all_node_names=["rally-node-0", "rally-node-1"],
-            ip="10.17.22.23",
-            http_port=9200,
-        )
-
-        p = provisioner.BareProvisioner(
-            es_installer=installer,
-            plugin_installers=[
-                provisioner.PluginInstaller(
-                    self.MockRallyTeamXPackPlugin(),
-                    java_home="/usr/local/javas/java8",
-                    hook_handler_class=self.NoopHookHandler,
-                )
-            ],
-            distribution_version="6.2.3",
-            apply_config=null_apply_config,
-        )
-
-        node_config = p.prepare({"elasticsearch": "/opt/elasticsearch-5.0.0.tar.gz"})
-        assert node_config.car_runtime_jdks == "8"
-        assert node_config.binary_path == "/opt/elasticsearch-5.0.0"
-        assert node_config.data_paths == ["/opt/elasticsearch-5.0.0/data"]
-
-        assert len(apply_config_calls) == 1
-        source_root_path, target_root_path, config_vars = apply_config_calls[0]
-
-        assert source_root_path == HOME_DIR + "/.rally/benchmarks/teams/default/my-car"
-        assert target_root_path == "/opt/elasticsearch-5.0.0"
-
-        self.maxDiff = None
-
-        assert config_vars == {
-            "cluster_settings": {"plugin.mandatory": ["x-pack-security"]},
-            "heap": "4g",
-            "runtime.jdk": "8",
-            "runtime.jdk.bundled": "true",
-            "cluster_name": "rally-benchmark",
-            "node_name": "rally-node-0",
-            "data_paths": ["/opt/elasticsearch-5.0.0/data"],
-            "log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server",
-            "heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump",
-            "node_ip": "10.17.22.23",
-            "network_host": "10.17.22.23",
-            "http_port": "9200",
-            "transport_port": "9300",
-            "all_node_ips": '["10.17.22.22","10.17.22.23"]',
-            "all_node_names": '["rally-node-0","rally-node-1"]',
-            "minimum_master_nodes": 2,
-            "install_root_path": "/opt/elasticsearch-5.0.0",
-            "plugin_name": "x-pack-security",
-            "xpack_security_enabled": True,
-        }
-
-    @mock.patch("glob.glob", lambda p: ["/opt/elasticsearch-6.3.0"])
+    @mock.patch("glob.glob", lambda p: ["/opt/elasticsearch-6.8.0"])
     @mock.patch("esrally.utils.io.decompress")
     @mock.patch("esrally.utils.io.ensure_dir")
     @mock.patch("esrally.mechanic.provisioner.PluginInstaller.install")
     @mock.patch("shutil.rmtree")
-    def test_prepare_distribution_ge_63_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress):
+    def test_prepare_distribution_with_plugins(self, mock_rm, mock_ensure_dir, mock_install, mock_decompress):
         """
         Test that plugin.mandatory is set to the meta plugin name (e.g. `x-pack`) and not
-        the specific plugin name (e.g. `x-pack-security`) for Elasticsearch >=6.3.0
+        the specific plugin name (e.g. `x-pack-security`)
 
         See: https://github.com/elastic/elasticsearch/pull/28710
         """
@@ -261,20 +175,20 @@ def null_apply_config(source_root_path, target_root_path, config_vars):
                     hook_handler_class=self.NoopHookHandler,
                 )
             ],
-            distribution_version="6.3.0",
+            distribution_version="6.8.0",
             apply_config=null_apply_config,
         )
 
-        node_config = p.prepare({"elasticsearch": "/opt/elasticsearch-6.3.0.tar.gz"})
+        node_config = p.prepare({"elasticsearch": "/opt/elasticsearch-6.8.0.tar.gz"})
         assert node_config.car_runtime_jdks == "8"
-        assert node_config.binary_path == "/opt/elasticsearch-6.3.0"
-        assert node_config.data_paths == ["/opt/elasticsearch-6.3.0/data"]
+        assert node_config.binary_path == "/opt/elasticsearch-6.8.0"
+        assert node_config.data_paths == ["/opt/elasticsearch-6.8.0/data"]
 
         assert len(apply_config_calls) == 1
         source_root_path, target_root_path, config_vars = apply_config_calls[0]
 
         assert source_root_path == HOME_DIR + "/.rally/benchmarks/teams/default/my-car"
-        assert target_root_path == "/opt/elasticsearch-6.3.0"
+        assert target_root_path == "/opt/elasticsearch-6.8.0"
 
         self.maxDiff = None
 
@@ -285,7 +199,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars):
             "runtime.jdk.bundled": "true",
             "cluster_name": "rally-benchmark",
             "node_name": "rally-node-0",
-            "data_paths": ["/opt/elasticsearch-6.3.0/data"],
+            "data_paths": ["/opt/elasticsearch-6.8.0/data"],
             "log_path": HOME_DIR + "/.rally/benchmarks/races/unittest/logs/server",
             "heap_dump_path": HOME_DIR + "/.rally/benchmarks/races/unittest/heapdump",
             "node_ip": "10.17.22.23",
@@ -295,7 +209,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars):
             "all_node_ips": '["10.17.22.22","10.17.22.23"]',
             "all_node_names": '["rally-node-0","rally-node-1"]',
             "minimum_master_nodes": 2,
-            "install_root_path": "/opt/elasticsearch-6.3.0",
+            "install_root_path": "/opt/elasticsearch-6.8.0",
             "plugin_name": "x-pack-security",
             "xpack_security_enabled": True,
         }

tests/telemetry_test.py

@@ -2973,6 +2973,7 @@ def test_stores_cluster_level_metrics_on_attach(self, metrics_store_add_meta_inf
             "version": {
                 "build_hash": "abc123",
                 "number": "6.0.0-alpha1",
+                "build_flavor": "oss",
             },
         }