Add Docker-based tests for proxy connection mode (#51658)

Adds more tests for the new "proxy" remote cluster connection mode, using a Docker-based setup, as well as testing SNI-based routing using HAProxy. - Checks that the new proxy mode can work in situations where the publish host of the nodes in the remote cluster are not routable. - Checks that the new proxy mode can work with clusters where nodes are hidden behind HAProxy. - Checks that the new proxy mode can work with clusters where nodes are hidden behind HAProxy, using SNI to identify the nodes/cluster behind HAProxy. Relates #49067
elastic · Feb 3, 2020 · 39b4710 · 39b4710
1 parent 493f77d
commit 39b4710
Show file tree

Hide file tree

Showing 8 changed files with 696 additions and 0 deletions.
diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.test.fixtures'
+
+testFixtures.useFixture()
+
+configurations {
+  restSpec
+}
+
+dependencies {
+  restSpec project(':rest-api-spec')
+  testCompile project(':client:rest-high-level')
+}
+
+task copyKeystore(type: Sync) {
+  from project(':x-pack:plugin:core')
+    .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+  into "${buildDir}/certs"
+  doLast {
+    file("${buildDir}/certs").setReadable(true, false)
+    file("${buildDir}/certs/testnode.jks").setReadable(true, false)
+  }
+}
+
+preProcessFixture {
+  if (TestFixturesPlugin.dockerComposeSupported()) {
+    if ('default'.equalsIgnoreCase(System.getProperty('tests.distribution', 'default'))) {
+      dependsOn ":distribution:docker:buildDockerImage"
+    } else {
+      dependsOn ":distribution:docker:buildOssDockerImage"
+    }
+  }
+  dependsOn copyKeystore
+  doLast {
+    // tests expect to have an empty repo
+    project.delete(
+      "${buildDir}/repo",
+      "${buildDir}/oss-repo"
+    )
+    createAndSetWritable(
+      "${buildDir}/repo",
+      "${buildDir}/oss-repo",
+      "${buildDir}/logs/default-1",
+      "${buildDir}/logs/default-2",
+      "${buildDir}/logs/oss-1",
+      "${buildDir}/logs/oss-2"
+    )
+  }
+}
+
+if (TestFixturesPlugin.dockerComposeSupported()) {
+  dockerCompose {
+    tcpPortsToIgnoreWhenWaiting = [9600, 9601]
+    if ('default'.equalsIgnoreCase(System.getProperty('tests.distribution', 'default'))) {
+      useComposeFiles = ['docker-compose.yml']
+    } else {
+      useComposeFiles = ['docker-compose-oss.yml']
+    }
+  }
+}
+
+def createAndSetWritable(Object... locations) {
+  locations.each { location ->
+    File file = file(location)
+    file.mkdirs()
+    file.setWritable(true, false)
+  }
+}
+
+processTestResources {
+  from({ zipTree(configurations.restSpec.singleFile) }) {
+    include 'rest-api-spec/api/**'
+  }
+  from project(':x-pack:plugin:core')
+    .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+  dependsOn configurations.restSpec
+}
+
+task integTest(type: Test) {
+  outputs.doNotCacheIf('Build cache is disabled for Docker tests') { true }
+  maxParallelForks = '1'
+  include '**/*IT.class'
+  // don't add the tasks to build the docker images if we have no way of testing them
+  if (TestFixturesPlugin.dockerComposeSupported()) {
+    dependsOn ":distribution:docker:buildDockerImage"
+  }
+}
+
+check.dependsOn integTest
diff --git a/qa/remote-clusters/docker-compose-oss.yml b/qa/remote-clusters/docker-compose-oss.yml
@@ -0,0 +1,76 @@
+# Only used for testing the docker images
+version: '3.7'
+services:
+  elasticsearch-oss-1:
+    image: elasticsearch:test
+    environment:
+       - node.name=elasticsearch-oss-1
+       - cluster.initial_master_nodes=elasticsearch-oss-1
+       - cluster.name=elasticsearch-oss-1
+       - bootstrap.memory_lock=true
+       - network.publish_host=127.0.0.1
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false
+    volumes:
+       - ./build/oss-repo:/tmp/es-repo
+       - ./build/logs/oss-1:/usr/share/elasticsearch/logs
+    ports:
+      - "9200"
+      - "9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
+  elasticsearch-oss-2:
+    image: elasticsearch:test
+    environment:
+       - node.name=elasticsearch-oss-2
+       - cluster.initial_master_nodes=elasticsearch-oss-2
+       - cluster.name=elasticsearch-oss-2
+       - bootstrap.memory_lock=true
+       - network.publish_host=127.0.0.1
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false
+    volumes:
+       - ./build/oss-repo:/tmp/es-repo
+       - ./build/logs/oss-2:/usr/share/elasticsearch/logs
+    ports:
+      - "9200"
+      - "9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
+  haproxy:
+    image: haproxy:2.1.2
+    ports:
+      - "9600"
+    volumes:
+      - ./haproxy-oss.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml
@@ -0,0 +1,109 @@
+# Only used for testing the docker images
+version: '3.7'
+services:
+  elasticsearch-default-1:
+    image: elasticsearch:test
+    environment:
+       - node.name=elasticsearch-default-1
+       - cluster.initial_master_nodes=elasticsearch-default-1
+       - cluster.name=elasticsearch-default-1
+       - bootstrap.memory_lock=true
+       - network.publish_host=127.0.0.1
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false
+       - xpack.security.enabled=true
+       - xpack.security.transport.ssl.enabled=true
+       - xpack.security.http.ssl.enabled=true
+       - xpack.security.authc.token.enabled=true
+       - xpack.security.audit.enabled=true
+       - xpack.security.authc.realms.file.file1.order=0
+       - xpack.security.authc.realms.native.native1.order=1
+       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.http.ssl.verification_mode=certificate
+       - xpack.security.transport.ssl.verification_mode=certificate
+       - xpack.license.self_generated.type=trial
+    volumes:
+       - ./build/repo:/tmp/es-repo
+       - ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
+       - ./build/logs/default-1:/usr/share/elasticsearch/logs
+       - ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
+    ports:
+      - "9200"
+      - "9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    entrypoint: /docker-test-entrypoint.sh
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "-u", "x_pack_rest_user:x-pack-test-password", "-k", "https://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
+  elasticsearch-default-2:
+    image: elasticsearch:test
+    environment:
+       - node.name=elasticsearch-default-2
+       - cluster.initial_master_nodes=elasticsearch-default-2
+       - cluster.name=elasticsearch-default-2
+       - bootstrap.memory_lock=true
+       - network.publish_host=127.0.0.1
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - path.repo=/tmp/es-repo
+       - node.attr.testattr=test
+       - cluster.routing.allocation.disk.watermark.low=1b
+       - cluster.routing.allocation.disk.watermark.high=1b
+       - cluster.routing.allocation.disk.watermark.flood_stage=1b
+       - script.max_compilations_rate=2048/1m
+       - node.store.allow_mmap=false
+       - xpack.security.enabled=true
+       - xpack.security.transport.ssl.enabled=true
+       - xpack.security.http.ssl.enabled=true
+       - xpack.security.authc.token.enabled=true
+       - xpack.security.audit.enabled=true
+       - xpack.security.authc.realms.file.file1.order=0
+       - xpack.security.authc.realms.native.native1.order=1
+       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
+       - xpack.http.ssl.verification_mode=certificate
+       - xpack.security.transport.ssl.verification_mode=certificate
+       - xpack.license.self_generated.type=trial
+    volumes:
+       - ./build/repo:/tmp/es-repo
+       - ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
+       - ./build/logs/default-2:/usr/share/elasticsearch/logs
+       - ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
+    ports:
+      - "9200"
+      - "9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    entrypoint: /docker-test-entrypoint.sh
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "-u", "x_pack_rest_user:x-pack-test-password", "-k", "https://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
+  haproxy:
+    image: haproxy:2.1.2
+    ports:
+      - "9600"
+    volumes:
+      - ./haproxy-default.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
diff --git a/qa/remote-clusters/docker-test-entrypoint.sh b/qa/remote-clusters/docker-test-entrypoint.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+cd /usr/share/elasticsearch/bin/
+./elasticsearch-users useradd x_pack_rest_user -p x-pack-test-password -r superuser || true
+echo "testnode" > /tmp/password
+cat /tmp/password  | ./elasticsearch-keystore add -x -f -v 'xpack.security.transport.ssl.keystore.secure_password'
+cat /tmp/password  | ./elasticsearch-keystore add -x -f -v 'xpack.security.http.ssl.keystore.secure_password'
+/usr/local/bin/docker-entrypoint.sh | tee > /usr/share/elasticsearch/logs/console.log
diff --git a/qa/remote-clusters/haproxy-default.cfg b/qa/remote-clusters/haproxy-default.cfg
@@ -0,0 +1,27 @@
+global
+  log localhost local1 notice
+  maxconn 2000
+  daemon
+
+defaults
+  log global
+  mode tcp
+  retries 3
+  option tcpka
+  option tcp-check
+  timeout client 30s
+  timeout server 30s
+  timeout connect 5s
+
+frontend ft_ssl
+  bind *:9600
+  tcp-request inspect-delay 5s
+  tcp-request content accept if { req_ssl_hello_type 1 }
+  default_backend bk_ssl
+
+backend bk_ssl
+  use-server server1 if { req_ssl_sni -i application1.example.com }
+  server server1 elasticsearch-default-1:9300 weight 0 check
+  use-server server2 if { req_ssl_sni -i application2.example.com }
+  server server2 elasticsearch-default-2:9300 weight 0 check
+  server default elasticsearch-default-2:9300 check
diff --git a/qa/remote-clusters/haproxy-oss.cfg b/qa/remote-clusters/haproxy-oss.cfg
@@ -0,0 +1,21 @@
+global
+  log localhost local1 notice
+  maxconn 2000
+  daemon
+
+defaults
+  log global
+  mode tcp
+  retries 3
+  option tcpka
+  option tcp-check
+  timeout client 30s
+  timeout server 30s
+  timeout connect 5s
+
+frontend ft_reg
+  bind *:9600
+  default_backend bk_reg
+
+backend bk_reg
+  server default elasticsearch-oss-2:9300 check