Create etcd docker tests

tests/conftest.py

@@ -26,6 +26,23 @@ def stop_docker(container):
         subprocess.call(["docker", "rm", "-f", "-v", cid])
 
 
+def yield_url(url: str):
+    timeout = 10
+    while True:
+        try:
+            LOGGER.debug("trying to connected to alluxio")
+            r = requests.get(url + "/v1/files?path=/")
+            LOGGER.debug("successfullly connected to alluxio")
+            if r.ok:
+                return url
+                break
+        except Exception as e:  # noqa: E722
+            timeout -= 1
+            if timeout < 0:
+                raise SystemError from e
+            time.sleep(10)
+
+
 @pytest.fixture(scope="module")
 def docker_alluxio():
     if "ALLUXIO_URL" in os.environ:
@@ -67,30 +84,93 @@ def docker_alluxio():
     subprocess.check_output(shlex.split(run_cmd_master))
     subprocess.check_output(shlex.split(run_cmd_worker))
     url = "http://127.0.0.1:28080"
-    timeout = 10
-    while True:
-        try:
-            LOGGER.debug("trying to connected to alluxio")
-            r = requests.get(url + "/v1/files?path=/")
-            LOGGER.debug("successfullly connected to alluxio")
-            if r.ok:
-                yield url
-                break
-        except Exception as e:  # noqa: E722
-            timeout -= 1
-            if timeout < 0:
-                raise SystemError from e
-            time.sleep(10)
+    yield yield_url(url)
     stop_docker(worker_container)
     stop_docker(master_container)
 
 
+@pytest.fixture(scope="module")
+def docker_alluxio_with_etcd():
+    if "ALLUXIO_URL" in os.environ:
+        # assume we already have a server already set up
+        yield os.getenv("ALLUXIO_URL")
+        return
+    master_container = "alluxio-master"
+    worker_container = "alluxio-worker"
+    etcd_container = "etcd"
+    network_cmd = "docker network create alluxio_network"
+
+    run_cmd_master = (
+        "docker run --platform linux/amd64 -d --rm --net=alluxio_network -p 19999:19999 -p 19998:19998 "
+        f"--name=alluxio-master -v {TEST_DIR}:/opt/alluxio/ufs "
+        '-e ALLUXIO_JAVA_OPTS=" -Dalluxio.master.hostname=alluxio-master '
+        "-Dalluxio.security.authentication.type=NOSASL "
+        "-Dalluxio.security.authorization.permission.enabled=false "
+        "-Dalluxio.security.authorization.plugins.enabled=false "
+        "-Dalluxio.master.journal.type=NOOP "
+        "-Dalluxio.worker.membership.manager.type=ETCD "
+        "-Dalluxio.etcd.endpoints=http://etcd:2379 "
+        "-Dalluxio.master.scheduler.initial.wait.time=1s "
+        "-Dalluxio.dora.client.ufs.root=file:/// "
+        '-Dalluxio.underfs.xattr.change.enabled=false " alluxio/alluxio:310-SNAPSHOT master'
+    )
+    run_cmd_worker = (
+        "docker run --platform linux/amd64 -d --rm -h localhost --net=alluxio_network -p 28080:28080 -p 29999:29999 -p 29997:29997 "
+        f"--name=alluxio-worker --shm-size=1G -v {TEST_DIR}:/opt/alluxio/ufs "
+        '-e ALLUXIO_JAVA_OPTS=" -Dalluxio.master.hostname=alluxio-master '
+        "-Dalluxio.security.authentication.type=NOSASL "
+        "-Dalluxio.security.authorization.permission.enabled=false "
+        "-Dalluxio.security.authorization.plugins.enabled=false "
+        "-Dalluxio.worker.membership.manager.type=ETCD "
+        "-Dalluxio.worker.container.hostname=alluxio-worker "
+        "-Dalluxio.etcd.endpoints=http://etcd:2379 "
+        "-Dalluxio.dora.client.ufs.root=file:/// "
+        '-Dalluxio.underfs.xattr.change.enabled=false " alluxio/alluxio:310-SNAPSHOT worker'
+    )
+
+    run_cmd_etcd = (
+        "docker run --platform linux/amd64 -d --rm --net=alluxio_network -p 4001:4001 -p 2380:2380 -p 2379:2379 "
+        f"-v {TEST_DIR}:/etc/ssl/certs "
+        "--name etcd quay.io/coreos/etcd:latest "
+        "/usr/local/bin/etcd "
+        "--data-dir=/etcd-data "
+        "--name etcd1 "
+        "--listen-client-urls http://0.0.0.0:2379 "
+        "--advertise-client-urls http://0.0.0.0:2379"
+    )
+
+    stop_docker(worker_container)
+    stop_docker(master_container)
+    stop_docker(etcd_container)
+    subprocess.run(
+        shlex.split(network_cmd)
+    )  # could return error code if network already exists
+    subprocess.check_output(shlex.split(run_cmd_etcd))
+    subprocess.check_output(shlex.split(run_cmd_master))
+    subprocess.check_output(shlex.split(run_cmd_worker))
+    url = "http://127.0.0.1:28080"
+    yield yield_url(url)
+    stop_docker(worker_container)
+    stop_docker(master_container)
+    stop_docker(etcd_container)
+
+
 @pytest.fixture
 def alluxio_client(docker_alluxio):
-
     LOGGER.debug(f"get AlluxioClient connect to {docker_alluxio}")
     parsed_url = urlparse(docker_alluxio)
     host = parsed_url.hostname
     port = parsed_url.port
     alluxio_client = AlluxioClient(worker_hosts=host, worker_http_port=port)
     yield alluxio_client
+
+
+@pytest.fixture
+def etcd_alluxio_client(docker_alluxio_with_etcd):
+    LOGGER.debug(
+        f"get etcd AlluxioFileSystem connect to {docker_alluxio_with_etcd}"
+    )
+    parsed_url = urlparse(docker_alluxio_with_etcd)
+    host = parsed_url.hostname
+    etcd_alluxio_client = AlluxioClient(etcd_hosts=host)
+    yield etcd_alluxio_client

tests/test_docker_alluxio.py

@@ -3,3 +3,7 @@
 
 def test_simple(alluxio_client):
     alluxio_client.listdir(TEST_ROOT)  # no error
+
+
+def test_simple_etcd(etcd_alluxio_client):
+    etcd_alluxio_client.listdir(TEST_ROOT)  # no error

tests/test_read_range_docker.py

@@ -110,3 +110,7 @@ def test_alluxio_client(alluxio_client: AlluxioClient):
             length,
         )
     LOGGER.debug("Passed corner test cases")
+
+
+def test_etcd_alluxio_client(etcd_alluxio_client: AlluxioClient):
+    test_alluxio_client(etcd_alluxio_client)