-
Notifications
You must be signed in to change notification settings - Fork 5.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added functional testing framework using TestContainers
- Loading branch information
Showing
17 changed files
with
335 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
...cution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWithContainers.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.facebook.presto.nativeworker; | ||
|
|
||
| import org.testcontainers.containers.BindMode; | ||
| import org.testcontainers.containers.Container; | ||
| import org.testcontainers.containers.GenericContainer; | ||
| import org.testcontainers.containers.Network; | ||
| import org.testcontainers.containers.wait.strategy.Wait; | ||
| import org.testng.annotations.AfterClass; | ||
| import org.testng.annotations.BeforeClass; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| import java.io.IOException; | ||
| import java.time.Duration; | ||
|
|
||
| import static org.testng.Assert.assertTrue; | ||
| import static org.testng.Assert.fail; | ||
|
|
||
| public class TestPrestoNativeWithContainers | ||
| { | ||
| private static final String PRESTO_COORDINATOR_IMAGE = System.getProperty("coordinatorImage", "presto-coordinator:latest"); | ||
| private static final String PRESTO_WORKER_IMAGE = System.getProperty("workerImage", "presto-worker:latest"); | ||
| private static final String BASE_DIR = System.getProperty("user.dir"); | ||
| private static final Network network = Network.newNetwork(); | ||
| private GenericContainer<?> coordinator; | ||
| private GenericContainer<?> worker; | ||
|
|
||
| @BeforeClass | ||
| public void setUp() | ||
| throws InterruptedException | ||
| { | ||
| coordinator = new GenericContainer<>(PRESTO_COORDINATOR_IMAGE) | ||
| .withExposedPorts(8081) | ||
| .withNetwork(network).withNetworkAliases("presto-coordinator") | ||
| .withFileSystemBind(BASE_DIR + "/testcontainers/coordinator/etc", "/opt/presto-server/etc", BindMode.READ_WRITE) | ||
| .withFileSystemBind(BASE_DIR + "/testcontainers/coordinator/entrypoint.sh", "/opt/entrypoint.sh", BindMode.READ_ONLY) | ||
| .waitingFor(Wait.forLogMessage(".*======== SERVER STARTED ========.*", 1)) | ||
| .withStartupTimeout(Duration.ofSeconds(120)); | ||
|
|
||
| worker = new GenericContainer<>(PRESTO_WORKER_IMAGE) | ||
| .withExposedPorts(7777) | ||
| .withNetwork(network).withNetworkAliases("presto-worker") | ||
| .withFileSystemBind(BASE_DIR + "/testcontainers/nativeworker/velox-etc", "/opt/presto-server/etc", BindMode.READ_ONLY) | ||
| .withFileSystemBind(BASE_DIR + "/testcontainers/nativeworker/entrypoint.sh", "/opt/entrypoint.sh", BindMode.READ_ONLY) | ||
| .waitingFor(Wait.forLogMessage(".*Announcement succeeded: HTTP 202.*", 1)); | ||
|
|
||
| coordinator.start(); | ||
| worker.start(); | ||
| } | ||
|
|
||
| @AfterClass | ||
| public void tearDown() | ||
| { | ||
| coordinator.stop(); | ||
| worker.stop(); | ||
| } | ||
|
|
||
| private Container.ExecResult executeQuery(String sql) | ||
| throws IOException, InterruptedException | ||
| { | ||
| // Command to run inside the coordinator container using the presto-cli. | ||
| String[] command = { | ||
| "/opt/presto-cli", | ||
| "--server", | ||
| "presto-coordinator:8081", | ||
| "--execute", | ||
| sql | ||
| }; | ||
|
|
||
| Container.ExecResult execResult = coordinator.execInContainer(command); | ||
| if (execResult.getExitCode() != 0) { | ||
| String errorDetails = "Stdout: " + execResult.getStdout() + "\nStderr: " + execResult.getStderr(); | ||
| fail("Presto CLI exited with error code: " + execResult.getExitCode() + "\n" + errorDetails); | ||
| } | ||
| return execResult; | ||
| } | ||
|
|
||
| @Test | ||
| public void testBasics() | ||
| throws IOException, InterruptedException | ||
| { | ||
| String selectRuntimeNodes = "select * from system.runtime.nodes"; | ||
| executeQuery(selectRuntimeNodes); | ||
| String showCatalogs = "show catalogs"; | ||
| executeQuery(showCatalogs); | ||
| String showSession = "show session"; | ||
| executeQuery(showSession); | ||
| } | ||
|
|
||
| @Test | ||
| public void testFunctions() | ||
| throws IOException, InterruptedException | ||
| { | ||
| String countValues = "SELECT COUNT(*) FROM (VALUES 1, 0, 0, 2, 3, 3) as t(x)"; | ||
| Container.ExecResult countResult = executeQuery(countValues); | ||
| assertTrue(countResult.getStdout().contains("6"), "Count is incorrect."); | ||
|
|
||
| String sqlArrayIntegers = "SELECT array_sort(ARRAY [5, 20, null, 5, 3, 50])"; | ||
| Container.ExecResult execResultIntegers = executeQuery(sqlArrayIntegers); | ||
| assertTrue(execResultIntegers.getStdout().contains("[3, 5, 5, 20, 50, null]"), "Integer array not sorted correctly."); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| # Prestissimo: Functional Testing Using Containers | ||
|
|
||
| This Java test framework allows you to run test cases by deploying Presto coordinator and worker nodes in containers. | ||
| #### To set up Docker services and basic build tools on Ubuntu 22.04 x86_64 machine, run the following command: | ||
| ``` | ||
| apt install podman-docker | ||
| apt install make | ||
| apt install cmake | ||
| ``` | ||
| Note: Run the following commands to fix an open bug related to CNI network in podman. | ||
| Reference: https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394 | ||
| ``` | ||
| curl -O http://archive.ubuntu.com/ubuntu/pool/universe/g/golang-github-containernetworking-plugins/containernetworking-plugins_1.1.1+ds1-3build1_amd64.deb | ||
| dpkg -i containernetworking-plugins_1.1.1+ds1-3build1_amd64.deb | ||
| ``` | ||
|
|
||
| ## Quick Start | ||
|
|
||
| ### 1. Build Presto using Maven | ||
| The container images required for functional tests are integrated into the presto-native-execution and are built under a Maven profile named _docker-build_. | ||
|
|
||
| ```bash | ||
| ./mvnw clean install -DskipTests -Pdocker-build | ||
| ``` | ||
|
|
||
| ### 2. Run functional tests | ||
| #### Using Command Line | ||
| Export the following environment variables: | ||
| ```bash | ||
| export TESTCONTAINERS_RYUK_DISABLED=true | ||
| export DOCKER_HOST=unix:///run/podman/podman.sock | ||
| ``` | ||
| Then, run the functional test using a command similar to this example for `TestPrestoNativeWithContainers`: | ||
| ```bash | ||
| ./mvnw test -pl presto-native-execution -Dtest=com.facebook.presto.nativeworker.TestPrestoNativeWithContainers | ||
| ``` | ||
|
|
||
| #### Using IntelliJ | ||
| Go to the tests with containers at `TestPrestoNativeWithContainers`. | ||
| Edit the run/debug configuration of the test or test case, and add the following as environment variables: | ||
| ``` | ||
| TESTCONTAINERS_RYUK_DISABLED=true | ||
| DOCKER_HOST=unix:///run/podman/podman.sock | ||
| ``` | ||
| Then, run or debug the test. | ||
|
|
||
| #### Note: Existing java and native docker files are reused for functional testing. The coordinator and worker configurations are available in `presto-native-execution/testcontainers`. | ||
|
|
||
|
|
||
|
|
16 changes: 16 additions & 0 deletions
16
presto-native-execution/testcontainers/coordinator/entrypoint.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #!/bin/sh | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -e | ||
|
|
||
| $PRESTO_HOME/bin/launcher run |
9 changes: 9 additions & 0 deletions
9
presto-native-execution/testcontainers/coordinator/etc/catalog/hive.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| connector.name=hive-hadoop2 | ||
| hive.metastore=file | ||
| hive.metastore.catalog.dir=file:/var/lib/presto/dwrf_data/ | ||
| hive.storage-format=DWRF | ||
| hive.pushdown-filter-enabled=true | ||
| hive.compression-codec=SNAPPY | ||
| hive.orc-compression-codec=ZSTD | ||
| hive.allow-drop-table=true | ||
| hive.metastore.authentication.type=NONE |
1 change: 1 addition & 0 deletions
1
presto-native-execution/testcontainers/coordinator/etc/catalog/tpcds.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| connector.name=tpcds |
1 change: 1 addition & 0 deletions
1
presto-native-execution/testcontainers/coordinator/etc/catalog/tpch.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| connector.name=tpch |
8 changes: 8 additions & 0 deletions
8
presto-native-execution/testcontainers/coordinator/etc/config.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| coordinator=true | ||
| presto.version=testversion | ||
| node-scheduler.include-coordinator=false | ||
| http-server.http.port=8081 | ||
| discovery-server.enabled=true | ||
| discovery.uri=http://presto-coordinator:8081 | ||
| native-execution-enabled=true | ||
| list-built-in-functions-only=false |
9 changes: 9 additions & 0 deletions
9
presto-native-execution/testcontainers/coordinator/etc/jvm.config
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| -server | ||
| -Xmx1G | ||
| -XX:+UseG1GC | ||
| -XX:G1HeapRegionSize=32M | ||
| -XX:+UseGCOverheadLimit | ||
| -XX:+ExplicitGCInvokesConcurrent | ||
| -XX:+HeapDumpOnOutOfMemoryError | ||
| -XX:+ExitOnOutOfMemoryError | ||
| -Djdk.attach.allowAttachSelf=true |
1 change: 1 addition & 0 deletions
1
presto-native-execution/testcontainers/coordinator/etc/log.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| com.facebook.presto=DEBUG |
3 changes: 3 additions & 0 deletions
3
presto-native-execution/testcontainers/coordinator/etc/node.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| node.environment=testing | ||
| node.location=testing-location | ||
| node.data-dir=/var/lib/presto/data |
16 changes: 16 additions & 0 deletions
16
presto-native-execution/testcontainers/nativeworker/entrypoint.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #!/bin/sh | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| GLOG_logtostderr=1 presto_server \ | ||
| --etc-dir=/opt/presto-server/etc | ||
|
|
1 change: 1 addition & 0 deletions
1
presto-native-execution/testcontainers/nativeworker/velox-etc/catalog/hive.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| connector.name=hive |
5 changes: 5 additions & 0 deletions
5
presto-native-execution/testcontainers/nativeworker/velox-etc/config.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| presto.version=testversion | ||
| http-server.http.port=7777 | ||
| discovery.uri=http://presto-coordinator:8081 | ||
| system-memory-gb=2 | ||
| native.sidecar=false |
3 changes: 3 additions & 0 deletions
3
presto-native-execution/testcontainers/nativeworker/velox-etc/node.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| node.environment=testing | ||
| node.id=e4901aae-a1c9-4ff7-97a9-5687835ad54c | ||
| node.location=testing-location |
1 change: 1 addition & 0 deletions
1
presto-native-execution/testcontainers/nativeworker/velox-etc/velox.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| mutable-config=true |