diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 3e406f1df..7ab4df5ee 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -250,10 +250,51 @@ jobs: .coverage coverage.xml + test-lambda: + name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }} + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + cloud-provider: [aws] + steps: + - uses: actions/checkout@v3 + - name: Setup parameters file + shell: bash + env: + PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} + run: | + gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ + .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py + - name: Download wheel(s) + uses: actions/download-artifact@v3 + with: + name: manylinux_x86_64_py${{ matrix.python-version }} + path: dist + - name: Show wheels downloaded + run: ls -lh dist + shell: bash + - name: Run tests + run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION} + env: + PYTHON_VERSION: ${{ matrix.python-version }} + cloud_provider: ${{ matrix.cloud-provider }} + PYTEST_ADDOPTS: --color=yes --tb=short + TOX_PARALLEL_NO_SPINNER: 1 + shell: bash + - uses: actions/upload-artifact@v3 + with: + name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }} + path: | + .coverage + coverage.xml + combine-coverage: if: ${{ success() || failure() }} name: Combine coverage - needs: [lint, test, test-fips] + needs: [lint, test, test-fips, test-lambda] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/Jenkinsfile b/Jenkinsfile index a92dd75ac..3488b6f8c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -47,12 +47,10 @@ timestamps { println("Exception computing commit hash from: ${response}") } parallel ( - 'Test Python 37': { build job: 'RT-PyConnector37-PC',parameters: params}, 'Test Python 38': { build job: 'RT-PyConnector38-PC',parameters: params}, 'Test Python 39': { build job: 'RT-PyConnector39-PC',parameters: params}, 'Test Python 310': { build job: 'RT-PyConnector310-PC',parameters: params}, 'Test Python 311': { build job: 'RT-PyConnector311-PC',parameters: params}, - 'Test Python Lambda 37': { build job: 'RT-PyConnector37-PC-Lambda',parameters: params} ) } } diff --git a/ci/docker/connector_test_lambda/Dockerfile310 b/ci/docker/connector_test_lambda/Dockerfile310 new file mode 100644 index 000000000..759f121ba --- /dev/null +++ b/ci/docker/connector_test_lambda/Dockerfile310 @@ -0,0 +1,12 @@ +FROM public.ecr.aws/lambda/python:3.10-x86_64 + +RUN yum install -y git + +WORKDIR /home/user/snowflake-connector-python +RUN chmod 777 /home/user/snowflake-connector-python +ENV PATH="${PATH}:/opt/python/cp310-cp310/bin/" +ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/" + +RUN pip3 install -U pip setuptools wheel tox tox-external_wheels + +CMD [ "app.handler" ] diff --git a/ci/docker/connector_test_lambda/Dockerfile311 b/ci/docker/connector_test_lambda/Dockerfile311 new file mode 100644 index 000000000..d35115607 --- /dev/null +++ b/ci/docker/connector_test_lambda/Dockerfile311 @@ -0,0 +1,12 @@ +FROM public.ecr.aws/lambda/python:3.11-x86_64 + +RUN yum install -y git + +WORKDIR /home/user/snowflake-connector-python +RUN chmod 777 /home/user/snowflake-connector-python +ENV PATH="${PATH}:/opt/python/cp311-cp311/bin/" +ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/" + +RUN pip3 install -U pip setuptools wheel tox tox-external_wheels + +CMD [ "app.handler" ] diff --git a/ci/docker/connector_test_lambda/Dockerfile38 b/ci/docker/connector_test_lambda/Dockerfile38 new file mode 100644 index 000000000..73521fa44 --- /dev/null +++ b/ci/docker/connector_test_lambda/Dockerfile38 @@ -0,0 +1,12 @@ +FROM public.ecr.aws/lambda/python:3.8-x86_64 + +RUN yum install -y git + +WORKDIR /home/user/snowflake-connector-python +RUN chmod 777 /home/user/snowflake-connector-python +ENV PATH="${PATH}:/opt/python/cp38-cp38/bin/" +ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/" + +RUN pip3 install -U pip setuptools wheel tox tox-external_wheels + +CMD [ "app.handler" ] diff --git a/ci/docker/connector_test_lambda/Dockerfile39 b/ci/docker/connector_test_lambda/Dockerfile39 new file mode 100644 index 000000000..101992262 --- /dev/null +++ b/ci/docker/connector_test_lambda/Dockerfile39 @@ -0,0 +1,12 @@ +FROM public.ecr.aws/lambda/python:3.9-x86_64 + +RUN yum install -y git + +WORKDIR /home/user/snowflake-connector-python +RUN chmod 777 /home/user/snowflake-connector-python +ENV PATH="${PATH}:/opt/python/cp39-cp39/bin/" +ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/" + +RUN pip3 install -U pip setuptools wheel tox tox-external_wheels + +CMD [ "app.handler" ] diff --git a/ci/docker/connector_test_lambda/app.py b/ci/docker/connector_test_lambda/app.py new file mode 100644 index 000000000..c52f756f8 --- /dev/null +++ b/ci/docker/connector_test_lambda/app.py @@ -0,0 +1,81 @@ +import logging +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from subprocess import PIPE, Popen + +LOGGER = logging.getLogger(__name__) +REPO_PATH = "/home/user/snowflake-connector-python" +PY_SHORT_VER = f"{sys.version_info[0]}{sys.version_info[1]}" # 38, 39, 310, 311 +ARCH = "x86" # x86, aarch64 + + +def run_tests(): + """Run tests using tox""" + LOGGER.info("Running tests..") + test_log, err = Popen( + [ + "python", + "-m", + "tox", + "-e", + f"py{PY_SHORT_VER}{{-lambda}}-ci", + "-c", + f"{REPO_PATH}/tox.ini", + "--workdir", + REPO_PATH, + "--external_wheels", + f"{REPO_PATH}/dist/*.whl", + ], + stdout=PIPE, + stderr=PIPE, + ).communicate() + LOGGER.info(test_log) + LOGGER.info(err) + return test_log.decode("utf-8") + + +def parse_test_xml_output(): + """Parse test summary from xml report generated""" + LOGGER.info("Parsing test result output") + test_status = "UNKNOWN" + + default_xml_fp = f"{REPO_PATH}/junit.py{PY_SHORT_VER}-lambda-ci-dev.xml" + files = sorted(Path(REPO_PATH).glob(f"junit.py{PY_SHORT_VER}-lambda-ci-*.xml")) + file_path = files[0].as_posix() if files else default_xml_fp + + try: + root = ET.parse(file_path).getroot() + for child in root: + failure_count = child.attrib.get("failures") + except Exception as ex: + LOGGER.exception(ex) + return test_status + + if int(failure_count) == 0: + test_status = "SUCCESS" + else: + test_status = "FAILURE" + return test_status + + +def handler(events, context): + """ + Lambda handler for testing Python Connector. + """ + test_status = "UNKNOWN" + test_result_log = "N/A" + response = {} + response["statusCode"] = 500 + response["testStatus"] = test_status + + # run tests + test_result_log = run_tests() + + # parse result output + test_status = parse_test_xml_output() + + response["statusCode"] = 200 + response["testStatus"] = test_status + response["testLog"] = test_result_log + return response diff --git a/ci/test_lambda_docker.sh b/ci/test_lambda_docker.sh new file mode 100755 index 000000000..34f2d4454 --- /dev/null +++ b/ci/test_lambda_docker.sh @@ -0,0 +1,56 @@ +#!/bin/bash -x + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CONNECTOR_DIR="$( dirname "${THIS_DIR}")" +PYTHON_VERSION="${1:-3.8}" +PYTHON_SHORT_VERSION="$(echo "$PYTHON_VERSION" | tr -d .)" +# In case this is not run locally and not on Jenkins + +if [[ ! -d "$CONNECTOR_DIR/dist/" ]] || [[ $(ls $CONNECTOR_DIR/dist/*cp${PYTHON_SHORT_VERSION}*manylinux2014*.whl) == '' ]]; then + echo "Missing wheel files, going to compile Python connector in Docker..." + $THIS_DIR/build_docker.sh $PYTHON_VERSION + cp $CONNECTOR_DIR/dist/repaired_wheels/*cp${PYTHON_SHORT_VERSION}*manylinux2014*.whl $CONNECTOR_DIR/dist/ +fi + +cd $THIS_DIR/docker/connector_test_lambda + +CONTAINER_NAME="test_lambda_connector${PYTHON_SHORT_VERSION}" +DOCKERFILE="Dockerfile${PYTHON_SHORT_VERSION}" + +echo "[Info] Start building lambda docker image" +docker build -t ${CONTAINER_NAME}:1.0 -f ${DOCKERFILE} . + +user_id=$(id -u $USER) + +docker run --network=host \ + -e LANG=en_US.UTF-8 \ + -e TERM=vt102 \ + -e SF_USE_OPENSSL_ONLY=True \ + -e PIP_DISABLE_PIP_VERSION_CHECK=1 \ + -e LOCAL_USER_ID=$user_id \ + -e CRYPTOGRAPHY_ALLOW_OPENSSL_102=1 \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e SF_REGRESS_LOGS \ + -e SF_PROJECT_ROOT \ + -e cloud_provider \ + -e PYTEST_ADDOPTS \ + -e GITHUB_ACTIONS \ + --mount type=bind,source="${CONNECTOR_DIR}",target=/home/user/snowflake-connector-python \ + ${CONTAINER_NAME}:1.0 & + +# sleep for sometime to make sure docker run is up and running +sleep 5 + +# call the lambda function +lambda_result=$(curl -XPOST "http://localhost:8080/2015-03-31/functions/function/invocations" -d '{}') +echo "Lambda result:$lambda_result" + +# stop all docker processes +docker stop $(docker ps -a -q) + +# reflect status of the test on the job +status=$(echo "$lambda_result" | grep SUCCESS) +if [[ -z "$status" ]]; then + exit 1 +fi