SNOW-862917: Trigger lambda tests on GHA

.github/workflows/build_test.yml

@@ -250,10 +250,51 @@ jobs:
             .coverage
             coverage.xml
 
+  test-lambda:
+    name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        cloud-provider: [aws]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup parameters file
+        shell: bash
+        env:
+          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
+        run: |
+          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
+          .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v3
+        with:
+          name: manylinux_x86_64_py${{ matrix.python-version }}
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Run tests
+        run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION}
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+      - uses: actions/upload-artifact@v3
+        with:
+          name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+          path: |
+            .coverage
+            coverage.xml
+
   combine-coverage:
     if: ${{ success() || failure() }}
     name: Combine coverage
-    needs: [lint, test, test-fips]
+    needs: [lint, test, test-fips, test-lambda]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

Jenkinsfile

@@ -47,12 +47,10 @@ timestamps {
           println("Exception computing commit hash from: ${response}")
           }
         parallel (
-          'Test Python 37': { build job: 'RT-PyConnector37-PC',parameters: params},
           'Test Python 38': { build job: 'RT-PyConnector38-PC',parameters: params},
           'Test Python 39': { build job: 'RT-PyConnector39-PC',parameters: params},
           'Test Python 310': { build job: 'RT-PyConnector310-PC',parameters: params},
           'Test Python 311': { build job: 'RT-PyConnector311-PC',parameters: params},
-          'Test Python Lambda 37': { build job: 'RT-PyConnector37-PC-Lambda',parameters: params}
           )
         }
       }

ci/docker/connector_test_lambda/Dockerfile310

@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.10-x86_64
+
+RUN yum install -y git
+
+WORKDIR /home/user/snowflake-connector-python
+RUN chmod 777 /home/user/snowflake-connector-python
+ENV PATH="${PATH}:/opt/python/cp310-cp310/bin/"
+ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
+
+RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+
+CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile311

@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.11-x86_64
+
+RUN yum install -y git
+
+WORKDIR /home/user/snowflake-connector-python
+RUN chmod 777 /home/user/snowflake-connector-python
+ENV PATH="${PATH}:/opt/python/cp311-cp311/bin/"
+ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
+
+RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+
+CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile38

@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.8-x86_64
+
+RUN yum install -y git
+
+WORKDIR /home/user/snowflake-connector-python
+RUN chmod 777 /home/user/snowflake-connector-python
+ENV PATH="${PATH}:/opt/python/cp38-cp38/bin/"
+ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
+
+RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+
+CMD [ "app.handler" ]

ci/docker/connector_test_lambda/Dockerfile39

@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.9-x86_64
+
+RUN yum install -y git
+
+WORKDIR /home/user/snowflake-connector-python
+RUN chmod 777 /home/user/snowflake-connector-python
+ENV PATH="${PATH}:/opt/python/cp39-cp39/bin/"
+ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
+
+RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+
+CMD [ "app.handler" ]

ci/docker/connector_test_lambda/app.py

@@ -0,0 +1,81 @@
+import logging
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from subprocess import PIPE, Popen
+
+LOGGER = logging.getLogger(__name__)
+REPO_PATH = "/home/user/snowflake-connector-python"
+PY_SHORT_VER = f"{sys.version_info[0]}{sys.version_info[1]}"  # 38, 39, 310, 311
+ARCH = "x86"  # x86, aarch64
+
+
+def run_tests():
+    """Run tests using tox"""
+    LOGGER.info("Running tests..")
+    test_log, err = Popen(
+        [
+            "python",
+            "-m",
+            "tox",
+            "-e",
+            f"py{PY_SHORT_VER}{{-lambda}}-ci",
+            "-c",
+            f"{REPO_PATH}/tox.ini",
+            "--workdir",
+            REPO_PATH,
+            "--external_wheels",
+            f"{REPO_PATH}/dist/*.whl",
+        ],
+        stdout=PIPE,
+        stderr=PIPE,
+    ).communicate()
+    LOGGER.info(test_log)
+    LOGGER.info(err)
+    return test_log.decode("utf-8")
+
+
+def parse_test_xml_output():
+    """Parse test summary from xml report generated"""
+    LOGGER.info("Parsing test result output")
+    test_status = "UNKNOWN"
+
+    default_xml_fp = f"{REPO_PATH}/junit.py{PY_SHORT_VER}-lambda-ci-dev.xml"
+    files = sorted(Path(REPO_PATH).glob(f"junit.py{PY_SHORT_VER}-lambda-ci-*.xml"))
+    file_path = files[0].as_posix() if files else default_xml_fp
+
+    try:
+        root = ET.parse(file_path).getroot()
+        for child in root:
+            failure_count = child.attrib.get("failures")
+    except Exception as ex:
+        LOGGER.exception(ex)
+        return test_status
+
+    if int(failure_count) == 0:
+        test_status = "SUCCESS"
+    else:
+        test_status = "FAILURE"
+    return test_status
+
+
+def handler(events, context):
+    """
+    Lambda handler for testing Python Connector.
+    """
+    test_status = "UNKNOWN"
+    test_result_log = "N/A"
+    response = {}
+    response["statusCode"] = 500
+    response["testStatus"] = test_status
+
+    # run tests
+    test_result_log = run_tests()
+
+    # parse result output
+    test_status = parse_test_xml_output()
+
+    response["statusCode"] = 200
+    response["testStatus"] = test_status
+    response["testLog"] = test_result_log
+    return response

ci/test_lambda_docker.sh

@@ -0,0 +1,56 @@
+#!/bin/bash -x
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CONNECTOR_DIR="$( dirname "${THIS_DIR}")"
+PYTHON_VERSION="${1:-3.8}"
+PYTHON_SHORT_VERSION="$(echo "$PYTHON_VERSION" | tr -d .)"
+# In case this is not run locally and not on Jenkins
+
+if [[ ! -d "$CONNECTOR_DIR/dist/" ]] || [[ $(ls $CONNECTOR_DIR/dist/*cp${PYTHON_SHORT_VERSION}*manylinux2014*.whl) == '' ]]; then
+  echo "Missing wheel files, going to compile Python connector in Docker..."
+  $THIS_DIR/build_docker.sh $PYTHON_VERSION
+  cp $CONNECTOR_DIR/dist/repaired_wheels/*cp${PYTHON_SHORT_VERSION}*manylinux2014*.whl $CONNECTOR_DIR/dist/
+fi
+
+cd $THIS_DIR/docker/connector_test_lambda
+
+CONTAINER_NAME="test_lambda_connector${PYTHON_SHORT_VERSION}"
+DOCKERFILE="Dockerfile${PYTHON_SHORT_VERSION}"
+
+echo "[Info] Start building lambda docker image"
+docker build -t ${CONTAINER_NAME}:1.0 -f ${DOCKERFILE} .
+
+user_id=$(id -u $USER)
+
+docker run --network=host \
+    -e LANG=en_US.UTF-8 \
+    -e TERM=vt102 \
+    -e SF_USE_OPENSSL_ONLY=True \
+    -e PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    -e LOCAL_USER_ID=$user_id \
+    -e CRYPTOGRAPHY_ALLOW_OPENSSL_102=1 \
+    -e AWS_ACCESS_KEY_ID \
+    -e AWS_SECRET_ACCESS_KEY \
+    -e SF_REGRESS_LOGS \
+    -e SF_PROJECT_ROOT \
+    -e cloud_provider \
+    -e PYTEST_ADDOPTS \
+    -e GITHUB_ACTIONS \
+    --mount type=bind,source="${CONNECTOR_DIR}",target=/home/user/snowflake-connector-python \
+    ${CONTAINER_NAME}:1.0 &
+
+# sleep for sometime to make sure docker run is up and running
+sleep 5
+
+# call the lambda function
+lambda_result=$(curl -XPOST "http://localhost:8080/2015-03-31/functions/function/invocations" -d '{}')
+echo "Lambda result:$lambda_result"
+
+# stop all docker processes
+docker stop $(docker ps -a -q)
+
+# reflect status of the test on the job
+status=$(echo "$lambda_result" | grep SUCCESS)
+if [[ -z "$status" ]]; then
+    exit 1
+fi