From d2e660dae32dfb9ee56102424622cac33059eb2b Mon Sep 17 00:00:00 2001 From: Austin Vazquez Date: Thu, 14 Sep 2023 16:29:29 -0700 Subject: [PATCH] feat: enable running MWAA on Finch Signed-off-by: Austin Vazquez --- mwaa-local-env | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/mwaa-local-env b/mwaa-local-env index 61a4242fa..2fd870bb3 100755 --- a/mwaa-local-env +++ b/mwaa-local-env @@ -3,6 +3,14 @@ AIRFLOW_VERSION=2_6 DOCKER_COMPOSE_PROJECT_NAME=aws-mwaa-local-runner-$AIRFLOW_VERSION +RUNNER=docker +RUNNER_COMPOSE=docker-compose +finch -v >/dev/null 2>&1 +if [ $? -eq 0 ]; then + RUNNER=finch + RUNNER_COMPOSE=finch compose +fi + display_help() { # Display Help echo "======================================" @@ -18,23 +26,23 @@ display_help() { echo "test-requirements Install requirements on an ephemeral instance of the container." echo "package-requirements Download requirements WHL files into plugins folder." echo "test-startup-script Execute shell script on an ephemeral instance of the container." - echo "validate-prereqs Validate pre-reqs installed (docker, docker-compose, python3, pip3)" + echo "validate-prereqs Validate pre-reqs installed ($RUNNER, $RUNNER_COMPOSE, python3, pip3)" echo } validate_prereqs() { - docker -v >/dev/null 2>&1 + $RUNNER -v >/dev/null 2>&1 if [ $? -ne 0 ]; then - echo -e "'docker' is not installed or not runnable without sudo. \xE2\x9D\x8C" + echo -e "'$RUNNER' is not installed or not runnable without sudo. \xE2\x9D\x8C" else - echo -e "Docker is Installed. \xE2\x9C\x94" + echo -e "$RUNNER is Installed. \xE2\x9C\x94" fi - docker-compose -v >/dev/null 2>&1 + $RUNNER_COMPOSE -v >/dev/null 2>&1 if [ $? -ne 0 ]; then - echo -e "'docker-compose' is not installed. \xE2\x9D\x8C" + echo -e "'$RUNNER_COMPOSE' is not installed. \xE2\x9D\x8C" else - echo -e "Docker compose is Installed. \xE2\x9C\x94" + echo -e "$RUNNER_COMPOSE is Installed. \xE2\x9C\x94" fi python3 --version >/dev/null 2>&1 @@ -53,7 +61,7 @@ validate_prereqs() { } build_image() { - docker build --rm --compress -t amazon/mwaa-local:$AIRFLOW_VERSION ./docker + $RUNNER build --rm -t amazon/mwaa-local:$AIRFLOW_VERSION ./docker } case "$1" in @@ -61,43 +69,43 @@ validate-prereqs) validate_prereqs ;; test-requirements) - BUILT_IMAGE=$(docker images -q amazon/mwaa-local:$AIRFLOW_VERSION) + BUILT_IMAGE=$($RUNNER images -q amazon/mwaa-local:$AIRFLOW_VERSION) if [[ -n "$BUILT_IMAGE" ]]; then echo "Container amazon/mwaa-local:$AIRFLOW_VERSION exists. Skipping build" else echo "Container amazon/mwaa-local:$AIRFLOW_VERSION not built. Building locally." build_image fi - docker run -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/plugins:/usr/local/airflow/plugins -v $(pwd)/requirements:/usr/local/airflow/requirements -it amazon/mwaa-local:$AIRFLOW_VERSION test-requirements + $RUNNER run -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/plugins:/usr/local/airflow/plugins -v $(pwd)/requirements:/usr/local/airflow/requirements -it amazon/mwaa-local:$AIRFLOW_VERSION test-requirements ;; test-startup-script) - BUILT_IMAGE=$(docker images -q amazon/mwaa-local:$AIRFLOW_VERSION) + BUILT_IMAGE=$($RUNNER images -q amazon/mwaa-local:$AIRFLOW_VERSION) if [[ -n "$BUILT_IMAGE" ]]; then echo "Container amazon/mwaa-local:$AIRFLOW_VERSION exists. Skipping build" else echo "Container amazon/mwaa-local:$AIRFLOW_VERSION not built. Building locally." build_image fi - docker run -v $(pwd)/startup_script:/usr/local/airflow/startup -it amazon/mwaa-local:$AIRFLOW_VERSION test-startup-script + $RUNNER run -v $(pwd)/startup_script:/usr/local/airflow/startup -it amazon/mwaa-local:$AIRFLOW_VERSION test-startup-script ;; package-requirements) - BUILT_IMAGE=$(docker images -q amazon/mwaa-local:$AIRFLOW_VERSION) + BUILT_IMAGE=$($RUNNER images -q amazon/mwaa-local:$AIRFLOW_VERSION) if [[ -n "$BUILT_IMAGE" ]]; then echo "Container amazon/mwaa-local:$AIRFLOW_VERSION exists. Skipping build" else echo "Container amazon/mwaa-local:$AIRFLOW_VERSION not built. Building locally." build_image fi - docker run -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/plugins:/usr/local/airflow/plugins -v $(pwd)/requirements:/usr/local/airflow/requirements -it amazon/mwaa-local:$AIRFLOW_VERSION package-requirements + $RUNNER run -v $(pwd)/dags:/usr/local/airflow/dags -v $(pwd)/plugins:/usr/local/airflow/plugins -v $(pwd)/requirements:/usr/local/airflow/requirements -it amazon/mwaa-local:$AIRFLOW_VERSION package-requirements ;; build-image) build_image ;; reset-db) - docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME -f ./docker/docker-compose-resetdb.yml up --abort-on-container-exit + $RUNNER_COMPOSE -p $DOCKER_COMPOSE_PROJECT_NAME -f ./docker/docker-compose-resetdb.yml up ;; start) - docker-compose -p $DOCKER_COMPOSE_PROJECT_NAME -f ./docker/docker-compose-local.yml up + $RUNNER_COMPOSE -p $DOCKER_COMPOSE_PROJECT_NAME -f ./docker/docker-compose-local.yml up ;; help) display_help