justfile

set dotenv-load := false

COLOR := "\\033[0;34m"
NO_COLOR := "\\033[0m"

# Show all available recipes
@_default:
    printf "\n{{ COLOR }}# Catalog (path: \`catalog/\`)\n"
    printf "============================{{ NO_COLOR }}\n"
    just --list --unsorted


SERVICE := env_var_or_default("SERVICE", "scheduler")

###########
# Version #
###########

export CATALOG_PY_VERSION := `grep '# PYTHON' requirements_prod.txt | awk -F= '{print $2}'`
export CATALOG_AIRFLOW_VERSION := `grep '^apache-airflow' requirements_prod.txt | awk -F= '{print $3}'`

# Print the required Python version
@py-version:
    echo $CATALOG_PY_VERSION

# Print the current Airflow version
@airflow-version:
    echo $CATALOG_AIRFLOW_VERSION

###########
# Install #
###########

# Create a virtual environment using the project Python version
venv:
    # Invokes `python<version>`, like `python3.10`
    bash -c "python$CATALOG_PY_VERSION -m venv venv/"

# Check that the active Python version matches the required Python version
check-py-version:
    #!/usr/bin/env python
    import os
    import sys
    current = f"{sys.version_info[0]}.{sys.version_info[1]}"
    required = os.environ.get("CATALOG_PY_VERSION")
    if current != required:
        print(f"Detected Python version {current} but required {required}", file=sys.stderr)
        sys.exit(1)

# Install dependencies
install *args: check-py-version
    python -m pip install -r requirements_dev.txt

######
# Up #
######

# Bring up services specific to the catalog profile
up *flags:
    env COMPOSE_PROFILES="catalog" just ../up {{ flags }}

# Bring up services specific to the catalog profile, except Airflow
up-deps *flags:
    env COMPOSE_PROFILES="catalog_dependencies" just ../up {{ flags }}

# Load sample data into upstream DB
init: up
    cd .. && ./load_sample_data.sh -c

# Tear down all the services and recreate the ones of the catalog profile
recreate:
    just ../down -v
    just up --force-recreate --build
    just init

##################
# Administration #
##################

# Launch a Bash shell in an existing container under `SERVICE`
shell:
    env DC_USER="airflow" just ../exec {{ SERVICE }} /bin/bash

# Launch an IPython shell in a new container under `SERVICE`
ipython: up-deps
    env DC_USER="airflow" just ../run \
        --rm \
        --workdir /opt/airflow/catalog/dags \
        {{ SERVICE }} \
        bash -c \'ipython\'

# Launch a `pgcli` shell in the PostgreSQL container
pgcli db_user_pass="deploy" db_name="openledger": up
    just ../_pgcli upstream_db {{ db_user_pass }} {{ db_name }} upstream_db

#########
# Tests #
#########

# Run a command in a test container under `SERVICE`
_mount-test command: up-deps
    env DC_USER="airflow" just ../run \
        --rm \
        -e AIRFLOW_VAR_INGESTION_LIMIT=1000000 \
        -w /opt/airflow/catalog \
        --volume {{ justfile_directory() }}/../docker:/opt/airflow/docker/ \
        {{ SERVICE }} \
        {{ command }}

# Launch a Bash shell in a test container under `SERVICE`
test-session:
    just _mount-test bash

# Run pytest in a test container under `SERVICE`
test *args:
    just _mount-test "bash -c \'pytest {{ args }}\'"

#############
# Utilities #
#############

# Generate the DAG documentation
generate-dag-docs fail_on_diff="false":
    #!/bin/bash
    set -e
    just ../run \
      {{ SERVICE }} \
      "bash -c 'python catalog/utilities/dag_doc_gen/dag_doc_generation.py && chmod 666 /opt/airflow/catalog/utilities/dag_doc_gen/DAGs.md'"
    # Move the file to the top level, since that level is not mounted into the container
    mv utilities/dag_doc_gen/DAGs.md DAGs.md
    echo -n "Running linting..."
    # Linting step afterwards is necessary since the generated output differs greatly from what prettier expects
    just ../lint prettier catalog/DAGs.md &>/dev/null || true
    echo "Done!"
    if {{ fail_on_diff }}; then
      set +e
      git diff --exit-code DAGs.md
      if [ $? -ne 0 ]; then
          printf "\n\n\e[31m!! Changes found in DAG documentation, please run 'just generate-dag-docs' locally and commit difference !!\n\n"
          exit 1
      fi
    fi

# Generate files for a new provider
add-provider provider_name endpoint +media_types="image":
    python3 templates/create_provider_ingester.py "{{ provider_name }}" "{{ endpoint }}" -m {{ media_types }}