Skip to content

Commit

Permalink
Test
Browse files Browse the repository at this point in the history
  • Loading branch information
@robbibt
robbibt authored Aug 29, 2024
1 parent ceae2d5 commit 578dc8d
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 90 deletions.
130 changes: 92 additions & 38 deletions .github/workflows/test_changed_notebooks.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,101 @@ on:
pull_request:
branches: [ develop ]

permissions:
id-token: write # Required for OIDC authentication to AWS
contents: read

jobs:
test_files:
runs-on: ubuntu-latest

# Automatically activate our mamba environment when it's ready
defaults:
run:
shell: bash -l {0}

steps:
- uses: actions/checkout@v3
# with:
# fetch-depth: 0

# - name: Set up Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.x'

- name: Get changed notebook files
id: changed-notebooks
uses: tj-actions/changed-files@v44
with:
files: '**/*.ipynb'
separator: ' '

- name: Print changed notebook files
if: steps.changed-notebooks.outputs.any_changed == 'true'
run: |
echo "Changed notebook files:"
echo "${{ steps.changed-notebooks.outputs.all_changed_files }}"
- name: Get changed Python files
id: changed-py-files
uses: tj-actions/changed-files@v44
with:
files: '**/*.py'
separator: ' '

- name: Print changed Python files
if: steps.changed-py-files.outputs.any_changed == 'true'
run: |
echo "Changed Python files:"
echo "${{ steps.changed-py-files.outputs.all_changed_files }}"
- uses: actions/checkout@v4
with:
path: dea-notebooks

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly
aws-region: ap-southeast-2
role-duration-seconds: 7200 # 2 hours

- name: Copy tide modelling files with the AWS CLI
run: aws s3 sync s3://dea-non-public-data/tide_models/tide_models tide_models

- name: Login to Amazon ECR Private
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2

- name: Pull dea-sandbox image from ECR
run: |
docker pull 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox
aws-region: ap-southeast-2
role-duration-seconds: 7200 # 2 hours

- name: Get Database Credentials
run: |
username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text)
echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV
- name: Open Port Forward to RDS
run: |
npx basti connect \
--custom-target-vpc vpc-086904199e505c1f6 \
--custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \
--custom-target-port 5432 \
--local-port 5432 &
npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432
echo "PGPORT=5432" >> $GITHUB_ENV
echo "PGHOST=localhost" >> $GITHUB_ENV
- name: Get changed notebook files
id: changed-notebooks
uses: tj-actions/changed-files@v44
with:
files: '**/*.ipynb'
separator: ' '

# - name: Run pytest on Python files
# if: steps.changed-py-files.outputs.any_changed == 'true'
# run: |
# pytest Tests/dea_tools
- name: Print changed notebook files
if: steps.changed-notebooks.outputs.any_changed == 'true'
run: |
echo "Changed notebook files:"
echo "${{ steps.changed-notebooks.outputs.all_changed_files }}"
- name: Run the Notebook tests
if: steps.changed-notebooks.outputs.any_changed == 'true'
run: |
sudo chown -R 1000:100 ./dea-notebooks
cd ./dea-notebooks
docker run --rm \
--net=host \
--env DATACUBE_DB_URL \
--env AWS_SESSION_TOKEN \
--env AWS_REGION \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env AWS_SESSION_TOKEN \
--volume ${GITHUB_WORKSPACE}/dea-notebooks:/home/jovyan/dea-notebooks \
--volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \
--env GDAL_HTTP_MAX_RETRY=3 \
--entrypoint /bin/bash \
538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable \
-c "
set -ex
set -o pipefail
cd /home/jovyan/dea-notebooks
pip3 install ./Tools
pytest Tests/dea_tools
pytest --durations=10 --nbval-lax ${{ steps.changed-notebooks.outputs.all_changed_files }}
"
104 changes: 52 additions & 52 deletions .github/workflows/test_notebooks.yml
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,53 @@

name: Test notebooks

on:
push:
branches: [ develop, stable, nbtests ]
paths-ignore:
- '**/*.md' # ignore markdown files
- '**/*.rst' # ignore restructured text files
- '.github/**' # ignore anything in .github folder
- '!.github/workflows/test_notebooks.yml' # except test_notebooks.yml
pull_request:
branches: [ develop, stable ]
paths-ignore:
- '**/*.md'
- '**/*.rst'
- '.github/**'
- '!.github/workflows/test_notebooks.yml'

permissions:
id-token: write # This is required for requesting the JSON web token
contents: read # This is required for actions/checkout

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
path: dea-notebooks

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly
aws-region: ap-southeast-2

- name: Copy tide modelling files with the AWS CLI
run: |
aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014
aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide
- name: Login to Amazon ECR Private
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1

- name: Set up Datacube and test
run: |
sudo chown -R 1000:100 ./dea-notebooks
cd ./dea-notebooks
CURRENT_UID=1000:100 docker-compose up -d
docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh
docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh
# name: Test notebooks

# on:
# push:
# branches: [ develop, stable, nbtests ]
# paths-ignore:
# - '**/*.md' # ignore markdown files
# - '**/*.rst' # ignore restructured text files
# - '.github/**' # ignore anything in .github folder
# - '!.github/workflows/test_notebooks.yml' # except test_notebooks.yml
# pull_request:
# branches: [ develop, stable ]
# paths-ignore:
# - '**/*.md'
# - '**/*.rst'
# - '.github/**'
# - '!.github/workflows/test_notebooks.yml'

# permissions:
# id-token: write # This is required for requesting the JSON web token
# contents: read # This is required for actions/checkout

# jobs:
# test:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# with:
# path: dea-notebooks

# - name: Configure AWS credentials
# uses: aws-actions/configure-aws-credentials@v1
# with:
# role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly
# aws-region: ap-southeast-2

# - name: Copy tide modelling files with the AWS CLI
# run: |
# aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014
# aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide

# - name: Login to Amazon ECR Private
# id: login-ecr
# uses: aws-actions/amazon-ecr-login@v1

# - name: Set up Datacube and test
# run: |
# sudo chown -R 1000:100 ./dea-notebooks
# cd ./dea-notebooks
# CURRENT_UID=1000:100 docker-compose up -d
# docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh
# docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh

0 comments on commit 578dc8d

Please sign in to comment.