From d4c65f3c31757d2d565a188e509ad191d296f056 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 19 Nov 2024 10:59:17 -0500 Subject: [PATCH] Add romanisim to downstream --- .github/workflows/downstream.yml | 25 +++++++++++++++ .github/workflows/get_webbpsf.yml | 31 ++++++++++++++++++ .github/workflows/webbpsf_data.yml | 51 ++++++++++++++++++++++++++++++ tox.ini | 7 ++++ 4 files changed, 114 insertions(+) create mode 100644 .github/workflows/get_webbpsf.yml create mode 100644 .github/workflows/webbpsf_data.yml diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index f15f1ec1..cda547b8 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -39,6 +39,7 @@ jobs: crds_contexts: uses: ./.github/workflows/contexts.yml + jwst: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 needs: [ environment, crds_contexts ] @@ -67,6 +68,30 @@ jobs: envs: | - linux: py311-test-romancal-cov-xdist + get_romanism_data: + uses: ./.github/workflows/webbpsf_data.yml + + romanisim_data_cache: + needs: [ get_romanism_data ] + uses: ./.github/workflows/get_webbpsf.yml + + romanisim: + needs: [ romanisim_data_cache ] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + with: + libraries: | + brew: + - eigen + - fftw + setenv: | + WEBBPSF_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}webbpsf-data/ + GALSIM_CAT_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits + FFTW_DIR: /opt/homebrew/opt/fftw/lib/ + cache-path: ${{ needs.romanisim_data_cache.outputs.cache_path }} + cache-key: ${{ needs.romanisim_data_cache.outputs.cache_key }} + envs: | + - linux: py311-test-romanisim-cov-xdist + astropy: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 if: (github.repository == 'spacetelescope/gwcs' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Downstream CI'))) diff --git a/.github/workflows/get_webbpsf.yml b/.github/workflows/get_webbpsf.yml new file mode 100644 index 00000000..0b7d7c7c --- /dev/null +++ b/.github/workflows/get_webbpsf.yml @@ -0,0 +1,31 @@ +name: retrieve latest data cache key + +on: + workflow_call: + outputs: + cache_key: + value: ${{ jobs.retrieve_cache.outputs.cache_key }} + cache_path: + value: ${{ jobs.retrieve_cache.outputs.cache_path }} + +jobs: + retrieve_cache: + name: retrieve latest data cache key + runs-on: ubuntu-latest + steps: + - name: retrieve latest data cache key + id: latest_cache_key + run: | + # use actions/gh-actions-cache to allow filtering by key + gh extension install actions/gh-actions-cache + CACHE_KEY=$(gh actions-cache list -R ${{ github.repository }} --key data- --sort created-at | cut -f 1 | head -n 1) + if [ "$CACHE_KEY" == '' ]; then exit 1; fi + echo cache_key=$CACHE_KEY >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + - run: echo cache_key=${{ steps.latest_cache_key.outputs.cache_key }} + - id: cache_path + run: echo cache_path=/tmp/data/ >> $GITHUB_OUTPUT + outputs: + cache_key: ${{ steps.latest_cache_key.outputs.cache_key }} + cache_path: ${{ steps.cache_path.outputs.cache_path }} diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml new file mode 100644 index 00000000..9d45e9d5 --- /dev/null +++ b/.github/workflows/webbpsf_data.yml @@ -0,0 +1,51 @@ +name: download and cache data + +on: + workflow_call: + schedule: + - cron: "42 4 * * 3" + workflow_dispatch: + inputs: + webbpsf_minimal: + description: minimal WebbPSF dataset + type: boolean + required: false + default: true + +jobs: + download_webbpsf_data: + uses: spacetelescope/webbpsf/.github/workflows/download_data.yml@beda656c80a0254e6f80649d9c9c49235634522f # v1.4.0 + with: + minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} + combine_data_cache: + needs: [ download_webbpsf_data ] + runs-on: ubuntu-latest + env: + GALSIM_DATA_URL: https://github.com/GalSim-developers/GalSim/raw/releases/2.4/examples/data/ + steps: + - name: download GalSim data + run: | + mkdir galsim_data + wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example.fits -O galsim_data/real_galaxy_catalog_23.5_example.fits + wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_selection.fits -O galsim_data/real_galaxy_catalog_23.5_example_selection.fits + wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_fits.fits -O galsim_data/real_galaxy_catalog_23.5_example_fits.fits + - id: galsim_data + run: echo "hash=${{ hashFiles( 'galsim_data/' ) }}" >> $GITHUB_OUTPUT + - run: echo GALSIM_PATH=/tmp/data/galsim_data/ >> $GITHUB_ENV + - run: | + mkdir -p ${{ env.GALSIM_PATH }} + mv ./galsim_data/* ${{ env.GALSIM_PATH }} + - run: echo GALSIM_CAT_PATH=${{ env.GALSIM_PATH }}real_galaxy_catalog_23.5_example.fits >> $GITHUB_ENV + - name: retrieve cached WebbPSF data + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: ${{ needs.download_webbpsf_data.outputs.cache_path }} + key: ${{ needs.download_webbpsf_data.outputs.cache_key }} + - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ /tmp/data/ + - run: echo WEBBPSF_PATH=/tmp/data/webbpsf-data/ >> $GITHUB_ENV + # save a new cache to the same generalized data directory, combined with extra data + - name: save a single combined data cache + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: /tmp/data/ + key: data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }} diff --git a/tox.ini b/tox.ini index f5e2ae74..52500f21 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ description = run tests jwst: of JWST pipeline romancal: of Romancal pipeline + romanisim: of Romanisim image simulation dev: with the latest developer version of key dependencies pyargs: with --pyargs on installed package warnings: treating warnings as errors @@ -57,6 +58,10 @@ passenv = CI CODECOV_* DISPLAY + WEBBPSF_PATH + GALSIM_CAT_PATH + FFTW_DIR + LIBRARY_PATH set_env = dev: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple @@ -70,6 +75,7 @@ deps = cov: pytest-cov jwst: jwst[test] @ git+https://github.com/spacetelescope/jwst.git romancal: romancal[test] @ git+https://github.com/spacetelescope/romancal.git + romanisim: romanisim[test] @ git+https://github.com/spacetelescope/romanisim.git numpy123: numpy==1.23.* numpy125: numpy==1.25.* pass_env = @@ -84,6 +90,7 @@ commands = pyargs: {toxinidir}/docs --pyargs gwcs \ jwst: --pyargs jwst --ignore-glob=timeconversion --ignore-glob=associations --ignore-glob=scripts --show-capture=no \ romancal: --pyargs romancal \ + romanisim: --pyargs romanisim \ cov: --cov=. --cov-config=pyproject.toml --cov-report=term-missing --cov-report=xml \ {posargs}