Add romanisim to downstream

spacetelescope · Nov 19, 2024 · d4c65f3 · d4c65f3
1 parent 7d874c3
commit d4c65f3
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 0 deletions.
diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml
@@ -39,6 +39,7 @@ jobs:
   crds_contexts:
     uses: ./.github/workflows/contexts.yml
 
+
   jwst:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
     needs: [ environment, crds_contexts ]
@@ -67,6 +68,30 @@ jobs:
       envs: |
         - linux: py311-test-romancal-cov-xdist
 
+  get_romanism_data:
+    uses: ./.github/workflows/webbpsf_data.yml
+
+  romanisim_data_cache:
+    needs: [ get_romanism_data ]
+    uses: ./.github/workflows/get_webbpsf.yml
+
+  romanisim:
+    needs: [ romanisim_data_cache ]
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
+    with:
+      libraries: |
+        brew:
+          - eigen
+          - fftw
+      setenv: |
+        WEBBPSF_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}webbpsf-data/
+        GALSIM_CAT_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits
+        FFTW_DIR: /opt/homebrew/opt/fftw/lib/
+      cache-path: ${{ needs.romanisim_data_cache.outputs.cache_path }}
+      cache-key: ${{ needs.romanisim_data_cache.outputs.cache_key }} 
+      envs: |
+        - linux: py311-test-romanisim-cov-xdist
+
   astropy:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
     if: (github.repository == 'spacetelescope/gwcs' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Downstream CI')))

diff --git a/.github/workflows/get_webbpsf.yml b/.github/workflows/get_webbpsf.yml
@@ -0,0 +1,31 @@
+name: retrieve latest data cache key
+
+on:
+  workflow_call:
+    outputs:
+      cache_key:
+        value: ${{ jobs.retrieve_cache.outputs.cache_key }}
+      cache_path:
+        value: ${{ jobs.retrieve_cache.outputs.cache_path }}
+
+jobs:
+  retrieve_cache:
+    name: retrieve latest data cache key
+    runs-on: ubuntu-latest
+    steps:
+      - name: retrieve latest data cache key
+        id: latest_cache_key
+        run: |
+          # use actions/gh-actions-cache to allow filtering by key
+          gh extension install actions/gh-actions-cache
+          CACHE_KEY=$(gh actions-cache list -R ${{ github.repository }} --key data- --sort created-at | cut -f 1 | head -n 1)
+          if [ "$CACHE_KEY" == '' ]; then exit 1; fi
+          echo cache_key=$CACHE_KEY >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - run: echo cache_key=${{ steps.latest_cache_key.outputs.cache_key }}
+      - id: cache_path
+        run: echo cache_path=/tmp/data/ >> $GITHUB_OUTPUT
+    outputs:
+      cache_key: ${{ steps.latest_cache_key.outputs.cache_key }}
+      cache_path: ${{ steps.cache_path.outputs.cache_path }}
diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml
@@ -0,0 +1,51 @@
+name: download and cache data
+
+on:
+  workflow_call:
+  schedule:
+    - cron: "42 4 * * 3"
+  workflow_dispatch:
+    inputs:
+      webbpsf_minimal:
+        description: minimal WebbPSF dataset
+        type: boolean
+        required: false
+        default: true
+
+jobs:
+  download_webbpsf_data:
+    uses: spacetelescope/webbpsf/.github/workflows/download_data.yml@beda656c80a0254e6f80649d9c9c49235634522f  # v1.4.0
+    with:
+      minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }}
+  combine_data_cache:
+    needs: [ download_webbpsf_data ]
+    runs-on: ubuntu-latest
+    env:
+      GALSIM_DATA_URL: https://github.com/GalSim-developers/GalSim/raw/releases/2.4/examples/data/
+    steps:
+      - name: download GalSim data    
+        run: |
+          mkdir galsim_data
+          wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example.fits -O  galsim_data/real_galaxy_catalog_23.5_example.fits
+          wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_selection.fits -O  galsim_data/real_galaxy_catalog_23.5_example_selection.fits
+          wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_fits.fits -O  galsim_data/real_galaxy_catalog_23.5_example_fits.fits
+      - id: galsim_data
+        run: echo "hash=${{ hashFiles( 'galsim_data/' ) }}" >> $GITHUB_OUTPUT
+      - run: echo GALSIM_PATH=/tmp/data/galsim_data/ >> $GITHUB_ENV
+      - run: |
+          mkdir -p ${{ env.GALSIM_PATH }}
+          mv ./galsim_data/* ${{ env.GALSIM_PATH }}
+      - run: echo GALSIM_CAT_PATH=${{ env.GALSIM_PATH }}real_galaxy_catalog_23.5_example.fits >> $GITHUB_ENV
+      - name: retrieve cached WebbPSF data
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
+        with:
+          path: ${{ needs.download_webbpsf_data.outputs.cache_path }}
+          key: ${{ needs.download_webbpsf_data.outputs.cache_key }}
+      - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ /tmp/data/
+      - run: echo WEBBPSF_PATH=/tmp/data/webbpsf-data/ >> $GITHUB_ENV
+      # save a new cache to the same generalized data directory, combined with extra data
+      - name: save a single combined data cache
+        uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
+        with:
+          path: /tmp/data/
+          key: data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }}
diff --git a/tox.ini b/tox.ini
@@ -45,6 +45,7 @@ description =
     run tests
     jwst: of JWST pipeline
     romancal: of Romancal pipeline
+    romanisim: of Romanisim image simulation
     dev: with the latest developer version of key dependencies
     pyargs: with --pyargs on installed package
     warnings: treating warnings as errors
@@ -57,6 +58,10 @@ passenv =
     CI
     CODECOV_*
     DISPLAY
+    WEBBPSF_PATH
+    GALSIM_CAT_PATH
+    FFTW_DIR
+    LIBRARY_PATH
 set_env =
     dev: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
 
@@ -70,6 +75,7 @@ deps =
     cov: pytest-cov
     jwst: jwst[test] @ git+https://github.com/spacetelescope/jwst.git
     romancal: romancal[test] @ git+https://github.com/spacetelescope/romancal.git
+    romanisim: romanisim[test] @ git+https://github.com/spacetelescope/romanisim.git
     numpy123: numpy==1.23.*
     numpy125: numpy==1.25.*
 pass_env =
@@ -84,6 +90,7 @@ commands =
     pyargs: {toxinidir}/docs --pyargs gwcs \
     jwst: --pyargs jwst --ignore-glob=timeconversion --ignore-glob=associations --ignore-glob=scripts --show-capture=no \
     romancal: --pyargs romancal \
+    romanisim: --pyargs romanisim \
     cov: --cov=. --cov-config=pyproject.toml --cov-report=term-missing --cov-report=xml \
     {posargs}