Add Workflow to check compatibility of TARDIS and Carsus (#313)

* Initial commit change env file path change env file path change path again change path again skip first step path again Package path * Add step to download the CMFGEN database * Add URLs to the CMFGEN database Fix atom data path * Add tardis build job repo path test * Add strategy for TARDIS build job * Fix lock file path Fix package path * Add step to run notebook * Upload spectrum notebook Fix notebook path Fix notebook path Require carsus build to run again before running tardis build * Fix atom data path Fix output format of atom data notebook Fix atom data path again Print atom data path Fix atom data path * Change URL used to get the reference data file * Download the file from the workflow * Upload reference data as artifact * Try downloading all reference data * Clone my fork of tardis refdata * Use lfs when cloning * Separate commands * Add new env variables, generate reference data for comparision using the new atomic data file * Clone my fork with arraydiff unpinned Fix atomic data path * Fix notebook path when creating artifact * Fix paths in environment variables * Install bokeh * Fix paths after moving branch * Rename notebook * Remove variables related to matrix strategy * Remove step to download reference data * Create copy of the quickstart notebook * Delete markdown * Add refdata_gen notebook instead of quickstart * Change branch when cloning carsus in the workflow * Try removing chianti reader * Put back elements in the atomic data * Add chianti data back but without collisions * Change Chianti version * Continue workflow if step fails * Revert back Chianti version * Add step to fail job if spectra dont match * test * Exit if contents of tmp file are set to 0 * Exit code * revert comments * On workflow dispatch * Rename steps and use variables * Replace NBCONVERT_CMD variable with NBCONVERT_FLAGS * Comment out Carsus build job temporarily, download reference data from ci helpers * Disable workflow dispatch temporarily * Fix paths for refdata compare notebook * Carsus build is necessary for TARDIS build * Change file name in notebook * Delete lines commented out which are not necessary * Do not clone my fork of the carsus repository * Change title of the notebook and rename it * Change file and step names * No need for inputs * Check file contents according to changes in tardis-sn/tardis-refdata#58 * Change path of refdata_compare_result file * Change trigger back to workflow_dispatch * Update URL to use TARDIS fork * Do not pin bokeh * Add name to comparison notebook artifact
tardis-sn · Feb 22, 2023 · 7114302 · 7114302
1 parent 4aa55e4
commit 7114302
Show file tree

Hide file tree

Showing 2 changed files with 338 additions and 0 deletions.
diff --git a/.github/workflows/bridge.yml b/.github/workflows/bridge.yml
@@ -0,0 +1,140 @@
+name: tardis-carsus-compatibility-check
+
+on: 
+  workflow_dispatch
+
+
+env:
+  XUVTOP: /tmp/chianti
+  CHIANTI_DL_URL: https://download.chiantidatabase.org
+  CHIANTI_DB_VER: CHIANTI_v9.0.1_database.tar.gz
+  NBCONVERT_FLAGS: --execute --ExecutePreprocessor.timeout=600 --to html
+  CMFGEN_DL_URL: http://kookaburra.phyast.pitt.edu/hillier/cmfgen_files
+  CMFGEN_DB_VER: atomic_data_15nov16.tar.gz
+  # original reference data
+  REF1_PATH: ${{ github.workspace }}/tardis-refdata/unit_test_data_org.h5
+  # generated reference data using the new atomic file 
+  REF2_PATH: ${{ github.workspace }}/tardis-refdata/unit_test_data.h5
+  PYTEST_FLAGS: --tardis-refdata=${{ github.workspace }}/tardis-refdata 
+                --generate-reference
+
+defaults:
+      run:
+        shell: bash -l {0}
+
+jobs:
+  carsus-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: carsus/
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.XUVTOP }}
+          key: ${{ runner.os }}-${{ env.CHIANTI_DB_VER }}
+        id: chianti-cache
+
+      - name: Download Chianti database
+        run: |
+            mkdir -p ${{ env.XUVTOP }}
+            wget -q ${{ env.CHIANTI_DL_URL }}/${{ env.CHIANTI_DB_VER }} -O ${{ env.XUVTOP }}/chianti.tar.gz
+            tar -zxf ${{ env.XUVTOP }}/chianti.tar.gz -C ${{ env.XUVTOP }}
+        if: steps.chianti-cache.outputs.cache-hit != 'true'
+
+      - uses: actions/cache@v2
+        with:
+          path: /tmp/atomic
+          key: ${{ env.CMFGEN_DB_VER }}
+        id: cmfgen-cache
+
+      - name: Download CMFGEN database
+        run: |
+            wget -q -U "Mozilla/4.0" ${{ env.CMFGEN_DL_URL }}/${{ env.CMFGEN_DB_VER }} -O /tmp/atomic.tar.gz
+            tar -zxf /tmp/atomic.tar.gz -C /tmp
+        if: steps.cmfgen-cache.outputs.cache-hit != 'true'
+
+      - name: Setup carsus environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            environment-file: carsus/carsus_env3.yml
+            activate-environment: carsus
+            use-mamba: true
+
+      - name: Install package
+        run: pip install -e carsus/
+
+      - name: Run notebooks
+        run: |
+          jupyter nbconvert ${{ env.NBCONVERT_FLAGS }} carsus/docs/tardis_atomdata_ref.ipynb
+        env:
+          CARSUS_REFDATA: ${{ github.workspace }}/carsus-refdata
+
+      - name: Upload Atom Data
+        uses: actions/upload-artifact@v2
+        with:
+          name: atom-data
+          path: carsus/docs/kurucz_cd23_chianti_H_He.h5
+
+  tardis-build:
+    needs: carsus-build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: tardis-sn/tardis
+
+      - name: Download refdata_compare notebook
+        run: wget https://raw.githubusercontent.com/tardis-sn/tardis-refdata/master/notebooks/ref_data_compare_from_paths.ipynb
+
+      - name: Download reference data
+        run: bash .ci-helpers/download_reference_data.sh
+
+      - name: Setup TARDIS environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: tardis
+            use-mamba: true
+
+      - name: Update environment
+        run: mamba update -n tardis --file conda-linux-64.lock
+
+      - name: Install bokeh
+        run: mamba install bokeh --channel conda-forge --no-update-deps --yes
+
+      - name: Install package
+        run: pip install -e .
+
+      - name: Download atom data
+        uses: actions/download-artifact@v2
+        with:
+          name: atom-data
+          path: carsus/docs/kurucz_cd23_chianti_H_He.h5
+
+      - name: Replace Atomic Data
+        run: cp carsus/docs/kurucz_cd23_chianti_H_He.h5/kurucz_cd23_chianti_H_He.h5 tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5
+
+      - name: Generate Reference Data Copy
+        run: cp tardis-refdata/unit_test_data.h5 tardis-refdata/unit_test_data_org.h5 
+
+      - name: Generate New Reference Data
+        run:  pytest tardis ${{ env.PYTEST_FLAGS }}
+
+      - name: Compare Reference Data
+        run: |
+            jupyter nbconvert ${{ env.NBCONVERT_FLAGS }} ref_data_compare_from_paths.ipynb
+      
+      - name: Upload comparison notebook
+        uses: actions/upload-artifact@v2
+        with:
+          name: Comparison Notebook
+          path: ref_data_compare_from_paths.html
+
+      - name: Check if reference data are equal
+        run: if [ "$(cat ../refdata_compare_result)" == "REFDATA COMPARISON FAILED" ]; then exit 1; fi
diff --git a/docs/tardis_atomdata_ref.ipynb b/docs/tardis_atomdata_ref.ipynb
@@ -0,0 +1,198 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Generate Atomic Data Reference for TARDIS\n",
+    "This noteboook generates atomic data to be compared with the [atomic data stored in tardis refdata repository](https://github.com/tardis-sn/tardis-refdata/blob/master/atom_data/kurucz_cd23_chianti_H_He.h5) for testing purposes. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T10:44:42.401734Z",
+     "start_time": "2022-08-02T10:44:42.398590Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from carsus.io.nist import NISTWeightsComp, NISTIonizationEnergies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T10:44:55.909963Z",
+     "start_time": "2022-08-02T10:44:42.949132Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "atomic_weights = NISTWeightsComp()\n",
+    "ionization_energies = NISTIonizationEnergies('H-Zn', )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T08:10:38.821252Z",
+     "start_time": "2022-08-02T08:10:13.465692Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "!wget -qO /tmp/gfall.dat https://media.githubusercontent.com/media/tardis-sn/carsus-db/master/gfall/gfall_latest.dat"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T08:10:59.427252Z",
+     "start_time": "2022-08-02T08:10:59.409344Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from carsus.io.kurucz import GFALLReader\n",
+    "\n",
+    "gfall_reader = GFALLReader('H-Zn',\n",
+    "                           '/tmp/gfall.dat')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T08:11:11.360903Z",
+     "start_time": "2022-08-02T08:11:11.353226Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from carsus.io.chianti_ import ChiantiReader\n",
+    "\n",
+    "chianti_reader = ChiantiReader('H-He', \n",
+    "                               collisions=False, \n",
+    "                               priority=30)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T08:11:12.931369Z",
+     "start_time": "2022-08-02T08:11:11.700068Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from carsus.io.zeta import KnoxLongZeta\n",
+    "\n",
+    "zeta_data = KnoxLongZeta()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-08-02T08:11:30.555303Z",
+     "start_time": "2022-08-02T08:11:13.631612Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "from carsus.io.output import TARDISAtomData\n",
+    "\n",
+    "atom_data = TARDISAtomData(atomic_weights,\n",
+    "                           ionization_energies,\n",
+    "                           gfall_reader,\n",
+    "                           zeta_data,\n",
+    "                           chianti_reader)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2022-07-29T10:28:50.219288Z",
+     "start_time": "2022-07-29T10:28:49.208901Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "atom_data.to_hdf('kurucz_cd23_chianti_H_He.h5')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  },
+  "varInspector": {
+   "cols": {
+    "lenName": 16,
+    "lenType": 16,
+    "lenVar": 40
+   },
+   "kernels_config": {
+    "python": {
+     "delete_cmd_postfix": "",
+     "delete_cmd_prefix": "del ",
+     "library": "var_list.py",
+     "varRefreshCmd": "print(var_dic_list())"
+    },
+    "r": {
+     "delete_cmd_postfix": ") ",
+     "delete_cmd_prefix": "rm(",
+     "library": "var_list.r",
+     "varRefreshCmd": "cat(var_dic_list()) "
+    }
+   },
+   "types_to_exclude": [
+    "module",
+    "function",
+    "builtin_function_or_method",
+    "instance",
+    "_Feature"
+   ],
+   "window_display": false
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}