From 711430266e9f61a4708aae0e773f4f4af1083f45 Mon Sep 17 00:00:00 2001 From: Atharva Arya <55894364+atharva-2001@users.noreply.github.com> Date: Wed, 22 Feb 2023 21:11:12 +0530 Subject: [PATCH] Add Workflow to check compatibility of TARDIS and Carsus (#313) * Initial commit change env file path change env file path change path again change path again skip first step path again Package path * Add step to download the CMFGEN database * Add URLs to the CMFGEN database Fix atom data path * Add tardis build job repo path test * Add strategy for TARDIS build job * Fix lock file path Fix package path * Add step to run notebook * Upload spectrum notebook Fix notebook path Fix notebook path Require carsus build to run again before running tardis build * Fix atom data path Fix output format of atom data notebook Fix atom data path again Print atom data path Fix atom data path * Change URL used to get the reference data file * Download the file from the workflow * Upload reference data as artifact * Try downloading all reference data * Clone my fork of tardis refdata * Use lfs when cloning * Separate commands * Add new env variables, generate reference data for comparision using the new atomic data file * Clone my fork with arraydiff unpinned Fix atomic data path * Fix notebook path when creating artifact * Fix paths in environment variables * Install bokeh * Fix paths after moving branch * Rename notebook * Remove variables related to matrix strategy * Remove step to download reference data * Create copy of the quickstart notebook * Delete markdown * Add refdata_gen notebook instead of quickstart * Change branch when cloning carsus in the workflow * Try removing chianti reader * Put back elements in the atomic data * Add chianti data back but without collisions * Change Chianti version * Continue workflow if step fails * Revert back Chianti version * Add step to fail job if spectra dont match * test * Exit if contents of tmp file are set to 0 * Exit code * revert comments * On workflow dispatch * Rename steps and use variables * Replace NBCONVERT_CMD variable with NBCONVERT_FLAGS * Comment out Carsus build job temporarily, download reference data from ci helpers * Disable workflow dispatch temporarily * Fix paths for refdata compare notebook * Carsus build is necessary for TARDIS build * Change file name in notebook * Delete lines commented out which are not necessary * Do not clone my fork of the carsus repository * Change title of the notebook and rename it * Change file and step names * No need for inputs * Check file contents according to changes in tardis-sn/tardis-refdata#58 * Change path of refdata_compare_result file * Change trigger back to workflow_dispatch * Update URL to use TARDIS fork * Do not pin bokeh * Add name to comparison notebook artifact --- .github/workflows/bridge.yml | 140 +++++++++++++++++++++++ docs/tardis_atomdata_ref.ipynb | 198 +++++++++++++++++++++++++++++++++ 2 files changed, 338 insertions(+) create mode 100644 .github/workflows/bridge.yml create mode 100644 docs/tardis_atomdata_ref.ipynb diff --git a/.github/workflows/bridge.yml b/.github/workflows/bridge.yml new file mode 100644 index 000000000..83e0512dc --- /dev/null +++ b/.github/workflows/bridge.yml @@ -0,0 +1,140 @@ +name: tardis-carsus-compatibility-check + +on: + workflow_dispatch + + +env: + XUVTOP: /tmp/chianti + CHIANTI_DL_URL: https://download.chiantidatabase.org + CHIANTI_DB_VER: CHIANTI_v9.0.1_database.tar.gz + NBCONVERT_FLAGS: --execute --ExecutePreprocessor.timeout=600 --to html + CMFGEN_DL_URL: http://kookaburra.phyast.pitt.edu/hillier/cmfgen_files + CMFGEN_DB_VER: atomic_data_15nov16.tar.gz + # original reference data + REF1_PATH: ${{ github.workspace }}/tardis-refdata/unit_test_data_org.h5 + # generated reference data using the new atomic file + REF2_PATH: ${{ github.workspace }}/tardis-refdata/unit_test_data.h5 + PYTEST_FLAGS: --tardis-refdata=${{ github.workspace }}/tardis-refdata + --generate-reference + +defaults: + run: + shell: bash -l {0} + +jobs: + carsus-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + path: carsus/ + + - uses: actions/cache@v2 + with: + path: ${{ env.XUVTOP }} + key: ${{ runner.os }}-${{ env.CHIANTI_DB_VER }} + id: chianti-cache + + - name: Download Chianti database + run: | + mkdir -p ${{ env.XUVTOP }} + wget -q ${{ env.CHIANTI_DL_URL }}/${{ env.CHIANTI_DB_VER }} -O ${{ env.XUVTOP }}/chianti.tar.gz + tar -zxf ${{ env.XUVTOP }}/chianti.tar.gz -C ${{ env.XUVTOP }} + if: steps.chianti-cache.outputs.cache-hit != 'true' + + - uses: actions/cache@v2 + with: + path: /tmp/atomic + key: ${{ env.CMFGEN_DB_VER }} + id: cmfgen-cache + + - name: Download CMFGEN database + run: | + wget -q -U "Mozilla/4.0" ${{ env.CMFGEN_DL_URL }}/${{ env.CMFGEN_DB_VER }} -O /tmp/atomic.tar.gz + tar -zxf /tmp/atomic.tar.gz -C /tmp + if: steps.cmfgen-cache.outputs.cache-hit != 'true' + + - name: Setup carsus environment + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + environment-file: carsus/carsus_env3.yml + activate-environment: carsus + use-mamba: true + + - name: Install package + run: pip install -e carsus/ + + - name: Run notebooks + run: | + jupyter nbconvert ${{ env.NBCONVERT_FLAGS }} carsus/docs/tardis_atomdata_ref.ipynb + env: + CARSUS_REFDATA: ${{ github.workspace }}/carsus-refdata + + - name: Upload Atom Data + uses: actions/upload-artifact@v2 + with: + name: atom-data + path: carsus/docs/kurucz_cd23_chianti_H_He.h5 + + tardis-build: + needs: carsus-build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + repository: tardis-sn/tardis + + - name: Download refdata_compare notebook + run: wget https://raw.githubusercontent.com/tardis-sn/tardis-refdata/master/notebooks/ref_data_compare_from_paths.ipynb + + - name: Download reference data + run: bash .ci-helpers/download_reference_data.sh + + - name: Setup TARDIS environment + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: tardis + use-mamba: true + + - name: Update environment + run: mamba update -n tardis --file conda-linux-64.lock + + - name: Install bokeh + run: mamba install bokeh --channel conda-forge --no-update-deps --yes + + - name: Install package + run: pip install -e . + + - name: Download atom data + uses: actions/download-artifact@v2 + with: + name: atom-data + path: carsus/docs/kurucz_cd23_chianti_H_He.h5 + + - name: Replace Atomic Data + run: cp carsus/docs/kurucz_cd23_chianti_H_He.h5/kurucz_cd23_chianti_H_He.h5 tardis-refdata/atom_data/kurucz_cd23_chianti_H_He.h5 + + - name: Generate Reference Data Copy + run: cp tardis-refdata/unit_test_data.h5 tardis-refdata/unit_test_data_org.h5 + + - name: Generate New Reference Data + run: pytest tardis ${{ env.PYTEST_FLAGS }} + + - name: Compare Reference Data + run: | + jupyter nbconvert ${{ env.NBCONVERT_FLAGS }} ref_data_compare_from_paths.ipynb + + - name: Upload comparison notebook + uses: actions/upload-artifact@v2 + with: + name: Comparison Notebook + path: ref_data_compare_from_paths.html + + - name: Check if reference data are equal + run: if [ "$(cat ../refdata_compare_result)" == "REFDATA COMPARISON FAILED" ]; then exit 1; fi diff --git a/docs/tardis_atomdata_ref.ipynb b/docs/tardis_atomdata_ref.ipynb new file mode 100644 index 000000000..42aca7a76 --- /dev/null +++ b/docs/tardis_atomdata_ref.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generate Atomic Data Reference for TARDIS\n", + "This noteboook generates atomic data to be compared with the [atomic data stored in tardis refdata repository](https://github.com/tardis-sn/tardis-refdata/blob/master/atom_data/kurucz_cd23_chianti_H_He.h5) for testing purposes. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T10:44:42.401734Z", + "start_time": "2022-08-02T10:44:42.398590Z" + } + }, + "outputs": [], + "source": [ + "from carsus.io.nist import NISTWeightsComp, NISTIonizationEnergies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T10:44:55.909963Z", + "start_time": "2022-08-02T10:44:42.949132Z" + } + }, + "outputs": [], + "source": [ + "atomic_weights = NISTWeightsComp()\n", + "ionization_energies = NISTIonizationEnergies('H-Zn', )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T08:10:38.821252Z", + "start_time": "2022-08-02T08:10:13.465692Z" + } + }, + "outputs": [], + "source": [ + "!wget -qO /tmp/gfall.dat https://media.githubusercontent.com/media/tardis-sn/carsus-db/master/gfall/gfall_latest.dat" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T08:10:59.427252Z", + "start_time": "2022-08-02T08:10:59.409344Z" + } + }, + "outputs": [], + "source": [ + "from carsus.io.kurucz import GFALLReader\n", + "\n", + "gfall_reader = GFALLReader('H-Zn',\n", + " '/tmp/gfall.dat')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T08:11:11.360903Z", + "start_time": "2022-08-02T08:11:11.353226Z" + } + }, + "outputs": [], + "source": [ + "from carsus.io.chianti_ import ChiantiReader\n", + "\n", + "chianti_reader = ChiantiReader('H-He', \n", + " collisions=False, \n", + " priority=30)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T08:11:12.931369Z", + "start_time": "2022-08-02T08:11:11.700068Z" + } + }, + "outputs": [], + "source": [ + "from carsus.io.zeta import KnoxLongZeta\n", + "\n", + "zeta_data = KnoxLongZeta()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-08-02T08:11:30.555303Z", + "start_time": "2022-08-02T08:11:13.631612Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "from carsus.io.output import TARDISAtomData\n", + "\n", + "atom_data = TARDISAtomData(atomic_weights,\n", + " ionization_energies,\n", + " gfall_reader,\n", + " zeta_data,\n", + " chianti_reader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2022-07-29T10:28:50.219288Z", + "start_time": "2022-07-29T10:28:49.208901Z" + } + }, + "outputs": [], + "source": [ + "atom_data.to_hdf('kurucz_cd23_chianti_H_He.h5')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.12" + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {}, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}