Update the Carsus environment (tardis-sn#406)

* Updates the Carsus environment and fixes most issues

* Removes leftover testing references to SQL and arraydiff

* Fix tests requiring astropy parts

* Fixes deprecation and possibly astropy units issue

* Fixes some deprecations

* Workaround for ChiantiPy bug

* Another minor formatting issue

* Fixes CMFGEN tests

* Add comment regarding temperature "fix" for ChiantiPy

* Fixes TARDIS data export and hashing using pickle (sadly)

* Reverts accidental commit

.github/workflows/tests.yml

@@ -27,8 +27,7 @@ env:
   CHIANTI_DL_URL: https://download.chiantidatabase.org
   CHIANTI_DB_VER: CHIANTI_v9.0.1_database.tar.gz
   PYTEST_FLAGS: --remote-data --refdata=carsus-refdata
-                --cov=carsus --cov-report=xml --cov-report=html 
-                --arraydiff --arraydiff-reference-path=carsus-refdata/arraydiff  
+                --cov=carsus --cov-report=xml --cov-report=html   
   NBCONVERT_CMD: jupyter nbconvert --execute --ExecutePreprocessor.timeout=600 --to html
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   CMFGEN_DL_URL: http://kookaburra.phyast.pitt.edu/hillier/cmfgen_files

carsus/conftest.py

@@ -83,23 +83,9 @@ def pytest_collection_modifyitems(config, items):
     skip_not_with_refdata = pytest.mark.skip(
         reason="carsus-refdata folder location not specified"
     )
-    skip_not_with_testdb = pytest.mark.skip(
-        reason="filename for the testing database not specified"
-    )
     for item in items:
         if "with_refdata" in item.keywords and not config.getoption("--refdata"):
             item.add_marker(skip_not_with_refdata)
-        if "with_test_db" in item.keywords and not config.getoption("--test-db"):
-            item.add_marker(skip_not_with_testdb)
-
-
-@pytest.fixture(scope="session")
-def test_db_fname(request):
-    test_db_fname = request.config.getoption("--test-db")
-    if test_db_fname is None:
-        pytest.skip("--testing database was not specified")
-    else:
-        return str(Path(test_db_fname).expanduser().resolve())
 
 
 @pytest.fixture(scope="session")
@@ -129,37 +115,6 @@ def nndc_dirname():
     return str(DATA_DIR_PATH / "nndc")  # Mn-52, Ni-56
 
 
-@pytest.fixture(scope="session")
-def test_engine(test_db_url):
-    return create_engine(test_db_url)
-
-
-@pytest.fixture
-def test_session(test_engine, request):
-    # engine.echo=True
-    # connect to the database
-    connection = test_engine.connect()
-
-    # begin a non-ORM transaction
-    trans = connection.begin()
-
-    # bind an individual Session to the connection
-    session = Session(bind=connection)
-
-    def fin():
-        session.close()
-        # rollback - everything that happened with the
-        # Session above (including calls to commit())
-        # is rolled back.
-        trans.rollback()
-        # return connection to the Engine
-        connection.close()
-
-    request.addfinalizer(fin)
-
-    return session
-
-
 @pytest.fixture(scope="session")
 def refdata_path(request):
     refdata_path = request.config.getoption("--refdata")

carsus/data/basic_atomic_data.csv

@@ -57,7 +57,7 @@ atomic_number,symbol,name,group,period
 56,Ba,Barium,2.0,6
 57,La,Lanthanum,-1.0,6
 58,Ce,Cerium,-1.0,6
-59,Pr,Praseodymium -1   ,-1.0,6
+59,Pr,Praseodymium,-1.0,6
 60,Nd,Neodymium,-1.0,6
 61,Pm,Promethium,-1.0,6
 62,Sm,Samarium,-1.0,6

carsus/io/chianti_/chianti_.py

@@ -39,7 +39,7 @@ def versionRead():
 masterlist_ions = pickle.load(masterlist_ions_file).keys()
 # Exclude the "d" ions for now
 masterlist_ions = [_ for _ in masterlist_ions
-                   if re.match("^[a-z]+_\d+$", _)]
+                   if re.match(r"^[a-z]+_\d+$", _)]
 
 masterlist_version = versionRead()
 
@@ -109,7 +109,8 @@ class ChiantiIonReader(object):
 
     def __init__(self, ion_name):
 
-        self.ion = ch.ion(ion_name)
+        # dummy temperature to avoid bug in ChiantiPy https://github.com/chianti-atomic/ChiantiPy/issues/466
+        self.ion = ch.ion(ion_name, temperature=1)
         self._levels = None
         self._lines = None
         self._collisions = None

carsus/io/cmfgen/base.py

@@ -37,7 +37,7 @@ def load(self, fname):
         config = {
             "header": None,
             "index_col": False,
-            "sep": "\s+",
+            "sep": r"\s+",
             "skiprows": skiprows,
             "nrows": nrows,
             "engine": "python",
@@ -73,7 +73,7 @@ def load(self, fname):
         self.base = df
         self.header = header
         # Re-calculate Lam(A) values
-        self.base["Lam(A)"] = self.calc_Lam_A()
+        self.base["Lam(A)"] = self.calc_Lam_A().value
 
     def calc_Lam_A(self):
         """
@@ -112,7 +112,7 @@ def load(self, fname):
         config = {
             "header": None,
             "index_col": False,
-            "sep": "\s*\|\s*|-?\s+-?\s*|(?<=[^ED\s])-(?=[^\s])",
+            "sep": r"\s*\|\s*|-?\s+-?\s*|(?<=[^ED\s])-(?=[^\s])",
             "skiprows": skiprows,
             "nrows": nrows,
             "engine": "python",
@@ -177,11 +177,11 @@ class CMFGENCollisionalStrengthsParser(BaseParser):
 
     def load(self, fname):
         header = parse_header(fname)
-        skiprows, _ = find_row(fname, "ransition\T")
+        skiprows, _ = find_row(fname, r"ransition\T")
         config = {
             "header": None,
             "index_col": False,
-            "sep": "\s*-?\s+-?|(?<=[^edED])-|(?<=[FDPS]e)-",
+            "sep": r"\s*-?\s+-?|(?<=[^edED])-|(?<=[FDPS]e)-",
             "skiprows": skiprows,
             "engine": "python",
         }
@@ -194,7 +194,7 @@ def load(self, fname):
             config["nrows"] = end - config["skiprows"] - 2
 
         try:
-            _, columns = find_row(fname, "ransition\T")
+            _, columns = find_row(fname, r"ransition\T")
             columns = columns.split()
 
             # NOTE: Comment next line when trying new regexes
@@ -659,9 +659,11 @@ def cross_sections_squeeze(
             lower_level_label = target.attrs["Configuration name"]
             cross_section_type = target.attrs["Type of cross-section"]
 
-            # Remove the "[J]" term from J-splitted levels labels
-            ion_levels["label"] = ion_levels["label"].str.rstrip("]")
-            ion_levels["label"] = ion_levels["label"].str.split("[", expand=True)
+            if ion_levels["label"].str.contains(r"\[").any():
+                # Remove the "[J]" term from J-splitted levels labels
+                ion_levels["label"] = ion_levels["label"].str.rstrip("]")
+                # Drop the [J] term completely to avoid shape mismatch. Something to perhaps store in the future
+                ion_levels["label"] = ion_levels["label"].str.split("[", expand=True)[0]
 
             try:
                 match = ion_levels.set_index("label").loc[[lower_level_label]]

carsus/io/kurucz/gfall.py

@@ -252,8 +252,8 @@ def parse_gfall(self, gfall_raw=None):
         gfall["label_lower"] = gfall["label_lower"].str.strip()
         gfall["label_upper"] = gfall["label_upper"].str.strip()
 
-        gfall["label_lower"] = gfall["label_lower"].str.replace("\s+", " ")
-        gfall["label_upper"] = gfall["label_upper"].str.replace("\s+", " ")
+        gfall["label_lower"] = gfall["label_lower"].str.replace(r"\s+", " ")
+        gfall["label_upper"] = gfall["label_upper"].str.replace(r"\s+", " ")
 
         # Ignore lines with the labels "AVARAGE ENERGIES" and "CONTINUUM"
         ignored_labels = ["AVERAGE", "ENERGIES", "CONTINUUM"]

carsus/io/nist/ionization.py

@@ -227,21 +227,21 @@ def parse_ground_level(row):
             )
 
             try:
-                lvl_tokens = level.parseString(ground_level)
+                lvl_tokens = level.parse_string(ground_level)
             except ParseException:
                 raise
 
             lvl["parity"] = lvl_tokens["parity"]
 
             try:
-                lvl["J"] = lvl_tokens["J"]
+                lvl["J"] = lvl_tokens["J"][0]
             except KeyError:
                 pass
 
             # To handle cases where the ground level J has not been understood:
             # Take as assumption J=0
             if np.isnan(lvl["J"]):
-                lvl["J"] = "0"
+                lvl["J"] = 0
                 logger.warning(
                     f"Set `J=0` for ground state of species `{convert_atomic_number2symbol(row['atomic_number'])} {row['ion_charge']}`."
                 )
@@ -308,7 +308,7 @@ def get_ground_levels(self):
         """
         levels = self.parser.prepare_ground_levels()
         levels["g"] = 2 * levels["J"] + 1
-        levels["g"] = levels["g"].astype(np.int)
+        levels["g"] = levels["g"].astype(np.int64)
         levels["energy"] = 0.0
         levels = levels[["g", "energy"]]
         levels = levels.reset_index()

carsus/io/output/base.py

@@ -220,7 +220,7 @@ def to_hdf(self, fname):
             total_checksum = hashlib.md5()
             for key in f.keys():
                 # update the total checksum to sign the file
-                total_checksum.update(serialize_pandas_object(f[key]).to_buffer())
+                total_checksum.update(serialize_pandas_object(f[key]))
 
                 # save individual DataFrame/Series checksum
                 checksum = hash_pandas_object(f[key])
@@ -251,7 +251,6 @@ def to_hdf(self, fname):
                 "astropy",
                 "numpy",
                 "pandas",
-                "pyarrow",
                 "tables",
                 "ChiantiPy",
             ]

carsus/io/output/collisions.py

@@ -47,7 +47,7 @@ def prepare_collisions(self):
                 collisions_index
                 + ["g_ratio", "delta_e"]
                 + sorted(
-                    [col for col in self.collisions.columns if re.match("^t\d+$", col)]
+                    [col for col in self.collisions.columns if re.match(r"^t\d+$", col)]
                 )
             )
 

carsus/io/output/levels_lines.py

@@ -110,20 +110,19 @@ def ingest_multiple_sources(self, attribute):
         """
         gfall = getattr(self.gfall_reader, attribute)
         gfall["ds_id"] = 2
+        sources = [gfall]
 
         if self.chianti_reader is not None:
             chianti = getattr(self.chianti_reader, attribute)
             chianti["ds_id"] = 4
-        else:
-            chianti = pd.DataFrame(columns=gfall.columns)
+            sources.append(chianti)
 
         if self.cmfgen_reader is not None:
             cmfgen = getattr(self.cmfgen_reader, attribute)
             cmfgen["ds_id"] = 5
-        else:
-            cmfgen = pd.DataFrame(columns=gfall.columns)
+            sources.append(cmfgen)
 
-        return pd.concat([gfall, chianti, cmfgen], sort=True)
+        return pd.concat(sources, sort=True)
 
     # replace with functools.cached_property with Python > 3.8
     @property
@@ -148,7 +147,7 @@ def all_levels_data(self):
         levels = self.ingest_multiple_sources("levels")
 
         levels["g"] = 2 * levels["j"] + 1
-        levels["g"] = levels["g"].astype(np.int)
+        levels["g"] = levels["g"].astype(np.int64)
         levels = levels.drop(columns=["j", "label", "method"])
         levels = levels.reset_index()
         levels = levels.rename(columns={"ion_charge": "ion_number"})
@@ -444,7 +443,7 @@ def create_levels_lines(
             .values
         )
 
-        levels["level_number"] = levels["level_number"].astype(np.int)
+        levels["level_number"] = levels["level_number"].astype(np.int64)
 
         levels = levels[
             [
@@ -478,7 +477,7 @@ def create_levels_lines(
         lines["f_ul"] = lines["gf"] / lines["g_u"]
 
         # Calculate frequency
-        lines["nu"] = u.Quantity(lines["wavelength"], "AA").to("Hz", u.spectral())
+        lines["nu"] = u.Quantity(lines["wavelength"], "AA").to("Hz", u.spectral()).value
 
         # Create Einstein coefficients
         create_einstein_coeff(lines)
@@ -490,7 +489,7 @@ def create_levels_lines(
 
         # Create and append artificial levels for fully ionized ions
         artificial_fully_ionized_levels = create_artificial_fully_ionized(levels)
-        levels = levels.append(artificial_fully_ionized_levels, ignore_index=True)
+        levels = pd.concat([levels, artificial_fully_ionized_levels], ignore_index=True)
         levels = levels.sort_values(["atomic_number", "ion_number", "level_number"])
 
         self.levels = levels

carsus/io/output/macro_atom.py

@@ -42,13 +42,13 @@ def create_macro_atom(self):
 
         macro_atom = list()
         macro_atom_dtype = [
-            ("atomic_number", np.int),
-            ("ion_number", np.int),
-            ("source_level_number", np.int),
-            ("target_level_number", np.int),
-            ("transition_line_id", np.int),
-            ("transition_type", np.int),
-            ("transition_probability", np.float),
+            ("atomic_number", np.int64),
+            ("ion_number", np.int64),
+            ("source_level_number", np.int64),
+            ("target_level_number", np.int64),
+            ("transition_line_id", np.int64),
+            ("transition_type", np.int64),
+            ("transition_probability", np.float64),
         ]
 
         for line_id, row in lines.iterrows():
@@ -146,15 +146,15 @@ def create_macro_atom_references(self):
 
         macro_atom_references["count_down"] = macro_atom_references[
             "count_down"
-        ].astype(np.int)
+        ].astype(np.int64)
 
         macro_atom_references["count_up"] = macro_atom_references["count_up"].astype(
-            np.int
+            np.int64
         )
 
         macro_atom_references["count_total"] = macro_atom_references[
             "count_total"
-        ].astype(np.int)
+        ].astype(np.int64)
 
         self.macro_atom_references = macro_atom_references
 

carsus/io/output/photo_ionization.py

@@ -63,8 +63,8 @@ def cross_sections(self):
 
         cross_sections["energy"] = u.Quantity(cross_sections["energy"], "Ry").to(
             "Hz", equivalencies=u.spectral()
-        )
-        cross_sections["sigma"] = u.Quantity(cross_sections["sigma"], "Mbarn").to("cm2")
+        ).value
+        cross_sections["sigma"] = u.Quantity(cross_sections["sigma"], "Mbarn").to("cm2").value
         cross_sections["level_number"] = cross_sections["level_number"].astype("int")
         cross_sections = cross_sections.rename(
             columns={"energy": "nu", "sigma": "x_sect"}

carsus/io/tests/test_cmfgen.py

@@ -17,7 +17,6 @@
 data_dir = os.path.join(os.path.dirname(__file__), "data")
 
 
-@pytest.mark.with_refdata
 @pytest.fixture()
 def si1_reader():
     return CMFGENReader.from_config(