Merge pull request #1447 from knutfrode/dev

[run-ex] Updated parsing script, and harvested two new oils.

opendrift/models/openoil/adios/extra_oils/python_oils/parse.py

@@ -1,10 +1,12 @@
 def parse():
-    import NO00160_BREIDABLIKK as d
-    import NO00159_SVALE as d
-    import NO00161_STAER as d
-    import NO00162_YME as d
-    import NO00163_OFELIA as d
-    import NO00164_HEIDRUN_AARE as d
+    #import NO00160_BREIDABLIKK as d
+    #import NO00159_SVALE as d
+    #import NO00161_STAER as d
+    #import NO00162_YME as d
+    #import NO00163_OFELIA as d
+    #import NO00164_HEIDRUN_AARE as d
+    #import NO00165_LANGFJELLET as d
+    import NO00166_CALYPSO as d
     import adios_db.scripting as ads
     from adios_db.models.oil.values import Reference
     from adios_db.models.common.measurement import (MassFraction,
@@ -14,14 +16,16 @@ def parse():
 
     from adios_db.models.oil.physical_properties import DynamicViscosityPoint, PourPoint, FlashPoint
     from adios_db.models.oil.sara import Sara
+    from adios_db.models.oil.bulk_composition import BulkComposition, BulkCompositionList
     from adios_db.models.oil.distillation import DistCutList
 
-
     oil = ads.Oil(d.id)
     oil.metadata.name = d.name
     oil.metadata.API = (141.5/d.specific_gravity) - 131.5
     oil.metadata.product_type = d.product_type
     oil.metadata.reference = Reference(year=d.reference_year, reference=d.reference)
+    oil.metadata.sample_date = str(d.sample_date)
+    oil.metadata.location = d.location
 
     oil.sub_samples.extend([ads.Sample(), ads.Sample(), ads.Sample(), ads.Sample()])
     for i, (ss, w, vol, visc, dens) in enumerate(
@@ -50,7 +54,11 @@ def parse():
         oil.sub_samples[i].metadata.short_name = s
 
     oil.sub_samples[0].SARA = Sara(asphaltenes=ads.MassFraction(value=d.asphaltenes_percent, unit='%'))
-    oil.sub_samples[0].bulk_composition.append(ads.MassFraction(value=d.wax_percent, unit='%'))  # Probably wrong, a BulkComposition object should be used here
+    oil.sub_samples[0].bulk_composition = BulkCompositionList()
+    oil.sub_samples[0].bulk_composition.append(
+        BulkComposition(
+            name='wax_content',
+            measurement=ads.MassFraction(value=d.wax_percent, unit='%')))
 
     cut_temps = [c[0] for c in d.cuts]
     cut_frac = [c[1] for c in d.cuts]

opendrift/models/openoil/adios/extra_oils/templates/parse_oil_pdf.py

@@ -94,7 +94,7 @@ def __init__(self):
         if not os.path.exists(b+'.zip'):
             print('Parsing %s, this may take some minutes...' % f)
             import camelot.io as camelot
-            tables = camelot.read_pdf(f, pages='all', backend='poppler')
+            tables = camelot.read_pdf(f, pages='all', backend='ghostscript')  # or 'poppler'
             print('Total tables extracted: %s\n' % tables.n)
             tables.export(b+'.csv', f='csv', compress=True)
 
@@ -139,17 +139,21 @@ def __init__(self):
             if df.iloc[:, 1:].isnull().all().all():
                 print('Skipping empty table\n')
                 continue  # Do not print empty tables
-            if len(df.columns) not in [2, 5]:
+            if len(df.columns) not in [2, 3, 5]:
                 print('Skipping table without 2 or 5 columns\n')
                 continue
 
             color='lightgray'
-            if len(df.columns) == 2:
+            if len(df.columns) == 2 or len(df.columns) == 3:
+                if len(df.columns) == 3 and 'temp' in df.columns[0].lower():
+                    print(df.columns, 'COLS')
                 if 'temp' in df.columns[0].lower() and (
                         'vol' in df.columns[0].lower() or 'vol' in df.columns[1].lower()):
                     parent = cuts
                     width=60
                 else:
+                    if len(df.columns) == 3:
+                        continue
                     parent = properties
                     width=60
             elif len(df.columns) == 5:

tests/models/openoil/test_adios_dirjs.py

@@ -19,7 +19,7 @@ def test_get_all_oil_names(benchmark):
 
 def test_get_all_oil_names_location(benchmark):
     oils = benchmark(adios.dirjs.get_oil_names, location='NORWAY')
-    assert len(oils) >= 150 and len(oils) <= 201
+    assert len(oils) >= 203
 
 
 def test_bunker_c_1987():

tests/test_data/generated/oil_list_norway.txt

@@ -30,6 +30,7 @@ BRASSE 2018
 BREAM 2011
 BREIDABLIKK 2023
 BRYNHILD CRUDE 2015
+CALYPSO 2024
 CAURUS 2011
 DRAUGEN 2008
 DRIVIS 2017
@@ -90,6 +91,7 @@ JORDBAER 2011
 KRISTIN 2006
 KVITEBJORN 2009
 KVITEBJORN 2019
+LANGFJELLET 2023
 LAVRANS 1997
 LILLE PRINSEN 2022
 LILLEFRIGG KONDENSAT 1996