SMOS SBPCA data integrated (#735)

* SMOS SBPCA data integrated

* Update testdata

* Update .gitignore

* Update test data

* SMOS SBPCA integration fixes

.gitignore

@@ -18,3 +18,4 @@ run_celery/
 .coverage*
 */python_metadata/*
 node_modules
+UI/.angular

validator/fixtures/datasets.json

@@ -425,5 +425,30 @@
             },
             "user": null
         }
+    },
+    {
+        "model": "validator.dataset",
+        "pk": 17,
+        "fields": {
+            "short_name": "SMOS_SBPCA",
+            "pretty_name": "SMOS SBPCA",
+            "help_text": "SMOS SBPCA metrics exercise data",
+            "storage_path": "testdata/input_data/SMOS_SBPCA/",
+            "detailed_description": "This dataset is only available to members of the 'smos_developer' group. Please contact the administrator if you need access.",
+            "source_reference": "",
+            "citation": "",
+            "is_spatial_reference": false,
+            "versions": [
+                54
+            ],
+            "variables": [
+                19
+            ],
+            "resolution": {
+                "value": 15,
+                "unit": "km"
+            },
+            "user": null
+        }
     }
-]
+]

validator/fixtures/versions.json

@@ -774,5 +774,28 @@
                 1
             ]
         }
+    },
+    {
+        "model": "validator.datasetversion",
+        "pk": 54,
+        "fields": {
+            "short_name": "SMOS_SBPCA_v724",
+            "pretty_name": "v724",
+            "help_text": "SMOS L2 SM from SBPCA metrics exercise (ESA)",
+            "time_range_start": "2011-02-01",
+            "time_range_end": "2013-01-31",
+            "geographical_range": null,
+            "filters": [
+                1,
+                35,
+                36,
+                37,
+                38,
+                39,
+                40,
+                41,
+                42
+            ]
+        }
     }
-]
+]

validator/tests/test_validation.py

@@ -396,7 +396,7 @@ def test_validation(self):
 
         run.save()
 
-        for config in run.dataset_configurations.all():
+        for i, config in enumerate(run.dataset_configurations.all()):
             if config == run.spatial_reference_configuration:
                 config.filters.add(DataFilter.objects.get(name='FIL_ISMN_GOOD'))
             else:
@@ -1396,7 +1396,7 @@ def test_temporal_adapter(self):
     def test_setup_filtering_max(self):
         start_time = time.time()
 
-        for dataset in Dataset.objects.all():
+        for i, dataset in enumerate(Dataset.objects.all()):
             self.__logger.info(dataset.pretty_name)
             vs = dataset.versions.all()
             va = dataset.variables.all()
@@ -1432,8 +1432,7 @@ def test_setup_filtering_max(self):
                         # handles the case where all values are flagged (i.e. for SMOS L3)
                         if not len(data.index) > 1 or data[variable.short_name].empty:
                             unfiltered = reader.read(-155.42, 19.78, **read_kwargs)
-
-                            assert unfiltered.count()[variable.short_name] == len(unfiltered)
+                            assert unfiltered.count()[variable.short_name] == len(unfiltered[variable.short_name].dropna())
 
         print("Test duration: {}".format(time.time() - start_time))
 

validator/validation/globals.py

@@ -48,6 +48,7 @@
 SMOS_L3 = 'SMOS_L3'
 SMOS_L2 = 'SMOS_L2'
 SMAP_L2 = 'SMAP_L2'
+SMOS_SBPCA = 'SMOS_SBPCA'
 
 # dataset versions
 C3S_V201706 = 'C3S_V201706'
@@ -94,6 +95,7 @@
 SMOSL3_Level3_ASC = 'SMOSL3_v339_ASC'
 SMOSL2_700 = 'SMOSL2_v700'
 SMAPL2_V8 = 'SMAPL2_V8'
+SMOS_SBPCA_v724 = 'SMOS_SBPCA_v724'
 
 # dataset data variables PRETTY NAMES
 C3S_sm = 'C3S_sm'
@@ -113,6 +115,7 @@
 SMOSL3_sm = 'SMOSL3_sm'
 SMOSL2_sm = 'SMOSL2_sm'
 SMAPL2_soil_moisture = 'SMAPL2_soil_moisture'
+SMOS_SBPCA_sm = 'SMOS_SBPCA_sm'
 
 # left empty, because if in the future we want to exclude some datasets from the reference group it will be enough to
 # insert it's shortname to the list
@@ -128,6 +131,7 @@
                    'SMOS_IC': 0.25,
                    'ASCAT': 0.1,
                    'SMOS_L2': 0.135,  # 15km
+                   'SMOS_SBPCA': 0.135,  # 15km
                    'SMAP_L2': 0.35, }  # 35km
 
 START_TIME = datetime(1978, 1, 1).strftime('%Y-%m-%d')

validator/validation/readers.py

@@ -96,6 +96,9 @@ def create_reader(dataset, version) -> GriddedNcTs:
     if dataset.short_name == globals.SMAP_L2:
         reader = GriddedNcOrthoMultiTs(folder_name, ioclass_kws={'read_bulk': True})
 
+    if dataset.short_name == globals.SMOS_SBPCA:
+        reader = GriddedNcOrthoMultiTs(folder_name, ioclass_kws={'read_bulk': True})
+
     if dataset.user and len(dataset.user_dataset.all()):
         file = UserDatasetFile.objects.get(dataset=dataset)
         if file.file_name.endswith('nc') or file.file_name.endswith('nc4'):
@@ -120,7 +123,7 @@ def adapt_timestamp(reader, dataset, version):
             'base_time_reference': '2000-01-01',
         }
 
-    elif dataset.short_name == globals.SMOS_L2:
+    elif dataset.short_name in [globals.SMOS_L2, globals.SMOS_SBPCA]:
         tadapt_kwargs = {
             'time_offset_fields': 'Seconds',
             'time_units': 's',