Merge Candidate for Fields Change

.github/workflows/build-and-deploy.yml

@@ -13,8 +13,23 @@ jobs:
     permissions:
       id-token: write
       contents: read
-    
+
     steps:
+    - name: Checkout CoastNet
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-coastnet
+
+    - name: Checkout Qtrees
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-qtrees
+
+    - name: Checkout OpenOceans++
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-oopp
+
     - name: Checkout SlideRule Repository
       uses: actions/checkout@v3
 
@@ -30,7 +45,7 @@ jobs:
        make cluster-upload-terraform
 
     - name: Set Up Docker Buildx
-      uses: docker/setup-buildx-action@v3    
+      uses: docker/setup-buildx-action@v3
 
     - name: Setup Docker Environment
       run: |

.github/workflows/codeql-analysis.yml

@@ -35,6 +35,21 @@ jobs:
         language: [ 'cpp' ]
 
     steps:
+    - name: Checkout CoastNet
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-coastnet
+
+    - name: Checkout Qtrees
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-qtrees
+
+    - name: Checkout OpenOceans++
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-oopp
+
     - name: Checkout repository
       uses: actions/checkout@v3
 

.github/workflows/self_test.yml

@@ -29,6 +29,21 @@ jobs:
     container: ghcr.io/slideruleearth/sliderule-buildenv:latest
 
     steps:
+    - name: Checkout CoastNet
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-coastnet
+
+    - name: Checkout Qtrees
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-qtrees
+
+    - name: Checkout OpenOceans++
+      uses: actions/checkout@v3
+      with:
+        repository: SlideRuleEarth/ut-ATL24-oopp
+
     - name: Checkout sliderule repository
       uses: actions/checkout@v3
 

clients/python/sliderule/gedi.py

@@ -41,9 +41,6 @@
 # profiling times for each major function
 profiles = {}
 
-# gedi parameters
-ALL_BEAMS = -1
-
 ###############################################################################
 # LOCAL FUNCTIONS
 ###############################################################################

clients/python/sliderule/sliderule.py

@@ -584,12 +584,12 @@ def procoutputfile(parm, rsps):
     # Handle Local Files
     if "open_on_complete" in output and output["open_on_complete"]:
         if output["format"] == "parquet":
-            if "as_geo" in output and not output["as_geo"]:
-                # Return Parquet File as DataFrame
-                return geopandas.pd.read_parquet(path)
-            else:
+            if "as_geo" in output and output["as_geo"]:
                 # Return GeoParquet File as GeoDataFrame
                 return geopandas.read_parquet(path)
+            else:
+                # Return Parquet File as DataFrame
+                return geopandas.pd.read_parquet(path)
         elif output["format"] == "geoparquet":
             # Return Parquet File as DataFrame
             return geopandas.pd.read_parquet(path)
@@ -681,7 +681,7 @@ def todataframe(columns, time_key="time", lon_key="longitude", lat_key="latitude
 def simplifypolygon(parm):
     if "parms" not in parm:
         return
-    
+
     if "cmr" in parm["parms"]:
         polygon = parm["parms"]["cmr"]["polygon"]
     elif "poly" in parm["parms"]:
@@ -702,9 +702,9 @@ def simplifypolygon(parm):
         simplified_polygon.insert(0, point)
 
     if "cmr" not in parm["parms"]:
-        parm["parms"]["cmr"] = {}                    
+        parm["parms"]["cmr"] = {}
     parm["parms"]["cmr"]["polygon"] = simplified_polygon
-    
+
     logger.warning('Using simplified polygon (for CMR request only!), {} points using tolerance of {}'.format(len(simplified_coords), tolerance))
 
 #
@@ -722,13 +722,13 @@ def rethrow():
 #  Initialize
 #
 def init (
-    url=PUBLIC_URL, 
-    verbose=False, 
-    loglevel=logging.INFO, 
-    organization=0, 
-    desired_nodes=None, 
-    time_to_live=60, 
-    bypass_dns=False, 
+    url=PUBLIC_URL,
+    verbose=False,
+    loglevel=logging.INFO,
+    organization=0,
+    desired_nodes=None,
+    time_to_live=60,
+    bypass_dns=False,
     plugins=[],
     trust_env=DEFAULT_TRUST_ENV,
     log_handler=None,
@@ -1414,7 +1414,7 @@ def toregion(source, tolerance=0.0, cellsize=0.01, n_clusters=1):
 
             "poly": [{"lat": <lat1>, "lon": <lon1> }, ...],
 
-            "raster": {"data": <geojson file as string>,
+            "region_mask": {"data": <geojson file as string>,
 
             "clusters": [[{"lat": <lat1>, "lon": <lon1>}, ...], [{"lat": <lat1>, "lon": <lon1>}, ...]] }
 
@@ -1564,8 +1564,8 @@ def toregion(source, tolerance=0.0, cellsize=0.01, n_clusters=1):
         "gdf": gdf,
         "poly": polygon, # convex hull of polygons
         "clusters": clusters, # list of polygon clusters for cmr request
-        "raster": {
-            "data": datafile, # geojson file
+        "region_mask": {
+            "geojson": datafile, # geojson file
             "length": len(datafile), # geojson file length
             "cellsize": cellsize  # units are in crs/projection
         }

clients/python/tests/test_3dep.py

@@ -22,9 +22,9 @@ def test_sample(self, init):
     def test_as_numpy_array(self, init):
         parms = {
             "poly": region,
-            "degrade_flag": 0,
-            "l2_quality_flag": 1,
-            "beam": 0,
+            "degrade_filter": True,
+            "l2_quality_filter": True,
+            "beams": 0,
             "samples": {"3dep": {"asset": "usgs3dep-1meter-dem"}}
         }
         gdf = gedi.gedi04ap(parms, resources=['GEDI04_A_2019123154305_O02202_03_T00174_02_002_02_V002.h5'], as_numpy_array=True)
@@ -38,9 +38,9 @@ def test_as_numpy_array(self, init):
     def test_as_variable(self, init):
         parms = {
             "poly": region,
-            "degrade_flag": 0,
-            "l2_quality_flag": 1,
-            "beam": 0,
+            "degrade_filter": True,
+            "l2_quality_filter": True,
+            "beams": 0,
             "samples": {"3dep": {"asset": "usgs3dep-1meter-dem"}}
         }
         gdf = gedi.gedi04ap(parms, resources=['GEDI04_A_2019123154305_O02202_03_T00174_02_002_02_V002.h5'], as_numpy_array=False)
@@ -51,4 +51,4 @@ def test_as_variable(self, init):
                 for entry in gdf[key]:
                     if (type(entry) != numpy.ndarray) and (not math.isnan(entry)):
                         non_array_count += 1
-        assert non_array_count > 0
+        assert non_array_count > 0

clients/python/tests/test_defaults.py

@@ -1,7 +1,7 @@
 """Tests for sliderule APIs."""
 
 import pytest
-import sliderule
+from sliderule import sliderule, icesat2
 
 #
 # Default parms
@@ -11,9 +11,9 @@ class TestDefaultParms:
     def test_parms(self, init):
         d = sliderule.source("defaults")
         assert init
-        assert d["output"]["as_geo"]
-        assert d["cre"]["timeout"] == 600
-        assert d["samples"]["sampling_algo"] == "NearestNeighbour"
-        assert d["rqst"]["cluster_size_hint"] == 0
-        assert d["icesat2"]["surface_type"] == "SRT_LAND_ICE"
-        assert d["gedi"]["projection"] == "AUTOMATIC"
+        assert d["icesat2"]["output"]["as_geo"] == False, f'icesat2 output has invalid as_geo field: {d["icesat2"]["output"]}'
+        assert d["cre"]["timeout"] == 600, f'container runtime environment has invalid timeout: {d["cre"]}'
+        assert len(d["core"]["samples"]) == 0, f'core has invalid samples parameters: {d["core"]["samples"]}'
+        assert d["core"]["cluster_size_hint"] == 0, f'core has invalid cluster size hint: {d["core"]["cluster_size_hint"]}'
+        assert d["icesat2"]["srt"] == "land_ice", f'icesat2 has invalid surface reference type: {d["icesat2"]["srt"]}'
+        assert d["gedi"]["projection"] == "auto", f'gedi has invalid projection: {d["gedi"]["projection"]}'

clients/python/tests/test_gedi.py

@@ -14,9 +14,9 @@ def test_gedi(self, init):
         region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
         parms = {
             "poly": region["poly"],
-            "degrade_flag": 0,
-            "quality_flag": 1,
-            "beam": 0
+            "degrade_filter": True,
+            "quality_filter": True,
+            "beams": 0
         }
         gdf = gedi.gedi01bp(parms, resources=[resource])
         assert init
@@ -41,9 +41,9 @@ def test_gedi(self, init):
         region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
         parms = {
             "poly": region["poly"],
-            "degrade_flag": 0,
-            "quality_flag": 1,
-            "beam": 0
+            "degrade_filter": True,
+            "quality_filter": True,
+            "beams": 0
         }
         gdf = gedi.gedi02ap(parms, resources=[resource])
         assert init
@@ -90,9 +90,9 @@ def test_gedi(self, init):
         region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
         parms = {
             "poly": region["poly"],
-            "degrade_flag": 0,
-            "l2_quality_flag": 1,
-            "beam": 0
+            "degrade_filter": True,
+            "l2_quality_filter": True,
+            "beams": 0
         }
         gdf = gedi.gedi04ap(parms, resources=[resource])
         assert init

clients/python/tests/test_geojson.py

@@ -15,7 +15,7 @@ def test_atl03(self, init):
             region = sliderule.toregion(os.path.join(TESTDIR, testfile))
             parms = {
                 "poly": region["poly"],
-                "raster": region["raster"],
+                "region_mask": region["region_mask"],
                 "srt": icesat2.SRT_LAND,
                 "cnf": icesat2.CNF_SURFACE_HIGH,
                 "ats": 10.0,

clients/python/tests/test_luaerr.py

@@ -20,26 +20,24 @@ class TestAtl03s:
     def test_badasset(self, init):
         invalid_asset = "invalid-asset"
         rqst = {
-            "resource": [],
-            "parms": {"asset" : "invalid-asset"}
+            "resource": [], # invalid resource (should be resources if a list)
+            "parms": {"asset" : invalid_asset} # bogus asset
         }
         rsps = sliderule.source("atl03s", rqst, stream=True, callbacks=GLOBAL_callbacks)
         assert init
         assert(len(rsps) == 0)
-        assert("invalid asset specified: {}".format(invalid_asset) == GLOBAL_message)
 
 @pytest.mark.network
 class TestAtl06:
     def test_badasset(self, init):
         invalid_asset = "invalid-asset"
         rqst = {
-            "resource": [],
-            "parms": {"asset" : "invalid-asset"}
+            "resource": [], # invalid resource (should be resources if a list)
+            "parms": {"asset" : invalid_asset} # bogus asset
         }
         rsps = sliderule.source("atl06", rqst, stream=True, callbacks=GLOBAL_callbacks)
         assert init
         assert(len(rsps) == 0)
-        assert("invalid asset specified: {}".format(invalid_asset) == GLOBAL_message)
 
     def test_timeout(self, init):
         resource = "ATL03_20220208000041_07291401_005_01.h5"
@@ -50,4 +48,3 @@ def test_timeout(self, init):
         rsps = sliderule.source("atl06", rqst, stream=True, callbacks=GLOBAL_callbacks)
         assert init
         assert(len(rsps) == 0)
-        # assert("{} timed-out after 10 seconds".format(resource) in GLOBAL_message) # non-deterministic