Fix Tests (#20)

* Fix sampling endpoint by removing region setting, because actinia users have a cell limit
* Add required GRASS GIS addon to README.md
* adding some examples
* fix some tests
Co-authored-by: anikaweinmann <[email protected]>

README.md

@@ -31,6 +31,10 @@ python3 setup.py install
 After installation set the plugin name in the actinia core configuration
 and restart the actinia core server.
 
+### Required GRASS GIS Addons
+
+The statistic plugin needs the GRASS GIS addon [t.rast.sample](https://github.com/mundialis/t.rast.sample) to be installed in actinia.
+
 
 ## Testing locally
 

docker/actinia-plugin-tests/Dockerfile

@@ -1,4 +1,4 @@
-FROM mundialis/actinia-core:latest as actinia_test
+FROM mundialis/actinia:latest as actinia_test
 
 LABEL authors="Carmen Tawalika,Anika Weinmann"
 LABEL maintainer="[email protected],[email protected]"
@@ -18,10 +18,10 @@ RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_mic
   rm -f nc_spm_08_micro.zip && \
   mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08
 RUN grass -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84
-RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_mapset_modis2015_2016_lst.zip && \
-  unzip nc_spm_mapset_modis2015_2016_lst.zip && \
-  rm -f nc_spm_mapset_modis2015_2016_lst.zip && \
-  mv  modis_lst /actinia_core/grassdb/nc_spm_08/modis_lst
+RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_mapset_modis2015_2016_lst_grass8.zip && \
+  unzip nc_spm_mapset_modis2015_2016_lst_grass8.zip && \
+  rm -f nc_spm_mapset_modis2015_2016_lst_grass8.zip && \
+  mv modis_lst /actinia_core/grassdb/nc_spm_08/modis_lst
 RUN chown -R 1001:1001 /actinia_core/grassdb/nc_spm_08/modis_lst && chmod -R g+w /actinia_core/grassdb/nc_spm_08/modis_lst
 
 # copy needed files and configs for test

docker/actinia-plugin-tests/actinia-test.cfg

@@ -13,12 +13,6 @@ grass_default_location = nc_spm_08
 plugins = []
 force_https_urls = True
 
-[LIMITS]
-max_cell_limit = 22500000
-process_time_limt = 60
-process_num_limit = 20
-number_of_workers = 3
-
 [REDIS]
 redis_server_url = localhost
 redis_server_port = 6379
@@ -27,17 +21,8 @@ redis_queue_server_port = 6379
 worker_queue_name = actinia_job
 worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log
 
-[LOGGING]
-log_interface = fluentd
-log_fluent_host = fluentd
-log_fluent_port = 24224
-log_level = 1
-
 [MISC]
 tmp_workdir = /actinia_core/workspace/tmp
 download_cache = /actinia_core/workspace/download_cache
 secret_key = token_signing_key_changeme
-
-[MANAGEMENT]
-default_user = user
-default_user_group = group
+save_interim_results = False

examples/area.geojson

@@ -0,0 +1,25 @@
+{
+    "type": "FeatureCollection",
+    "crs": {
+        "type": "name",
+        "properties": {"name": "urn:x-ogc:def:crs:EPSG:3358"}
+    },
+    "features": [
+        {
+            "type": "Feature",
+            "properties": {"fid": "test"},
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [330000.0, 65000.0],
+                        [337000.0, 65000.0],
+                        [337000.0, 647000.0],
+                        [330000.0, 647000.0],
+                        [330000.0, 65000.0]
+                    ]
+                ]
+            }
+        }
+    ]
+}

examples/area2.geojson

@@ -0,0 +1,25 @@
+{
+    "type": "FeatureCollection",
+    "crs": {
+        "type": "name",
+        "properties": {"name": "urn:x-ogc:def:crs:EPSG:3358"}
+    },
+    "features": [
+        {
+            "type": "Feature",
+            "properties": {"fid": "test"},
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [635000.0, 220000.0],
+                        [637000.0, 220000.0],
+                        [637000.0, 221000.0],
+                        [635000.0, 221000.0],
+                        [635000.0, 220000.0]
+                    ]
+                ]
+            }
+        }
+    ]
+}

examples/examples.md

@@ -0,0 +1,52 @@
+# Examples
+```
+BASE_URL="http://localhost:8088/api/v3"
+AUTH='actinia-gdi:actinia-gdi'
+```
+
+## STRDS sampling
+
+Sampling STRDS at point coordinates
+```
+# async
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/modis_lst/strds/LST_Day_monthly/sampling_async -d @points.json
+
+# sync
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/modis_lst/strds/LST_Day_monthly/sampling_sync -d @points.json
+```
+
+Sampling STRDS at point coordinates by filtering the time of the STRDS
+```
+# async
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/modis_lst/strds/LST_Day_monthly/sampling_async -d @points_where.json
+
+# sync
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/modis_lst/strds/LST_Day_monthly/sampling_sync -d @points_where.json
+```
+
+Sampling STRDS at Points in a GeoJson
+```
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/modis_lst/strds/LST_Day_monthly/sampling_sync_geojson -d @points.geojson
+```
+
+Sampling STRDS by area
+```
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/modis_lst/strds/LST_Day_monthly/timestamp/2016-01-01T00:00:00/area_stats_sync -d @area.geojson
+```
+
+## Raster sampling
+Raster statistics of area
+```
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers/landuse96_28m/area_stats_sync -d @area2.geojson
+```
+
+Univar raster statistics of area
+```
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers/elevation/area_stats_univar_sync -d @area2.geojson
+```
+
+
+## Vector sampling
+```
+curl -u ${AUTH} -H 'Content-Type: application/json' -X POST ${BASE_URL}/locations/nc_spm_08/mapsets/PERMANENT/vector_layers/nc_state/sampling_async -d @points2.json
+```

examples/points.geojson

@@ -0,0 +1 @@
+{"type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "urn:x-ogc:def:crs:EPSG:3358"}}, "features": [{"type": "Feature", "properties": {"cat": 1}, "geometry": {"type": "Point", "coordinates": [330000.0, 65000.0]}}, {"type": "Feature", "properties": {"cat": 2}, "geometry": {"type": "Point", "coordinates": [500000.0, 500000.0]}}]}

examples/points.json

@@ -0,0 +1 @@
+{"points": [["a", "330000.0", "65000.0"], ["b", "300000.0", "60000.0"], ["c", "500000.0", "500000.0"]]}

examples/points2.json

@@ -0,0 +1 @@
+{"points": [["p1", "330000.0", "65000.0"],["p2", "331000.0", "649000.0"]]}

examples/points_where.json

@@ -0,0 +1 @@
+{"points": [["a", "330000.0", "65000.0"], ["b", "300000.0", "60000.0"], ["c", "500000.0", "500000.0"]], "where": "start_time >'2016-01-01'"}

setup.cfg

@@ -50,7 +50,7 @@ addopts = tests
 # e.g. --cov-report html (or xml) for html/xml output or --junitxml junit.xml
 # in order to write a coverage file that can be read by Jenkins.
 addopts =
-    --cov actinia_core --cov-report term-missing
+    --cov actinia_statistic_plugin --cov-report term-missing
     --verbose --tb=line -x
 norecursedirs =
     dist

src/actinia_statistic_plugin/endpoints.py

@@ -40,7 +40,7 @@
 __license__ = "GPLv3"
 __author__ = "Sören Gebbert, Markus Neteler"
 __copyright__ = (
-    "Copyright 2016-present, Sören Gebbert and mundialis GmbH & Co. KG"
+    "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG"
 )
 
 

src/actinia_statistic_plugin/ephemeral_raster_area_stats_univar.py

@@ -100,6 +100,9 @@ def _execute(self, location_name, mapset_name, raster_name):
             map_name=raster_name,
         )
         if rdc:
+            # # for debugging
+            # processing = AsyncEphemeralRasterAreaStatsUnivar(rdc)
+            # processing.run()
             enqueue_job(self.job_timeout, start_job, rdc)
 
         return rdc

src/actinia_statistic_plugin/raster_sampling.py

@@ -24,7 +24,7 @@
 __license__ = "GPLv3"
 __author__ = "Markus Neteler"
 __copyright__ = (
-    "Copyright 2022-present, Markus Neteler and mundialis GmbH & Co. KG"
+    "Copyright 2022-2022, Markus Neteler and mundialis GmbH & Co. KG"
 )
 
 

src/actinia_statistic_plugin/response_models.py

@@ -6,7 +6,7 @@
 
 __license__ = "GPLv3"
 __author__ = "Sören Gebbert, Markus Neteler"
-__copyright__ = "Copyright 2016-present, Sören Gebbert, Markus Neteler and "
+__copyright__ = "Copyright 2016-2022, Sören Gebbert, Markus Neteler and "
 "mundialis GmbH & Co. KG"
 
 

src/actinia_statistic_plugin/strds_sampling_geojson.py

@@ -253,6 +253,9 @@ def _execute(self, location_name, mapset_name, strds_name):
             map_name=strds_name,
         )
         if rdc:
+            # # for debugging
+            # processing = AsyncEphemeralSTRDSSamplingGeoJSON(rdc)
+            # processing.run()
             enqueue_job(self.job_timeout, start_job, rdc)
 
         return rdc

src/actinia_statistic_plugin/vector_sampling.py

@@ -23,7 +23,7 @@
 __license__ = "GPLv3"
 __author__ = "Markus Neteler"
 __copyright__ = (
-    "Copyright 2022-present, Markus Neteler and mundialis GmbH & Co. KG"
+    "Copyright 2022-2022, Markus Neteler and mundialis GmbH & Co. KG"
 )
 
 
@@ -193,17 +193,6 @@ def _execute(self):
 
         pc = {
             "list": [
-                {
-                    "id": "g_region",
-                    "module": "g.region",
-                    "inputs": [
-                        {
-                            "param": "vector",
-                            "value": "%s@%s" % (vector_name, self.mapset_name),
-                        }
-                    ],
-                    "flags": "p",
-                },
                 {
                     "id": "v_what",
                     "module": "v.what",

tests/test_raster_area_stats.py

@@ -10,11 +10,14 @@
     from test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX
 
 __license__ = "GPLv3"
-__author__ = "Sören Gebbert"
-__copyright__ = "Copyright 2016, Sören Gebbert"
-__maintainer__ = "Soeren Gebbert"
-__email__ = "[email protected]"
-
+__author__ = "Sören Gebbert, Anika Weinmann"
+__copyright__ = "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co.KG"
+__maintainer__ = "mundialis GmbH & Co. KG"
+
+LOCATION = "nc_spm_08"
+MAPSET = "PERMANENT"
+RASTER = "landuse96_28m"
+RASTER2 = "basin_50K"
 JSON = {
     "type": "FeatureCollection",
     "crs": {
@@ -45,8 +48,8 @@
 class RasterAreaStatsTestCase(ActiniaResourceTestCaseBase):
     def test_async_raster_area_stats_json(self):
         rv = self.server.post(
-            f"{URL_PREFIX}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers"
-            "/landuse96_28m/area_stats_async",
+            f"{URL_PREFIX}/locations/{LOCATION}/mapsets/{MAPSET}/raster_layers"
+            f"/{RASTER}/area_stats_async",
             headers=self.admin_auth_header,
             data=json_dump(JSON),
             content_type="application/json",
@@ -60,8 +63,8 @@ def test_async_raster_area_stats_json(self):
 
     def test_sync_raster_area_stats_1(self):
         rv = self.server.post(
-            f"{URL_PREFIX}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers"
-            "/landuse96_28m/area_stats_sync",
+            f"{URL_PREFIX}/locations/{LOCATION}/mapsets/{MAPSET}/raster_layers"
+            f"/{RASTER}/area_stats_sync",
             headers=self.admin_auth_header,
             data=json_dump(JSON),
             content_type="application/json",
@@ -82,8 +85,8 @@ def test_sync_raster_area_stats_1(self):
     def test_sync_raster_area_stats_2(self):
 
         rv = self.server.post(
-            f"{URL_PREFIX}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers"
-            "/basin_50K/area_stats_sync",
+            f"{URL_PREFIX}/locations/{LOCATION}/mapsets/{MAPSET}/raster_layers"
+            f"/{RASTER2}/area_stats_sync",
             headers=self.admin_auth_header,
             data=json_dump(JSON),
             content_type="application/json",
@@ -103,8 +106,8 @@ def test_sync_raster_area_stats_2(self):
 
     def test_sync_raster_area_stats_error_wrong_content_type(self):
         rv = self.server.post(
-            f"{URL_PREFIX}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers"
-            "/basin_50K/area_stats_sync",
+            f"{URL_PREFIX}/locations/{LOCATION}/mapsets/{MAPSET}/raster_layers"
+            f"/{RASTER}/area_stats_sync",
             headers=self.admin_auth_header,
             data="{}",
             content_type="application/json",
@@ -121,8 +124,8 @@ def test_sync_raster_area_stats_error_wrong_content_type(self):
 
     def test_sync_raster_area_stats_error_wrong_request_missing_json(self):
         rv = self.server.post(
-            f"{URL_PREFIX}/locations/nc_spm_08/mapsets/PERMANENT/raster_layers"
-            "/towns/area_stats_sync",
+            f"{URL_PREFIX}/locations/{LOCATION}/mapsets/{MAPSET}/raster_layers"
+            f"/{RASTER}/area_stats_sync",
             headers=self.admin_auth_header,
             data=None,
             content_type="application/json",