diff --git a/CHANGELOG.md b/CHANGELOG.md
index 61558a080..925d975dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,32 +23,35 @@ https://github.com/actinia-org/actinia-core/compare/4.3.1...main
released from main
...
-## [4.11.0] - 2023-11-02
+## [4.11.0](https://github.com/actinia-org/actinia-core/releases/tag/4.11.0) - 2023-11-02
+
released from main
+
### Added
-* enhance docker installation docs by @mmacata in https://github.com/actinia-org/actinia-core/pull/470
-* mdformat by @mmacata in https://github.com/actinia-org/actinia-core/pull/472
-* Add pre-commit to renovate config by @mmacata in https://github.com/actinia-org/actinia-core/pull/473
-* docs: add more details for user management by @neteler in https://github.com/actinia-org/actinia-core/pull/490
-* Enable import via vsicurl by @mmacata in https://github.com/actinia-org/actinia-core/pull/482
-* Allow separate config for worker Part 1 by @mmacata in https://github.com/actinia-org/actinia-core/pull/376
+
+- enhance docker installation docs by @mmacata in https://github.com/actinia-org/actinia-core/pull/470
+- mdformat by @mmacata in https://github.com/actinia-org/actinia-core/pull/472
+- Add pre-commit to renovate config by @mmacata in https://github.com/actinia-org/actinia-core/pull/473
+- docs: add more details for user management by @neteler in https://github.com/actinia-org/actinia-core/pull/490
+- Enable import via vsicurl by @mmacata in https://github.com/actinia-org/actinia-core/pull/482
+- Allow separate config for worker Part 1 by @mmacata in https://github.com/actinia-org/actinia-core/pull/376
### Fixed
-* mundialis->actinia-org by @mmacata in https://github.com/actinia-org/actinia-core/pull/471
-* Update actions/checkout action to v4 by @renovate in https://github.com/actinia-org/actinia-core/pull/474
-* Update docker/login-action action to v3 by @renovate in https://github.com/actinia-org/actinia-core/pull/477
-* Update docker/build-push-action action to v5 by @renovate in https://github.com/actinia-org/actinia-core/pull/476
-* Update docker/setup-buildx-action action to v3 by @renovate in https://github.com/actinia-org/actinia-core/pull/479
-* Update docker/setup-qemu-action action to v3 by @renovate in https://github.com/actinia-org/actinia-core/pull/480
-* Update docker/metadata-action action to v5 by @renovate in https://github.com/actinia-org/actinia-core/pull/478
-* Update update-version.yml by @mmacata in https://github.com/actinia-org/actinia-core/pull/475
-* Update pyproj in requirements for ubuntu by @mmacata in https://github.com/actinia-org/actinia-core/pull/484
-* chore(deps): update dependency werkzeug and Flask to v3 [security] by @renovate in https://github.com/actinia-org/actinia-core/pull/489
-**Full Changelog**: https://github.com/actinia-org/actinia-core/compare/4.10.0...4.11.0
+- mundialis->actinia-org by @mmacata in https://github.com/actinia-org/actinia-core/pull/471
+- Update actions/checkout action to v4 by @renovate in https://github.com/actinia-org/actinia-core/pull/474
+- Update docker/login-action action to v3 by @renovate in https://github.com/actinia-org/actinia-core/pull/477
+- Update docker/build-push-action action to v5 by @renovate in https://github.com/actinia-org/actinia-core/pull/476
+- Update docker/setup-buildx-action action to v3 by @renovate in https://github.com/actinia-org/actinia-core/pull/479
+- Update docker/setup-qemu-action action to v3 by @renovate in https://github.com/actinia-org/actinia-core/pull/480
+- Update docker/metadata-action action to v5 by @renovate in https://github.com/actinia-org/actinia-core/pull/478
+- Update update-version.yml by @mmacata in https://github.com/actinia-org/actinia-core/pull/475
+- Update pyproj in requirements for ubuntu by @mmacata in https://github.com/actinia-org/actinia-core/pull/484
+- chore(deps): update dependency werkzeug and Flask to v3 \[security\] by @renovate in https://github.com/actinia-org/actinia-core/pull/489
-generated with `gh api repos/actinia-org/actinia-core/releases/generate-notes -f tag_name=4.11.0 -f target_commitish=main -q .body`
+**Full Changelog**: https://github.com/actinia-org/actinia-core/compare/4.10.0...4.11.0
+generated with `gh api repos/actinia-org/actinia-core/releases/generate-notes -f tag_name=4.11.0 -f target_commitish=main -q .body`
## [4.10.0](https://github.com/actinia-org/actinia-core/releases/tag/4.10.0) - 2023-08-31
diff --git a/docs/docs/index.md b/docs/docs/index.md
index e1062112f..e027ca2cb 100644
--- a/docs/docs/index.md
+++ b/docs/docs/index.md
@@ -1,6 +1,7 @@
# Actinia - The GRASS GIS REST API
+
Fork Me On GitHub
diff --git a/src/actinia_core/processing/actinia_processing/ephemeral_processing.py b/src/actinia_core/processing/actinia_processing/ephemeral_processing.py
index fd8d0a7bc..15bbb9be1 100644
--- a/src/actinia_core/processing/actinia_processing/ephemeral_processing.py
+++ b/src/actinia_core/processing/actinia_processing/ephemeral_processing.py
@@ -192,6 +192,7 @@ def __init__(self, rdc):
self.temp_mapset_path = None
self.ginit = None
+ self.ginit_tmpfiles = list()
# Successfully finished message
self.finish_message = "Processing successfully finished"
@@ -1203,6 +1204,154 @@ def _cleanup(self):
and os.path.isdir(self.temp_grass_data_base)
):
shutil.rmtree(self.temp_grass_data_base, ignore_errors=True)
+ if self.ginit_tmpfiles:
+ for tmpfile in self.ginit_tmpfiles:
+ try:
+ os.remove(tmpfile)
+ except Exception as e:
+ self.message_logger.debug(
+ f"Temporary file {tmpfile} can't be removed: {e}"
+ )
+
+ def _check_pixellimit_rimport(self, process_executable_params):
+ """Check the current r.import command against the user cell limit.
+
+ Raises:
+ This method will raise an AsyncProcessError exception
+
+ """
+ rimport_inp = [x for x in process_executable_params if "input=" in x][
+ 0
+ ].split("=")[1]
+ rimport_out = [x for x in process_executable_params if "output=" in x][
+ 0
+ ].split("=")[1]
+ vrt_out = f"{rimport_out}_{os.getpid()}_tmp.vrt"
+ self.ginit_tmpfiles.append(vrt_out)
+
+ # define extent_region if set (otherwise empty list)
+ extent_region = [
+ x for x in process_executable_params if "extent=" in x
+ ]
+
+ # build VRT of rimport input
+ gdabuildvrt_params = list()
+ # if extent=region set, vrt only for region, not complete input
+ if extent_region:
+ # first query region extents
+ errorid, stdout_gregion, stderr_gregion = self.ginit.run_module(
+ "g.region", ["-ug"]
+ )
+ if errorid != 0:
+ raise AsyncProcessError(
+ "Unable to check the computational region size"
+ )
+ # parse region extents for creation of vrt (-te flag from gdalbuildvrt)
+ list_out_gregion = stdout_gregion.split("\n")
+ gdabuildvrt_params.append("-te")
+ gdabuildvrt_params.append(list_out_gregion[4]) # xmin/w
+ gdabuildvrt_params.append(list_out_gregion[3]) # ymin/s
+ gdabuildvrt_params.append(list_out_gregion[5]) # xmax/e
+ gdabuildvrt_params.append(list_out_gregion[2]) # ymax/n
+ # out and input for gdalbuildvrt
+ gdabuildvrt_params.append(vrt_out)
+ gdabuildvrt_params.append(rimport_inp)
+ # build vrt with previous defined parameters
+ (
+ errorid,
+ stdout_gdalbuildvrt,
+ stderr_gdalbuildvrt,
+ ) = self.ginit.run_module("/usr/bin/gdalbuildvrt", gdabuildvrt_params)
+
+ # gdalinfo for created vrt
+ gdalinfo_params = [vrt_out]
+ errorid, stdout_gdalinfo, stderr_gdalinfo = self.ginit.run_module(
+ "/usr/bin/gdalinfo", gdalinfo_params
+ )
+ # parse "Size" output of gdalinfo
+ rastersize_list = (
+ stdout_gdalinfo.split("Size is")[1].split("\n")[0].split(",")
+ )
+ # size = x-dim*y-dim
+ rastersize_x = int(rastersize_list[0])
+ rastersize_y = int(rastersize_list[1])
+ rastersize = rastersize_x * rastersize_y
+
+ # if different import/reprojection resolution set:
+ rimport_res = [
+ x for x in process_executable_params if "resolution=" in x
+ ]
+ res_val = None
+ # If raster exceeds cell limit already in original resolution, next part can be skipped
+ if rimport_res and (rastersize < self.cell_limit):
+ # determine estimated resolution
+ errorid, stdout_estres, stderr_estres = self.ginit.run_module(
+ "r.import", [vrt_out, "-e"]
+ )
+ if "Estimated" in stderr_estres:
+ # if data in different projection get rest_est with output of r.import -e
+ res_est = float(stderr_estres.split("\n")[-2].split(":")[1])
+ else:
+ # if data in same projection can use gdalinfo output
+ res_xy = (
+ stdout_gdalinfo.split("Pixel Size = (")[1]
+ .split(")\n")[0]
+ .split(",")
+ )
+ # get estimated resolution
+ # (analoug as done within r.import -e: estres = math.sqrt((n - s) * (e - w) / cells))
+ res_est = math.sqrt(abs(float(res_xy[0]) * float(res_xy[1])))
+ # determine set resolution value
+ resolution = rimport_res[0].split("=")[1]
+ if resolution == "value":
+ res_val = [
+ float(
+ [
+ x
+ for x in process_executable_params
+ if "resolution_value=" in x
+ ][0].split("=")[1]
+ )
+ ] * 2
+ elif resolution == "region":
+ # if already queried above reuse, otherwise execute g.region command
+ try:
+ stdout_gregion
+ except Exception:
+ (
+ errorid,
+ stdout_gregion,
+ stderr_gregion,
+ ) = self.ginit.run_module("g.region", ["-ug"])
+ res_val_ns = float(
+ [x for x in stdout_gregion.split("\n") if "nsres=" in x][
+ 0
+ ].split("=")[1]
+ )
+ res_val_ew = float(
+ [x for x in stdout_gregion.split("\n") if "ewres=" in x][
+ 0
+ ].split("=")[1]
+ )
+ res_val = [res_val_ns, res_val_ew]
+ if res_val:
+ if (res_val[0] < res_est) | (res_val[1] < res_est):
+ # only check if smaller resolution set
+ res_change_x = res_est / res_val[1]
+ res_change_y = res_est / res_val[0]
+ # approximate raster size after resampling
+ # by using factor of changed resolution
+ rastersize = (
+ rastersize_x * res_change_x * rastersize_y * res_change_y
+ )
+
+ # compare estimated raster output size with pixel limit
+ # and raise exception if exceeded
+ if rastersize > self.cell_limit:
+ raise AsyncProcessError(
+ "Processing pixel limit exceeded for raster import. "
+ "Please set e.g. region smaller."
+ )
def _check_reset_region(self):
"""Check the current region settings against the user cell limit.
@@ -1479,10 +1628,14 @@ def _run_module(self, process, poll_time=0.05):
)
self._send_resource_update(message)
+ # Check pixel limit for r.import operations
+ if process.executable == "r.import":
+ self._check_pixellimit_rimport(process.executable_params)
+
# Check reset region if a g.region call was present in the process
# chain. By default the initial value of last_module is "g.region" to
- # assure for first run of a process from the process chain, the
- # region settings are evaluated
+ # assure for first run of a process from the process chain, the region
+ # settings are evaluated
if (
self.last_module == "g.region"
and process.skip_permission_check is False
diff --git a/tests/test_download_cache.py b/tests/test_download_cache.py
index 9f4badc4d..fc7daebab 100644
--- a/tests/test_download_cache.py
+++ b/tests/test_download_cache.py
@@ -136,7 +136,7 @@ def test_download_cache(self):
# "HTML status code is wrong %i" % rv.status_code,
# )
# self.assertEqual(
- # rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
+ # rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
# )
# TODO: configure wrong download cache path differently (via config file)
@@ -158,7 +158,7 @@ def test_download_cache(self):
# "HTML status code is wrong %i" % rv.status_code,
# )
# self.assertEqual(
- # rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
+ # rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
# )
diff --git a/tests/test_raster_import_pixellimit.py b/tests/test_raster_import_pixellimit.py
new file mode 100644
index 000000000..779364d36
--- /dev/null
+++ b/tests/test_raster_import_pixellimit.py
@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+#######
+# actinia-core - an open source REST API for scalable, distributed, high
+# performance processing of geographical data that uses GRASS GIS for
+# computational tasks. For details, see https://actinia.mundialis.de/
+#
+# Copyright (c) 2023 Lina Krisztian and mundialis GmbH & Co. KG
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+#######
+
+"""
+Tests: Import raster with pixellimit check
+"""
+import unittest
+from flask.json import dumps as json_dumps
+
+try:
+ from .test_resource_base import (
+ ActiniaResourceTestCaseBase,
+ URL_PREFIX,
+ additional_external_data,
+ )
+except Exception:
+ from test_resource_base import (
+ ActiniaResourceTestCaseBase,
+ URL_PREFIX,
+ additional_external_data,
+ )
+
+__license__ = "GPLv3"
+__author__ = "Lina Krisztian"
+__copyright__ = "Copyright 2023, mundialis GmbH & Co. KG"
+__maintainer__ = "mundialis GmbH & Co. KG"
+
+
+class ImportRasterLayerPixellimitTestCase(ActiniaResourceTestCaseBase):
+ location = "nc_spm_08"
+ tmp_mapset = "mapset_rasterimport_pixellimit"
+ endpoint = f"/locations/{location}/mapsets/{tmp_mapset}/processing_async"
+ rimport_inp = "elevation"
+ # import resolution with which the process should fail:
+ rimport_res_fail = 0.1
+
+ def setUp(self):
+ # create new temp mapset
+ super(ImportRasterLayerPixellimitTestCase, self).setUp()
+ self.create_new_mapset(self.tmp_mapset, location_name=self.location)
+
+ def tearDown(self):
+ # delete mapset
+ self.delete_mapset(self.tmp_mapset, location_name=self.location)
+ super(ImportRasterLayerPixellimitTestCase, self).tearDown()
+
+ def test_pixellimit_allowed(self):
+ """
+ Test import of raster, for which pixellimit is not reached
+ and therefore allowed
+ """
+ raster_url = additional_external_data[self.rimport_inp]
+ raster = self.rimport_inp
+ process_chain = {
+ "version": 1,
+ "list": [
+ {
+ "id": "1",
+ "module": "r.import",
+ "inputs": [
+ {
+ "param": "input",
+ "value": raster_url,
+ },
+ {
+ "param": "output",
+ "value": raster,
+ },
+ ],
+ },
+ ],
+ }
+ rv = self.server.post(
+ URL_PREFIX + self.endpoint,
+ headers=self.admin_auth_header,
+ data=json_dumps(process_chain),
+ content_type="application/json",
+ )
+ # Import should succeed
+ self.waitAsyncStatusAssertHTTP(
+ rv,
+ headers=self.admin_auth_header,
+ http_status=200,
+ status="finished",
+ )
+
+ def test_pixellimit_not_allowed(self):
+ """
+ Test import of raster, for which pixellimit is reached
+ and therefore not allowed
+ """
+ raster_url = additional_external_data[self.rimport_inp]
+ raster = self.rimport_inp
+ process_chain = {
+ "version": 1,
+ "list": [
+ {
+ "id": "1",
+ "module": "r.import",
+ "inputs": [
+ {
+ "param": "input",
+ "value": raster_url,
+ },
+ {
+ "param": "output",
+ "value": raster,
+ },
+ {
+ "param": "resolution",
+ "value": "value",
+ },
+ {
+ "param": "resolution_value",
+ "value": f"{self.rimport_res_fail}",
+ },
+ ],
+ },
+ ],
+ }
+ rv = self.server.post(
+ URL_PREFIX + self.endpoint,
+ headers=self.admin_auth_header,
+ data=json_dumps(process_chain),
+ content_type="application/json",
+ )
+ # Import should fail with certain message (due too high resolution)
+ resp = self.waitAsyncStatusAssertHTTP(
+ rv,
+ headers=self.admin_auth_header,
+ http_status=400,
+ status="error",
+ )
+ self.assertTrue(
+ "Processing pixel limit exceeded for raster import"
+ in resp["exception"]["message"]
+ )
+
+ def test_pixellimit_importer(self):
+ """
+ Test import of raster with the importer
+ with pixellimit not reached and therefore allowed
+ """
+ raster_url = additional_external_data[self.rimport_inp]
+ raster = self.rimport_inp
+ process_chain = {
+ "version": 1,
+ "list": [
+ {
+ "id": "1",
+ "module": "importer",
+ "inputs": [
+ {
+ "import_descr": {
+ "source": raster_url,
+ "type": "raster",
+ },
+ "param": "map",
+ "value": raster,
+ },
+ ],
+ },
+ ],
+ }
+ rv = self.server.post(
+ URL_PREFIX + self.endpoint,
+ headers=self.admin_auth_header,
+ data=json_dumps(process_chain),
+ content_type="application/json",
+ )
+ # Import should succeed
+ self.waitAsyncStatusAssertHTTP(
+ rv,
+ headers=self.admin_auth_header,
+ http_status=200,
+ status="finished",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py
index 9a8b7ef61..1058065f2 100644
--- a/tests/test_resource_base.py
+++ b/tests/test_resource_base.py
@@ -67,6 +67,7 @@
"geology_30m_tif": f"{base_url_data}/geology_30m.tif",
"geology_30m_zip": f"{base_url_data}/geology_30m.zip",
"pointInBonn": f"{base_url_data}/pointInBonn.geojson",
+ "elevation": f"{base_url_data}/elevation.tif",
}