From 42aae87860c3da341a247873311c9dff2516ce93 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Tue, 22 Mar 2022 09:32:49 +0100 Subject: [PATCH] New vector layer sampling endpoint "sampling_sync" (#8) * Draft: New vector layer sampling endpoint "sampling_sync" This PR adds a new endpoint `ACTINIA_URL/locations/nc_spm_08/mapsets/PERMANENT/vector_layers/zipcodes_wake/sampling_sync` which somewhat corresponds to the existing timeseries entpoint "Sample a strds by point coordinates, asynchronous call" (see https://redocly.github.io/redoc/?url=https://actinia.mundialis.de/latest/swagger.json#tag/STRDS-Sampling/paths/~1locations~1{location_name}~1mapsets~1{mapset_name}~1strds~1{strds_name}~1sampling_async/post) as well as the raster_layers sampling_sync endpoint proposed in #7. Co-authored-by: Anika Weinmann --- src/actinia_statistic_plugin/endpoints.py | 28 ++- .../response_models.py | 158 ++++++++++++ .../vector_sampling.py | 226 ++++++++++++++++++ tests/test_vector_sample.py | 118 +++++++++ 4 files changed, 519 insertions(+), 11 deletions(-) create mode 100644 src/actinia_statistic_plugin/vector_sampling.py create mode 100644 tests/test_vector_sample.py diff --git a/src/actinia_statistic_plugin/endpoints.py b/src/actinia_statistic_plugin/endpoints.py index 3c9b933..561995e 100644 --- a/src/actinia_statistic_plugin/endpoints.py +++ b/src/actinia_statistic_plugin/endpoints.py @@ -32,8 +32,11 @@ SyncEphemeralRasterSamplingResource, ) -# from .raster_sampling_geojson import AsyncEphemeralRasterSamplingGeoJSONResource, \ -# SyncEphemeralRasterSamplingGeoJSONResource + +from .vector_sampling import ( + AsyncEphemeralVectorSamplingResource, + SyncEphemeralVectorSamplingResource, +) __license__ = "GPLv3" @@ -130,14 +133,17 @@ def create_endpoints(flask_api): flask_api.add_resource( SyncEphemeralRasterSamplingResource, "/locations//mapsets/" - "/raster_layers/" + "/raster_layers/", + ) + flask_api.add_resource( + AsyncEphemeralVectorSamplingResource, + "/locations//mapsets/" + "/vector_layers/" + "/sampling_async", + ) + flask_api.add_resource( + SyncEphemeralVectorSamplingResource, + "/locations//mapsets/" + "/vector_layers/" "/sampling_sync", ) - - -# flask_api.add_resource(AsyncEphemeralRasterSamplingGeoJSONResource, '/locations//mapsets/' -# '/raster_layers/' -# '/sampling_async_geojson') -# flask_api.add_resource(SyncEphemeralRasterSamplingGeoJSONResource, '/locations//mapsets/' -# '/raster_layers/' -# '/sampling_sync_geojson') diff --git a/src/actinia_statistic_plugin/response_models.py b/src/actinia_statistic_plugin/response_models.py index 71ec813..44e34e9 100644 --- a/src/actinia_statistic_plugin/response_models.py +++ b/src/actinia_statistic_plugin/response_models.py @@ -763,3 +763,161 @@ class RasterSamplingResponseModel(ProcessingResponseModel): }, "user_id": "actinia-gdi", } + + +class VectorSamplingResponseModel(ProcessingResponseModel): + """Response schema for a vector map sampling result. + This schema is a derivative of the ProcessingResponseModel that defines a different + *process_results* schema. + """ + + type = "object" + properties = deepcopy(ProcessingResponseModel.properties) + properties["process_results"] = {} + properties["process_results"]["type"] = "array" + properties["process_results"]["items"] = { + "type": "array", + "items": {"type": "string", "minItems": 3}, + } + required = deepcopy(ProcessingResponseModel.required) + example = { + "accept_datetime": "2022-03-17 13:41:43.981103", + "accept_timestamp": 1647524503.9810963, + "api_info": { + "endpoint": "syncephemeralvectorsamplingresource", + "method": "POST", + "path": f"{URL_PREFIX}/locations/nc_spm_08/mapsets/PERMANENT/vector_layers/zipcodes_wake/sampling_sync", + "request_url": f"http://localhost{URL_PREFIX}//locations/nc_spm_08/mapsets/PERMANENT/vector_layers/zipcodes_wake/sampling_sync", + }, + "datetime": "2022-03-17 13:41:44.467395", + "http_code": 200, + "message": "Processing successfully finished", + "process_chain_list": [ + { + "list": [ + { + "flags": "p", + "id": "g_region", + "inputs": [ + {"param": "vector", "value": "zipcodes_wake@PERMANENT"} + ], + "module": "g.region", + }, + { + "flags": "ag", + "id": "v_what", + "inputs": [ + {"param": "map", "value": "zipcodes_wake@PERMANENT"}, + { + "param": "coordinates", + "value": "638684.0,220210.0,635676.0,226371.0", + }, + ], + "module": "v.what", + "stdout": {"delimiter": "|", "format": "list", "id": "info"}, + }, + ], + "version": "1", + } + ], + "process_log": [ + { + "executable": "g.region", + "id": "g_region", + "mapset_size": 485, + "parameter": ["vector=zipcodes_wake@PERMANENT", "-p"], + "return_code": 0, + "run_time": 0.10039043426513672, + "stderr": [""], + "stdout": "projection: 99 (Lambert Conformal Conic)\nzone: 0\ndatum: nad83\nellipsoid: a=6378137 es=0.006694380022900787\nnorth: 258102.57214598\nsouth: 196327.52090104\nwest: 610047.86645109\neast: 677060.680666\nnsres: 498.18589714\newres: 500.09562847\nrows: 124\ncols: 134\ncells: 16616\n", + }, + { + "executable": "v.what", + "id": "v_what", + "mapset_size": 485, + "parameter": [ + "map=zipcodes_wake@PERMANENT", + "coordinates=638684.0,220210.0,635676.0,226371.0", + "-ag", + ], + "return_code": 0, + "run_time": 0.10032153129577637, + "stderr": [""], + "stdout": "East=638684\nNorth=220210\n\nMap=zipcodes_wake\nMapset=PERMANENT\nType=Area\nSq_Meters=130875884.223\nHectares=13087.588\nAcres=32340.135\nSq_Miles=50.5315\nLayer=1\nCategory=40\nDriver=sqlite\nDatabase=/actinia_core/workspace/temp_db/gisdbase_5ce6c4cf9b8f47628f816e89b7767819/nc_spm_08/PERMANENT/sqlite/sqlite.db\nTable=zipcodes_wake\nKey_column=cat\ncat=40\nOBJECTID=286\nWAKE_ZIPCO=1285870010.66\nPERIMETER=282815.79339\nZIPCODE_=37\nZIPCODE_ID=66\nZIPNAME=RALEIGH\nZIPNUM=27603\nZIPCODE=RALEIGH_27603\nNAME=RALEIGH\nSHAPE_Leng=285693.495599\nSHAPE_Area=1408742751.36\nEast=635676\nNorth=226371\n\nMap=zipcodes_wake\nMapset=PERMANENT\nType=Area\nSq_Meters=63169356.527\nHectares=6316.936\nAcres=15609.488\nSq_Miles=24.3898\nLayer=1\nCategory=42\nDriver=sqlite\nDatabase=/actinia_core/workspace/temp_db/gisdbase_5ce6c4cf9b8f47628f816e89b7767819/nc_spm_08/PERMANENT/sqlite/sqlite.db\nTable=zipcodes_wake\nKey_column=cat\ncat=42\nOBJECTID=298\nWAKE_ZIPCO=829874917.625\nPERIMETER=230773.26059\nZIPCODE_=39\nZIPCODE_ID=2\nZIPNAME=RALEIGH\nZIPNUM=27606\nZIPCODE=RALEIGH_27606\nNAME=RALEIGH\nSHAPE_Leng=212707.32257\nSHAPE_Area=679989401.948\n", + }, + ], + "process_results": [ + { + "p1": { + "Acres": "32340.135", + "Category": "40", + "Database": "/actinia_core/workspace/temp_db/gisdbase_5ce6c4cf9b8f47628f816e89b7767819/nc_spm_08/PERMANENT/sqlite/sqlite.db", + "Driver": "sqlite", + "East": "638684", + "Hectares": "13087.588", + "Key_column": "cat", + "Layer": "1", + "Map": "zipcodes_wake", + "Mapset": "PERMANENT", + "NAME": "RALEIGH", + "North": "220210", + "OBJECTID": "286", + "PERIMETER": "282815.79339", + "SHAPE_Area": "1408742751.36", + "SHAPE_Leng": "285693.495599", + "Sq_Meters": "130875884.223", + "Sq_Miles": "50.5315", + "Table": "zipcodes_wake", + "Type": "Area", + "WAKE_ZIPCO": "1285870010.66", + "ZIPCODE": "RALEIGH_27603", + "ZIPCODE_": "37", + "ZIPCODE_ID": "66", + "ZIPNAME": "RALEIGH", + "ZIPNUM": "27603", + "cat": "40", + } + }, + { + "p2": { + "Acres": "15609.488", + "Category": "42", + "Database": "/actinia_core/workspace/temp_db/gisdbase_5ce6c4cf9b8f47628f816e89b7767819/nc_spm_08/PERMANENT/sqlite/sqlite.db", + "Driver": "sqlite", + "East": "635676", + "Hectares": "6316.936", + "Key_column": "cat", + "Layer": "1", + "Map": "zipcodes_wake", + "Mapset": "PERMANENT", + "NAME": "RALEIGH", + "North": "226371", + "OBJECTID": "298", + "PERIMETER": "230773.26059", + "SHAPE_Area": "679989401.948", + "SHAPE_Leng": "212707.32257", + "Sq_Meters": "63169356.527", + "Sq_Miles": "24.3898", + "Table": "zipcodes_wake", + "Type": "Area", + "WAKE_ZIPCO": "829874917.625", + "ZIPCODE": "RALEIGH_27606", + "ZIPCODE_": "39", + "ZIPCODE_ID": "2", + "ZIPNAME": "RALEIGH", + "ZIPNUM": "27606", + "cat": "42", + } + }, + ], + "progress": {"num_of_steps": 2, "step": 2}, + "resource_id": "resource_id-6527a077-a74d-4195-a44c-90a75692bd22", + "status": "finished", + "time_delta": 0.4863121509552002, + "timestamp": 1647524504.4673853, + "urls": { + "resources": [], + "status": f"http://localhost{URL_PREFIX}//resources/actinia-gdi/resource_id-6527a077-a74d-4195-a44c-90a75692bd22", + }, + "user_id": "actinia-gdi", + } diff --git a/src/actinia_statistic_plugin/vector_sampling.py b/src/actinia_statistic_plugin/vector_sampling.py new file mode 100644 index 0000000..e30cdff --- /dev/null +++ b/src/actinia_statistic_plugin/vector_sampling.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +""" +Perform vector map sampling on a vector map layer based on input points. +""" + +import pickle +import tempfile +import json +from flask import jsonify, make_response +from copy import deepcopy +from flask_restful_swagger_2 import swagger +from flask_restful_swagger_2 import Schema +from actinia_core.models.response_models import ProcessingErrorResponseModel +from actinia_core.rest.ephemeral_processing import EphemeralProcessing +from actinia_core.rest.resource_base import ResourceBase +from actinia_core.core.common.redis_interface import enqueue_job +from flask.json import dumps +from actinia_core.core.common.app import auth +from actinia_core.core.common.api_logger import log_api_call +from .response_models import VectorSamplingResponseModel + + +__license__ = "GPLv3" +__author__ = "Sören Gebbert, Markus Neteler" +__copyright__ = "Copyright 2016-present, Sören Gebbert and mundialis GmbH & Co. KG" + + +class PointListModel(Schema): + """This schema defines the JSON input of the vector sampling resource""" + + type = "object" + properties = { + "points": { + "type": "array", + "items": { + "type": "array", + "items": {"type": "string", "maxItems": 3, "minItems": 3}, + }, + "description": "A list of coordinate points with unique ids [(id, x, y), (id, x, y), (id, x, y)]", + } + } + example = {"points": [["a", "1", "1"], ["b", "2", "2"], ["c", "3", "3"]]} + required = ["points"] + + +SCHEMA_DOC = { + "tags": ["Vector Sampling"], + "description": "Spatial sampling of a vector dataset with vector points. The vector points must " + "be in the same coordinate reference system as the location that contains the " + "vector dataset. The result of the sampling is located in the resource response" + "JSON document after the processing was finished, " + "as a list of values for each vector point. " + "Minimum required user role: user.", + "consumes": ["application/json"], + "parameters": [ + { + "name": "location_name", + "description": "The location name", + "required": True, + "in": "path", + "type": "string", + }, + { + "name": "mapset_name", + "description": "The name of the mapset that contains the required vector map layer", + "required": True, + "in": "path", + "type": "string", + }, + { + "name": "vector_name", + "description": "The name of the vector map layer to perform the vector map sampling from", + "required": True, + "in": "path", + "type": "string", + }, + { + "name": "points", + "description": "The sampling point array [[id, x, y],[id, x, y]]. " + "The coordinates of the sampling points must be in the same coordinate reference system as the location " + "that contains the vector dataset.", + "required": True, + "in": "body", + "schema": PointListModel, + }, + ], + "responses": { + "200": { + "description": "The result of the vector map sampling", + "schema": VectorSamplingResponseModel, + }, + "400": { + "description": "The error message and a detailed log why vector sampling did not succeed", + "schema": ProcessingErrorResponseModel, + }, + }, +} + + +class AsyncEphemeralVectorSamplingResource(ResourceBase): + """Perform vector map sampling on a vector map layer based on input points, asynchronous call""" + + decorators = [log_api_call, auth.login_required] + + def _execute(self, location_name, mapset_name, vector_name): + + rdc = self.preprocess( + has_json=True, + has_xml=False, + location_name=location_name, + mapset_name=mapset_name, + map_name=vector_name, + ) + if rdc: + enqueue_job(self.job_timeout, start_job, rdc) + + return rdc + + @swagger.doc(deepcopy(SCHEMA_DOC)) + def post(self, location_name, mapset_name, vector_name): + """Perform vector map sampling on a vector map layer based on input points asynchronously""" + self._execute(location_name, mapset_name, vector_name) + html_code, response_model = pickle.loads(self.response_data) + return make_response(jsonify(response_model), html_code) + + +class SyncEphemeralVectorSamplingResource(AsyncEphemeralVectorSamplingResource): + """Perform vector map sampling on a vector map layer based on input points, synchronous call""" + + decorators = [log_api_call, auth.login_required] + + @swagger.doc(deepcopy(SCHEMA_DOC)) + def post(self, location_name, mapset_name, vector_name): + """Perform vector map sampling on a vector map layer based on input points synchronously""" + check = self._execute(location_name, mapset_name, vector_name) + if check is not None: + http_code, response_model = self.wait_until_finish() + else: + http_code, response_model = pickle.loads(self.response_data) + return make_response(jsonify(response_model), http_code) + + +def start_job(*args): + processing = AsyncEphemeralVectorSampling(*args) + processing.run() + + +class AsyncEphemeralVectorSampling(EphemeralProcessing): + """Sample a vector map at vector points""" + + def __init__(self, *args): + EphemeralProcessing.__init__(self, *args) + self.response_model_class = VectorSamplingResponseModel + + def _execute(self): + + self._setup() + + # Points are stored in self.request_data + vector_name = self.map_name + points = self.request_data["points"] + + if not points or len(points) == 0: + raise AsyncProcessError("Empty coordinate list") + + coordinates_string = "" + for tuple in points: + if len(tuple) != 3: + raise AsyncProcessError("Wrong number of coordinate entries") + + id, x, y = tuple + coordinates_string += "%s,%s," % (x, y) + + pc = { + "list": [ + { + "id": "g_region", + "module": "g.region", + "inputs": [ + { + "param": "vector", + "value": "%s@%s" % (vector_name, self.mapset_name), + } + ], + "flags": "p", + }, + { + "id": "v_what", + "module": "v.what", + "inputs": [ + { + "param": "map", + "value": "%s@%s" % (vector_name, self.mapset_name), + }, + {"param": "coordinates", "value": coordinates_string[:-1]}, + ], + "stdout": {"id": "info", "format": "list", "delimiter": "|"}, + "flags": "ag", + }, + ], + "version": "1", + } + + self.request_data = pc + + # Run the process chain + EphemeralProcessing._execute(self, skip_permission_check=True) + + count = -1 + output_list = [] + # Convert the result of v.what into actinia response format (list of points + # with point ID, coordinate pair and vector map attributes) + for entry in self.module_results["info"]: + if "=" in entry: + key, val = entry.split("=") + if key == "East": + count += 1 + if "point" in locals(): + output_list.append(point) + point = {points[count][0]: {key: val}} + else: + point[points[count][0]][key] = val + + output_list.append(point) + + self.module_results = output_list diff --git a/tests/test_vector_sample.py b/tests/test_vector_sample.py new file mode 100644 index 0000000..4939ffa --- /dev/null +++ b/tests/test_vector_sample.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +import unittest +import time +from pprint import pprint +from flask.json import loads as json_load +from flask.json import dumps as json_dump + +try: + from .test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX +except: + from test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX + + +__license__ = "GPLv3" +__author__ = "Markus Neteler" +__copyright__ = "Copyright 2016-present, Markus Neteler and mundialis GmbH & Co. KG" + + +class RasterTestCase(ActiniaResourceTestCaseBase): + + # ################### Raster SAMPLING ################################### + + def test_async_sampling(self): + + rv = self.server.post( + URL_PREFIX + + "/locations/nc_spm_08/mapsets/PERMANENT/vector_layers/zipcodes_wake/" + "sampling_async", + headers=self.user_auth_header, + data=json_dump( + { + "points": [ + ["a", "638684.0", "220210.0"], + ["b", "635676.0", "226371.0"], + ] + } + ), + content_type="application/json", + ) + + pprint(json_load(rv.data)) + self.assertEqual( + rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code + ) + self.assertEqual( + rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype + ) + + resp = json_load(rv.data) + + rv_user_id = resp["user_id"] + rv_resource_id = resp["resource_id"] + + while True: + rv = self.server.get( + URL_PREFIX + "/resources/%s/%s" % (rv_user_id, rv_resource_id), + headers=self.user_auth_header, + ) + print(rv.data) + resp = json_load(rv.data) + if resp["status"] == "finished" or resp["status"] == "error": + break + time.sleep(0.2) + + self.assertEquals(resp["status"], "finished") + self.assertEqual( + rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code + ) + + value_list = json_load(rv.data)["process_results"] + + self.assertEqual(value_list[0][0], "easting") + self.assertEqual(value_list[0][1], "northing") + self.assertEqual(value_list[0][2], "site_name") + self.assertEqual(value_list[0][3], "zipcodes_wake") + self.assertEqual(value_list[0][4], "zipcodes_wake_label") + self.assertEqual(value_list[0][5], "zipcodes_wake_color") + + time.sleep(1) + + def test_sync_sampling(self): + + rv = self.server.post( + URL_PREFIX + + "/locations/nc_spm_08/mapsets/PERMANENT/vector_layers/zipcodes_wake/" + "sampling_sync", + headers=self.user_auth_header, + data=json_dump( + { + "points": [ + ["p1", "638684.0", "220210.0"], + ["p2", "635676.0", "226371.0"], + ] + } + ), + content_type="application/json", + ) + + pprint(json_load(rv.data)) + self.assertEqual( + rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code + ) + self.assertEqual( + rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype + ) + + value_list = json_load(rv.data)["process_results"] + + self.assertIn("East", value_list["p1"]) + self.assertIn("North", value_list["p2"]) + self.assertIn("ZIPCODE", value_list["p2"]) + self.assertEqual(value_list["p2"]["ZIPCODE"], "RALEIGH_27606") + + time.sleep(1) + + +if __name__ == "__main__": + unittest.main()