From 58d1d7bc3439873a32d47ef5a9eba6def6f6bf76 Mon Sep 17 00:00:00 2001 From: Anika Bettge Date: Fri, 19 Jul 2019 12:20:25 +0200 Subject: [PATCH 01/24] change in readme and adding mounted skript folder --- docker/README.md | 6 ++++++ docker/docker-compose-dev.yml | 1 + 2 files changed, 7 insertions(+) diff --git a/docker/README.md b/docker/README.md index 06ecfa49e..5e1989d57 100644 --- a/docker/README.md +++ b/docker/README.md @@ -3,11 +3,17 @@ To run actinia, run ``` docker-compose build docker-compose up + +# or DEV setup +docker-compose -f docker-compose-dev.yml build +docker-compose -f docker-compose-dev.yml up ``` For actinia-core development, run and enter the running container. ``` docker-compose run --rm --entrypoint /bin/bash -v $HOME/repos/actinia_core/src:/src/actinia_core/src actinia-core + +docker-compose -f docker-compose-dev.yml run --rm --entrypoint /bin/bash -v $HOME/repos/actinia/actinia_core/src:/src/actinia_core/src -v $HOME/repos/actinia/actinia_core/scripts:/src/actinia_core/scripts actinia-core ``` Inside the container, you can run GRASS with: diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml index 7341f702b..7ea9b77f2 100644 --- a/docker/docker-compose-dev.yml +++ b/docker/docker-compose-dev.yml @@ -12,6 +12,7 @@ services: - ./actinia-core-data/workspace/temp_db:/actinia_core/workspace/temp_db" - ./actinia-core-data/workspace/tmp:/actinia_core/workspace/tmp" - ./actinia-core-data/resources:/actinia_core/resources" + - ../scripts:/src/actinia_core/scripts ports: - "8088:8088" depends_on: From bbe87387ee2f8f4d5da24190e5a620ac2d40b8b1 Mon Sep 17 00:00:00 2001 From: Anika Bettge Date: Fri, 19 Jul 2019 12:22:28 +0200 Subject: [PATCH 02/24] merge readme --- docker/docker-compose-dev.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml index 9bf97e374..ed2589360 100644 --- a/docker/docker-compose-dev.yml +++ b/docker/docker-compose-dev.yml @@ -8,18 +8,11 @@ services: - ./actinia-core-data/grassdb:/actinia_core/grassdb:Z - ./actinia-core-data/pgpass:/mnt/pgpass:Z - ./actinia-core-data/geodata_dir:/mnt/geodata:Z -<<<<<<< HEAD - ./actinia-core-data/userdata:/actinia_core/userdata" - ./actinia-core-data/workspace/temp_db:/actinia_core/workspace/temp_db" - ./actinia-core-data/workspace/tmp:/actinia_core/workspace/tmp" - ./actinia-core-data/resources:/actinia_core/resources" - ../scripts:/src/actinia_core/scripts -======= - - ./actinia-core-data/userdata:/actinia_core/userdata - - ./actinia-core-data/workspace/temp_db:/actinia_core/workspace/temp_db - - ./actinia-core-data/workspace/tmp:/actinia_core/workspace/tmp - - ./actinia-core-data/resources:/actinia_core/resources ->>>>>>> 63820b5903a7092db1ff6638f4158272b818f2bb ports: - "8088:8088" depends_on: From 937d192d1810d42e1fd2209aa6d006f3a4a2f4e0 Mon Sep 17 00:00:00 2001 From: Anika Bettge Date: Thu, 26 Sep 2019 15:10:46 +0200 Subject: [PATCH 03/24] add python3-gdal dependency --- docker/actinia-core/Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index 005677691..4c6e71e41 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get update && apt-get upgrade -y && \ python3 \ python3-dateutil \ python3-dev \ + python3-gdal \ python3-magic \ python3-pip \ python3-ply \ @@ -82,10 +83,6 @@ RUN grass --tmp-location EPSG:4326 --exec g.extension -s extension=i.cutlines && grass --tmp-location EPSG:4326 --exec g.extension -s extension=v.out.png && \ grass --tmp-location EPSG:4326 --exec g.extension -s extension=v.centerpoint -# Set python shebangs to python3, just to be sure -WORKDIR /root/.grass7/addons/scripts -RUN find -type f | xargs sed -zi 's,#!/usr/bin/env python\n,#!/usr/bin/env python3\n,' - # install SNAPPY RUN apt-get install default-jdk maven -y ENV JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64" @@ -96,7 +93,6 @@ RUN update-alternatives --remove python /usr/bin/python3 # Install actinia-core WORKDIR /src - ADD https://api.github.com/repos/$SOURCE_GIT_REMOTE/$SOURCE_GIT_REPO/git/refs/$SOURCE_GIT_TYPE/$SOURCE_GIT_REF version.json RUN git clone -b ${SOURCE_GIT_REF} --single-branch ${SOURCE_GIT_URL}/${SOURCE_GIT_REMOTE}/${SOURCE_GIT_REPO}.git actinia_core From 7f193df319d91a4110c1c5cb64830634e24823a4 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Sun, 29 Sep 2019 18:47:51 +0200 Subject: [PATCH 04/24] execute tests after actinia installation - run tests after actinia_statistic_plugin install - run tests after actinia_core install --- docker/actinia-core/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index 4c6e71e41..e1beca4b0 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -70,7 +70,8 @@ RUN grass --tmp-location EPSG:4326 --exec g.version -rge RUN git config --global http.sslVerify false RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin WORKDIR /src/actinia_statistic_plugin -RUN pip3 install -r requirements.txt && python3 setup.py install +RUN pip3 install -r requirements.txt && python3 setup.py install \ + && python3 setup.py test # Install GRASS GIS addons WORKDIR /src @@ -97,7 +98,8 @@ ADD https://api.github.com/repos/$SOURCE_GIT_REMOTE/$SOURCE_GIT_REPO/git/refs/$S RUN git clone -b ${SOURCE_GIT_REF} --single-branch ${SOURCE_GIT_URL}/${SOURCE_GIT_REMOTE}/${SOURCE_GIT_REPO}.git actinia_core WORKDIR /src/actinia_core -RUN pip3 install -r requirements.txt && python3 setup.py install +RUN pip3 install -r requirements.txt && python3 setup.py install \ + && python3 setup.py test # Reduce the image size RUN apt-get autoremove -y From f0f2b57bf9a64502fbea35b2836cdbf2b8f546e1 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Sun, 29 Sep 2019 20:35:11 +0200 Subject: [PATCH 05/24] Install actinia_statistic_plugin after actinia_core - to be able to execute the tests, install actinia_statistic_plugin after actinia_core - silence wget in snap installer --- docker/actinia-core/Dockerfile | 23 ++++++++++++++--------- docker/actinia-core/snap/install.sh | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index e1beca4b0..4397f9fa9 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -66,13 +66,6 @@ RUN ln -s `grass --config path` /usr/local/grass7 # show installed version RUN grass --tmp-location EPSG:4326 --exec g.version -rge -# Install actinia-core plugins -RUN git config --global http.sslVerify false -RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin -WORKDIR /src/actinia_statistic_plugin -RUN pip3 install -r requirements.txt && python3 setup.py install \ - && python3 setup.py test - # Install GRASS GIS addons WORKDIR /src # Install the d.rast.multi module to render several maps at once @@ -98,8 +91,20 @@ ADD https://api.github.com/repos/$SOURCE_GIT_REMOTE/$SOURCE_GIT_REPO/git/refs/$S RUN git clone -b ${SOURCE_GIT_REF} --single-branch ${SOURCE_GIT_URL}/${SOURCE_GIT_REMOTE}/${SOURCE_GIT_REPO}.git actinia_core WORKDIR /src/actinia_core -RUN pip3 install -r requirements.txt && python3 setup.py install \ - && python3 setup.py test +RUN pip3 install -r requirements.txt && python3 setup.py install +## TODO: fix tests +#\ +# && python3 setup.py test + +# Install actinia-core plugins +RUN git config --global http.sslVerify false +# RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin +COPY actinia_statistic_plugin /src/actinia_statistic_plugin +WORKDIR /src/actinia_statistic_plugin +RUN pip3 install -r requirements.txt && python3 setup.py install +## TODO: fix tests +#\ +# && python3 setup.py test # Reduce the image size RUN apt-get autoremove -y diff --git a/docker/actinia-core/snap/install.sh b/docker/actinia-core/snap/install.sh index 3ad481c7b..6f2f600bb 100644 --- a/docker/actinia-core/snap/install.sh +++ b/docker/actinia-core/snap/install.sh @@ -15,7 +15,7 @@ mkdir -p /root/.snap/snap-python/snappy cp /src/snap/jpy/dist/*.whl "/root/.snap/snap-python/snappy" # install and update snap -wget -O /src/snap/esa-snap_all_unix_6_0.sh \ +wget -q -O /src/snap/esa-snap_all_unix_6_0.sh \ "http://step.esa.int/downloads/6.0/installers/esa-snap_all_unix_6_0.sh" sh /src/snap/esa-snap_all_unix_6_0.sh -q -varfile /src/snap/response.varfile /usr/local/snap/bin/snap --nosplash --nogui --modules --update-all From 306103de8879aca6904c440bad99913f481402e8 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Sun, 29 Sep 2019 20:53:04 +0200 Subject: [PATCH 06/24] fix latlong -> latlong_wgs84 --- docker/actinia-core-prod/start.sh | 2 +- docker/actinia-core/start-dev.sh | 2 +- docker/actinia-core/start.sh | 2 +- docs/tutorial_data_access.rst | 2 +- scripts/README.md | 4 ++-- scripts/ace | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/actinia-core-prod/start.sh b/docker/actinia-core-prod/start.sh index e6dfb2982..d739489b2 100644 --- a/docker/actinia-core-prod/start.sh +++ b/docker/actinia-core-prod/start.sh @@ -9,7 +9,7 @@ mkdir -p /actinia_core/resources # Create default location in mounted (!) directory grass -text -e -c 'EPSG:25832' /actinia_core/grassdb/utm32n -grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong +grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84 # TODO: use this location for tests and integrate sample data, see README # created here, because set in sample config as default location grass -text -e -c 'EPSG:3358' /actinia_core/grassdb/nc_spm_08 diff --git a/docker/actinia-core/start-dev.sh b/docker/actinia-core/start-dev.sh index 21ab5e9be..091f3f062 100644 --- a/docker/actinia-core/start-dev.sh +++ b/docker/actinia-core/start-dev.sh @@ -21,7 +21,7 @@ cp /root/.grass7/dblogin /tmp/:/root/.grass7/ # Create default location in mounted (!) directory grass -text -e -c 'EPSG:25832' /actinia_core/grassdb/utm32n -grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong +grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84 # TODO: use this location for tests and integrate sample data, see README # created here, because set in sample config as default location grass -text -e -c 'EPSG:3358' /actinia_core/grassdb/nc_spm_08 diff --git a/docker/actinia-core/start.sh b/docker/actinia-core/start.sh index d19020bcd..6e944c55b 100644 --- a/docker/actinia-core/start.sh +++ b/docker/actinia-core/start.sh @@ -21,7 +21,7 @@ cp /root/.grass7/dblogin /tmp/:/root/.grass7/ # Create default location in mounted (!) directory grass -text -e -c 'EPSG:25832' /actinia_core/grassdb/utm32n -grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong +grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84 # TODO: use this location for tests and integrate sample data, see README # created here, because set in sample config as default location grass -text -e -c 'EPSG:3358' /actinia_core/grassdb/nc_spm_08 diff --git a/docs/tutorial_data_access.rst b/docs/tutorial_data_access.rst index a87bf4edb..174095cb4 100644 --- a/docs/tutorial_data_access.rst +++ b/docs/tutorial_data_access.rst @@ -47,7 +47,7 @@ The output should look similar to this: { "locations": [ - "latlong", + "latlong_wgs84", "nc_spm_08" ], "status": "success" diff --git a/scripts/README.md b/scripts/README.md index be3e86d93..67eb24824 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -110,7 +110,7 @@ ace --location nc_spm_08 --list-mapsets All following commands can be executed in any active GRASS GIS location, since the location name at the actinia server is explicitly provided. In case the location option is not provided, the active location will be used. -The following command lists mapsets of the provided location **latlong**: +The following command lists mapsets of the provided location **latlong_wgs84**: ```bash ace --location latlong_wgs84 --list-mapsets @@ -122,7 +122,7 @@ but only if the actinia user is enabled; otherwise the following message appears ```bash ace --location latlong_wgs84 --list-mapsets {'message': "{'Status': 'error', 'Messages': 'Unauthorized access to location " - "'}"} + "'}"} ``` To list all raster maps available in the specified mapset belonging to the provided location **nc_spm_08**, run: diff --git a/scripts/ace b/scripts/ace index 52b9010cf..7fefa5cc7 100755 --- a/scripts/ace +++ b/scripts/ace @@ -551,9 +551,9 @@ def main(script: str, version: bool, list_jobs: str, info_job: str, ace --script commands.sh - Run the script commands.sh in the location *latlong* on the actinia server: + Run the script commands.sh in the location *switching* on the actinia server: - ace --location latlong --script commands.sh + ace --location latlong_wgs84 --script commands.sh List all running jobs of the current user From 19c3499bd58582aa895a0044f2a96be1f1c0aeb6 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Sun, 29 Sep 2019 22:46:33 +0200 Subject: [PATCH 07/24] restore correct actinia_statistic_plugin --- docker/actinia-core/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index 4397f9fa9..fde096884 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -98,8 +98,7 @@ RUN pip3 install -r requirements.txt && python3 setup.py install # Install actinia-core plugins RUN git config --global http.sslVerify false -# RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin -COPY actinia_statistic_plugin /src/actinia_statistic_plugin +RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin WORKDIR /src/actinia_statistic_plugin RUN pip3 install -r requirements.txt && python3 setup.py install ## TODO: fix tests From 20fbc72daf40043a8e21cf97b03b86895e1ed8fd Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Mon, 30 Sep 2019 11:05:53 +0200 Subject: [PATCH 08/24] fix latlong -> latlong_wgs84; fix docs typo --- scripts/actinia-algebra | 2 +- src/actinia_core/resources/common/user.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/actinia-algebra b/scripts/actinia-algebra index b38de7cb0..c592d44b9 100644 --- a/scripts/actinia-algebra +++ b/scripts/actinia-algebra @@ -68,7 +68,7 @@ G_REGION = { "verbose":False } -# actinia-algebra -s http://104.199.28.149:80 LL S2A_NDVI_1 "ndvi = (S2A_B08@S2A - S2A_B04@S2A)/(S2A_B08@S2A + S2A_B04@S2A)" ndvi -n 121 +# actinia-algebra -s http://104.199.28.149:80 latlong_wgs84 S2A_NDVI_1 "ndvi = (S2A_B08@S2A - S2A_B04@S2A)/(S2A_B08@S2A + S2A_B04@S2A)" ndvi -n 121 # Example with ECAD dataset # actinia-algebra ECAD algebra_test 'A = temperature_mean_1950_2013_monthly_celsius@PERMANENT * 1' test precipitation_monthly_mm_0 -n 3 diff --git a/src/actinia_core/resources/common/user.py b/src/actinia_core/resources/common/user.py index 5b391ec55..d1da2b5b0 100644 --- a/src/actinia_core/resources/common/user.py +++ b/src/actinia_core/resources/common/user.py @@ -72,7 +72,7 @@ def __init__(self, user_id, user_group=None, user_role=None, cell_limit = global_config.MAX_CELL_LIMIT, process_num_limit=global_config.PROCESS_NUM_LIMIT, process_time_limit=global_config.PROCESS_TIME_LIMT): - """Consrtuctor + """Constructor Initialize and create a user object. To commit a new user to the database, set all required permissions and call the commit() function. @@ -657,7 +657,7 @@ def create_user(user_id, "user1", "landsat"], "ECAD":["PERMANENT"], - "LL":["PERMANENT"]}, + "latlong_wgs84":["PERMANENT"]}, accessible_modules=global_config.MODULE_WHITE_LIST, cell_limit = global_config.MAX_CELL_LIMIT, process_num_limit=global_config.PROCESS_NUM_LIMIT, From e456e93b44d056881eae28cd5f6c2617d995e8a7 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Mon, 30 Sep 2019 23:43:47 +0200 Subject: [PATCH 09/24] added missing global_config.WORKER_LOGFILE - without global_config.WORKER_LOGFILE being defined redis won't start - updated GRASS GIS version that provided in current docker image --- tests/test_resource_base.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index bd14c5d07..02016c346 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -4,7 +4,7 @@ # performance processing of geographical data that uses GRASS GIS for # computational tasks. For details, see https://actinia.mundialis.de/ # -# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG +# Copyright (c) 2016-2019 Sören Gebbert and mundialis GmbH & Co. KG # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,7 +34,7 @@ __license__ = "GPLv3" __author__ = "Sören Gebbert" -__copyright__ = "Copyright 2016-2018, Sören Gebbert and mundialis GmbH & Co. KG" +__copyright__ = "Copyright 2016-2019, Sören Gebbert and mundialis GmbH & Co. KG" __maintainer__ = "Sören Gebbert" __email__ = "soerengebbert@googlemail.com" @@ -59,14 +59,15 @@ def setup_environment(): # Set the port to the test redis server global_config.REDIS_SERVER_SERVER = "localhost" global_config.REDIS_SERVER_PORT = 7000 + # Set the path to redis WORKER_LOGFILE + global_config.WORKER_LOGFILE = "/var/log/redis/redis" # home = os.getenv("HOME") - # GRASS - + # GRASS GIS # Setup the test environment - global_config.GRASS_GIS_BASE="/usr/local/grass-7.8.dev" - global_config.GRASS_GIS_START_SCRIPT="/usr/local/bin/grass78" + global_config.GRASS_GIS_BASE="/usr/local/grass79/" + global_config.GRASS_GIS_START_SCRIPT="/usr/local/bin/grass79" # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" # global_config.GRASS_DATABASE = "%s/graas/grass_test_db" % home global_config.GRASS_TMP_DATABASE = "/tmp" From 185d9ce81d2e1df02d369ec4a1b78a7210fc74e6 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Tue, 1 Oct 2019 00:00:54 +0200 Subject: [PATCH 10/24] set link to match actinia default config (needed for tests) --- docker/actinia-core/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index fde096884..e4c27ad5b 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -90,6 +90,9 @@ WORKDIR /src ADD https://api.github.com/repos/$SOURCE_GIT_REMOTE/$SOURCE_GIT_REPO/git/refs/$SOURCE_GIT_TYPE/$SOURCE_GIT_REF version.json RUN git clone -b ${SOURCE_GIT_REF} --single-branch ${SOURCE_GIT_URL}/${SOURCE_GIT_REMOTE}/${SOURCE_GIT_REPO}.git actinia_core +# set link to match actinia default config (needed for tests) +RUN ln -s /actinia_core /root/actinia + WORKDIR /src/actinia_core RUN pip3 install -r requirements.txt && python3 setup.py install ## TODO: fix tests From 9382e504b67c9c5f31cebd220cc0ca224eef4630 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Tue, 1 Oct 2019 00:03:10 +0200 Subject: [PATCH 11/24] WORKER_LOGFILE definition not needed WORKER_LOGFILE definition not needed due to link set in 185d9ce81d2e1df02d369ec4a1b78a7210fc74e6 --- tests/test_resource_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 02016c346..cf1a6e0e5 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -60,7 +60,7 @@ def setup_environment(): global_config.REDIS_SERVER_SERVER = "localhost" global_config.REDIS_SERVER_PORT = 7000 # Set the path to redis WORKER_LOGFILE - global_config.WORKER_LOGFILE = "/var/log/redis/redis" + # global_config.WORKER_LOGFILE = "/var/log/redis/redis" # home = os.getenv("HOME") From 9c5e28efd317388559aae5410d0d9900337ae378 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Wed, 2 Oct 2019 07:23:16 +0200 Subject: [PATCH 12/24] change to Debian base image - removed Ubuntu specific language-pack-en-base --- docker/actinia-core/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index e4c27ad5b..bf86d8a1d 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -26,7 +26,6 @@ RUN apt-get update && apt-get upgrade -y && \ bzip2 \ curl \ git \ - language-pack-en-base \ libcurl4-gnutls-dev \ libpython3-all-dev \ moreutils \ From bc9b989627f41d5435ca6a16e80aac0686946d26 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Wed, 2 Oct 2019 16:08:59 +0200 Subject: [PATCH 13/24] Ubuntu base image reinstated --- docker/actinia-core/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index bf86d8a1d..e4c27ad5b 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && apt-get upgrade -y && \ bzip2 \ curl \ git \ + language-pack-en-base \ libcurl4-gnutls-dev \ libpython3-all-dev \ moreutils \ From a0981d5f54551f032a05df35bf4047b6ee93a370 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Wed, 2 Oct 2019 20:17:30 +0200 Subject: [PATCH 14/24] table of ACL added --- docs/actinia_concepts.rst | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/docs/actinia_concepts.rst b/docs/actinia_concepts.rst index 393268bc7..aa9b11229 100644 --- a/docs/actinia_concepts.rst +++ b/docs/actinia_concepts.rst @@ -69,6 +69,69 @@ The following user-roles are supported: - Has access to persistent databases that were granted by a superadmin +Overview table: + ++------------+------------+------------+-----------+------------+-------------+ +| task | superadmin | admin | user | guest | notes | +| | | | | | | ++============+============+============+===========+============+=============+ +| amount | y | y | limited, | limited, | - | +| raster | | | selected | selected | | +| cells is | | | via redis | via redis | | +| unlimited | | | | | | ++------------+------------+------------+-----------+------------+-------------+ +| database | y | only to | limited, | limited, | - | +| access is | | persistent | defined | defined | | +| unlimited | | databases | in redis | in redis | | +| | | that were | | | | +| | | granted | | | | +| | | by a | | | | +| | | superadmin | | | | ++------------+------------+------------+-----------+------------+-------------+ +| location/ | y | y | can | has | - | +| mapset | | | create | access to | | +| access is | | | create, | persistent | | +| unlimited | | | modify | databases | | +| | | | and | that were | | +| | | | delete | granted | | +| | | | mapsets | by | | +| | | | in user | a | | +| | | | specific | superadmin,| | +| | | | databases | defined | | +| | | | , | in redis | | +| | | | defined | | | +| | | | in redis | | | ++------------+------------+------------+-----------+------------+-------------+ +| module | y | y | can run | has very | - | +| access is | | | computati | limited | | +| unlimited | | | onal | access to | | +| | | | tasks in | API calls | | +| | | | ephemeral | | | +| | | | and user | | | +| | | | specific | | | +| | | | databases | | | ++------------+------------+------------+-----------+------------+-------------+ +| get, | y | users | n | n | Only | +| create, | | with the | | | normal | +| delete a | | maximum | | | users | +| single | | user-role | | | (role=user) | +| user | | user of | | | can be | +| | | the same | | | created | +| | | user | | | | +| | | group | | | | ++------------+------------+------------+-----------+------------+-------------+ + +In the file actinia.cfg, limits and more can be defined: + +.. code:: bash + + [LIMITS] + max_cell_limit = 2000000 + process_time_limt = 60 + process_num_limit = 20 + number_of_workers = 3 + + The Actinia databases --------------------- From 918e1cdcf1b0873dcc73fa4c3c89ae3fbea431ee Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Thu, 3 Oct 2019 23:24:27 +0200 Subject: [PATCH 15/24] location name cosmetics --- scripts/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 67eb24824..ae42a1086 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -601,9 +601,9 @@ acp --location nc_spm_08 r.info slope_elev@test_mapset ```bash # create new location -ace --create-location latlon 4326 +ace --create-location mylatlon 4326 # create new mapset within location -ace --location latlon --create-mapset user1 +ace --location mylatlon --create-mapset user1 ``` ### Installing of GRASS GIS addons (extensions) @@ -611,9 +611,9 @@ ace --location latlon --create-mapset user1 ```bash # list existing addons, see also # https://grass.osgeo.org/grass7/manuals/addons/ -ace --location latlon g.extension -l +ace --location latlong_wgs84 g.extension -l # install machine learning addon r.learn.ml -ace --location latlon g.extension r.learn.ml +ace --location latlong_wgs84 g.extension r.learn.ml ``` From 99319a9d57202a80f08387a7a4ee2e855c1b2570 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Sun, 6 Oct 2019 20:54:56 +0200 Subject: [PATCH 16/24] Fix GRASS GIS and Python version - grass79 -> grass78 - grass-7.8.dev -> grass78 - python -> python3 This anticipates selected fixed from #43 to get the tests running --- Makefile | 14 +++++++------- src/actinia_core/resources/common/config.py | 4 ++-- tests/test_common_base.py | 2 +- tests/test_resource_base.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index cc92cba8c..ce0fd5953 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,22 @@ # Makefile to run setup.py clean: - python setup.py clean + python3 setup.py clean docs: - python setup.py docs + python3 setup.py docs build: - python setup.py build + python3 setup.py build install: - python setup.py install + python3 setup.py install bdist: - python setup.py bdist + python3 setup.py bdist dist: - python setup.py dist + python3 setup.py dist test: - python setup.py test + python3 setup.py test diff --git a/src/actinia_core/resources/common/config.py b/src/actinia_core/resources/common/config.py index 47e8158b7..554e08056 100644 --- a/src/actinia_core/resources/common/config.py +++ b/src/actinia_core/resources/common/config.py @@ -84,12 +84,12 @@ def __init__(self): self.GRASS_TMP_DATABASE = "%s/actinia/workspace/temp_db" % home # The directory to store temporary GRASS databases self.GRASS_RESOURCE_DIR = "%s/actinia/resources" % home # Directory to store exported resources self.GRASS_RESOURCE_QUOTA = 100 # The size quota of the resource storage in Gigibit - self.GRASS_GIS_BASE = "/usr/local/grass-7.8.dev" # Installation directory of GRASS + self.GRASS_GIS_BASE = "/usr/local/grass78/" # Installation directory of GRASS self.GRASS_GIS_START_SCRIPT = "/usr/local/bin/grass78" self.GRASS_ADDON_PATH = "%s/.grass7/addons/" % home self.GRASS_MODULES_XML_PATH = os.path.join(self.GRASS_GIS_BASE, "gui", "wxpython", "xml", "module_items.xml") self.GRASS_VENV = "%s/src/actinia/grass_venv/bin/activate_this.py" % home # The path to the activation - # script of the python2 venv + # script of the python2 venv (old) # DEFAULT LIMITS when a user is created self.MAX_CELL_LIMIT = 1000 * 1000 * 1000 # Maximum number of cells that are allowed to process diff --git a/tests/test_common_base.py b/tests/test_common_base.py index e4d7a1572..de162ddab 100644 --- a/tests/test_common_base.py +++ b/tests/test_common_base.py @@ -64,7 +64,7 @@ def setup_environment(): # GRASS # Setup the test environment - global_config.GRASS_GIS_BASE="/usr/local/grass-7.8.dev" + global_config.GRASS_GIS_BASE="/usr/local/grass78/" global_config.GRASS_GIS_START_SCRIPT="/usr/local/bin/grass78" # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" # global_config.GRASS_DATABASE = "%s/graas/grass_test_db" % home diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index cf1a6e0e5..01cdde549 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -66,8 +66,8 @@ def setup_environment(): # GRASS GIS # Setup the test environment - global_config.GRASS_GIS_BASE="/usr/local/grass79/" - global_config.GRASS_GIS_START_SCRIPT="/usr/local/bin/grass79" + global_config.GRASS_GIS_BASE="/usr/local/grass78/" + global_config.GRASS_GIS_START_SCRIPT="/usr/local/bin/grass78" # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" # global_config.GRASS_DATABASE = "%s/graas/grass_test_db" % home global_config.GRASS_TMP_DATABASE = "/tmp" From d322aeef161406b9c54eee6a98c9f014bd7bedb3 Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Thu, 10 Oct 2019 12:03:03 +0200 Subject: [PATCH 17/24] added api docs for user management --- .../resources/common/response_models.py | 121 +++++++++ src/actinia_core/resources/user_management.py | 239 +++++++++++------- 2 files changed, 263 insertions(+), 97 deletions(-) diff --git a/src/actinia_core/resources/common/response_models.py b/src/actinia_core/resources/common/response_models.py index 492d60c3b..1a365607a 100644 --- a/src/actinia_core/resources/common/response_models.py +++ b/src/actinia_core/resources/common/response_models.py @@ -1150,3 +1150,124 @@ def create_response_from_model(response_model_class=ProcessingResponseModel, return pickle.dumps([http_code, resp_dict]) else: return jsonify(resp_dict) + + +class UserListResponseModel(Schema): + """Response schema that is used to list all users. + + """ + type = 'object' + properties = { + 'status': { + 'type': 'string', + 'description': 'The status of the request' + }, + 'user_list': { + 'type': 'array', + 'items': {'type': 'string'}, + 'description': 'The names of all users' + } + } + required = ["status", "user_list"] + example = {"status": "success","user_list": ["actinia-gdi"]} + + + +class UserListResponseModel(Schema): + """Response schema that is used to list all users. + + """ + type = 'object' + properties = { + 'status': { + 'type': 'string', + 'description': 'The status of the request' + }, + 'user_list': { + 'type': 'array', + 'items': {'type': 'string'}, + 'description': 'The names of all users' + } + } + required = ["status", "user_list"] + example = {"status": "success","user_list": ["actinia-gdi"]} + + +class UserInfoResponseModel(Schema): + """Response schema that is used to show information about a user. + + """ + type = 'object' + properties = { + 'status': { + 'type': 'string', + 'description': 'The status of the request' + }, + 'permissions': { + 'type': 'object', + 'properties': { + 'cell_limit': { + 'type': 'string', + 'description': 'The limit of number of cells the user is allowed to process' + }, + 'process_num_limit': { + 'type': 'string', + 'description': 'The limit of number of processes the user is allowed to integrate into one process chain' + }, + 'process_time_limit': { + 'type': 'string', + 'description': 'The time a process must not exceed' + }, + 'accessible_datasets': { + 'type': 'object', + 'properties': {}, + 'description': 'The persistent GRASS databases the user is allowed to use. Contains one object for each location name with an array of strings containing all allowed mapset names. See example for more information.' + }, + 'accessible_modules': { + 'type': 'array', + 'items': {'type': 'string'}, + 'description': 'The GRASS modules the user is allowed to use' + } + }, + 'description': 'The names of all users' + }, + 'user_id': { + 'type': 'string', + 'description': 'The identifier of the user' + }, + 'user_role': { + 'type': 'string', + 'description': 'The role of the user' + }, + 'user_group': { + 'type': 'string', + 'description': 'The group of the user' + } + } + required = ["status"] + example = { + "Permissions": { + "accessible_datasets": { + "nc_spm_08": [ + "PERMANENT", + "user1", + "landsat" + ] + }, + "accessible_modules": [ + "r.blend", + "r.buffer", + "...", + "t.rast.univar", + "importer", + "exporter" + ], + "cell_limit": 100000000000, + "process_num_limit": 1000, + "process_time_limit": 31536000 + }, + "Status": "success", + "User group": "superadmin", + "User id": "actinia-gdi", + "User role": "superadmin" + } diff --git a/src/actinia_core/resources/user_management.py b/src/actinia_core/resources/user_management.py index 87588678e..2cac3227f 100644 --- a/src/actinia_core/resources/user_management.py +++ b/src/actinia_core/resources/user_management.py @@ -32,10 +32,13 @@ from flask import jsonify, make_response, g from flask_restful import reqparse +from flask_restful_swagger_2 import swagger from .base_login import LoginBase from .common.user import ActiniaUser from .common.app import auth from .common.logging_interface import log_api_call +from .common.response_models import UserListResponseModel, UserInfoResponseModel, SimpleResponseModel + __license__ = "GPLv3" __author__ = "Sören Gebbert" @@ -51,6 +54,17 @@ class UserListResource(LoginBase): def __init__(self): LoginBase.__init__(self) + @swagger.doc({ + 'tags': ['User Management'], + 'description': 'Get a list of all users. ' + 'Minimum required user role: admin.', + 'responses': { + '200': { + 'description': 'This response returns a list of user names.', + 'schema': UserListResponseModel + } + } + }) def get(self): """List all users in the database @@ -60,22 +74,15 @@ def get(self): Returns: flask.Response: A HTTP response with JSON payload containing a list of users - - A HTTP status 200 response JSON content:: - - { - "Status": "success", - "User list": [ - "soeren" - ] - } - """ user = ActiniaUser(None) user_list = user.list_all_users() - return make_response(jsonify({"Status":"success", - "User list":user_list})) + return make_response(jsonify(UserListResponseModel( + status="success", + user_list=user_list + )), 200) + class UserManagementResource(LoginBase): """Get, Create and Delete a single user @@ -90,6 +97,32 @@ class UserManagementResource(LoginBase): def __init__(self): LoginBase.__init__(self) + @swagger.doc({ + 'tags': ['User Management'], + 'description': 'Get information about the group, role and permissions ' + 'of a certain user. ' + 'Minimum required user role: admin.', + 'parameters': [ + { + 'name': 'user', + 'description': 'The unique name of the user', + 'required': True, + 'in': 'path', + 'type': 'string' + } + ], + 'responses': { + '200': { + 'description': 'This response returns information about a ' + 'certain user.', + 'schema': UserInfoResponseModel + }, + '400': { + 'description': 'The error message', + 'schema': SimpleResponseModel + } + } + }) def get(self, user_id): """Return the credentials of a single user @@ -103,55 +136,63 @@ def get(self, user_id): flask.Response: A HTTP response with JSON payload containing the credentials of the user - - A HTTP status 200 response JSON content:: - - { - "Permissions": { - "accessible_datasets": { - "nc_spm_08": [ - "PERMANENT", - "user1", - "landsat" - ] - }, - "accessible_modules": [ - "r.blend", - ... - "g.findfile", - "g.gisenv" - ], - "cell_limit": 100000000, - "process_num_limit": 5, - "process_time_limit": 60 - }, - "Status": "success", - "User id": "soeren", - "User role": "admin" - } - - A HTTP status 400 response JSON content:: - - { - "Messages": "User does not exist", - "Status": "error" - } - """ user = ActiniaUser(user_id) if user.exists() is False: - return make_response(jsonify({"Status":"error", - "Messages":"User <%s> does not exist"%user_id}), 400) + return make_response(jsonify(SimpleResponseModel( + status="error", + message="User <%s> does not exist"%user_id + )), 400) credentials = user.get_credentials() - return make_response(jsonify({"Status":"success", - "User id":credentials["user_id"], - "User role":credentials["user_role"], - "User group":credentials["user_group"], - "Permissions":credentials["permissions"]})) - + return make_response(jsonify(UserInfoResponseModel( + status="success", + permissions=credentials["permissions"], + user_id=credentials["user_id"], + user_role=credentials["user_role"], + user_group=credentials["user_group"] + )), 200) + + @swagger.doc({ + 'tags': ['User Management'], + 'description': 'Creates a new user in the database. ' + 'Minimum required user role: admin.', + 'parameters': [ + { + 'name': 'user_id', + 'description': 'The unique name of the user', + 'required': True, + 'in': 'path', + 'type': 'string' + }, + { + 'name': 'password', + 'description': 'The password of the new user', + 'required': True, + 'in': 'path', + 'type': 'string' + }, + { + 'name': 'group', + 'description': 'The group of the new user', + 'required': True, + 'in': 'path', + 'type': 'string' + } + ], + 'responses': { + '200': { + 'description': 'This response returns the status of user creation.', + 'schema': SimpleResponseModel + }, + '400': { + 'description': 'The error message', + 'schema': SimpleResponseModel + } + } + }) def post(self, user_id): """Create a user in the database @@ -165,21 +206,6 @@ def post(self, user_id): flask.Response: A HTTP response with JSON payload containing the status and messages - - A HTTP status 200 response JSON content:: - - { - "Messages:": "User thomas created", - "Status": "success" - } - - A HTTP status 400 response JSON content:: - - { - "Messages": "Unable to create user thomas", - "Status": "error" - } - """ # Password parser password_parser = reqparse.RequestParser() @@ -196,12 +222,40 @@ def post(self, user_id): if user is not None: if user.exists() is True: - return make_response(jsonify({"Status":"success", - "Messages:":"User %s created"%user_id}), 201) - - return make_response(jsonify({"Status":"error", - "Messages":"Unable to create user %s"%user_id}), 400) - + return make_response(jsonify(SimpleResponseModel( + status="success", + message="User %s created"%user_id + )), 201) + + return make_response(jsonify(SimpleResponseModel( + status="error", + message="Unable to create user %s"%user_id + )), 400) + + @swagger.doc({ + 'tags': ['User Management'], + 'description': 'Deletes a user. ' + 'Minimum required user role: admin.', + 'parameters': [ + { + 'name': 'user_id', + 'description': 'The unique name of the user', + 'required': True, + 'in': 'path', + 'type': 'string' + } + ], + 'responses': { + '200': { + 'description': 'This response returns the status of user deletion.', + 'schema': SimpleResponseModel + }, + '400': { + 'description': 'The error message', + 'schema': SimpleResponseModel + } + } + }) def delete(self, user_id): """Delete a specific user @@ -215,31 +269,22 @@ def delete(self, user_id): flask.Response: A HTTP response with JSON payload containing the status and messages - - A HTTP status 200 response JSON content:: - - { - "Messages": "User thomas deleted", - "Status": "success" - } - - A HTTP status 400 response JSON content:: - - { - "Messages": "Unable to delete user thomas. User does not exist.", - "Status": "error" - } - """ user = ActiniaUser(user_id) if user.exists() is False: - return make_response(jsonify({"Status":"error", - "Messages":"Unable to delete user %s. User does not exist."%user_id}), 400) + return make_response(jsonify(SimpleResponseModel( + status="error", + message="Unable to delete user %s. User does not exist."%user_id + )), 400) if user.delete() is True: - return make_response(jsonify({"Status":"success", - "Messages":"User %s deleted"%user_id})) - - return make_response(jsonify({"Status":"error", - "Messages":"Unable to delete user %s"%user_id}), 400) + return make_response(jsonify(SimpleResponseModel( + status="success", + message="User %s deleted"%user_id + )), 200) + + return make_response(jsonify(SimpleResponseModel( + status="error", + message="Unable to delete user %s"%user_id + )), 400) From c6d43aea1d5b39b1849ed0d4329e06446f85ebc9 Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Fri, 11 Oct 2019 09:23:46 +0200 Subject: [PATCH 18/24] remove duplicate code + enhance descriptions --- .../resources/common/response_models.py | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/actinia_core/resources/common/response_models.py b/src/actinia_core/resources/common/response_models.py index 1a365607a..c5b675ed6 100644 --- a/src/actinia_core/resources/common/response_models.py +++ b/src/actinia_core/resources/common/response_models.py @@ -1152,27 +1152,6 @@ def create_response_from_model(response_model_class=ProcessingResponseModel, return jsonify(resp_dict) -class UserListResponseModel(Schema): - """Response schema that is used to list all users. - - """ - type = 'object' - properties = { - 'status': { - 'type': 'string', - 'description': 'The status of the request' - }, - 'user_list': { - 'type': 'array', - 'items': {'type': 'string'}, - 'description': 'The names of all users' - } - } - required = ["status", "user_list"] - example = {"status": "success","user_list": ["actinia-gdi"]} - - - class UserListResponseModel(Schema): """Response schema that is used to list all users. @@ -1208,7 +1187,7 @@ class UserInfoResponseModel(Schema): 'properties': { 'cell_limit': { 'type': 'string', - 'description': 'The limit of number of cells the user is allowed to process' + 'description': 'The limit of number of raster cells the user is allowed to process' }, 'process_num_limit': { 'type': 'string', @@ -1221,12 +1200,12 @@ class UserInfoResponseModel(Schema): 'accessible_datasets': { 'type': 'object', 'properties': {}, - 'description': 'The persistent GRASS databases the user is allowed to use. Contains one object for each location name with an array of strings containing all allowed mapset names. See example for more information.' + 'description': 'The persistent GRASS GIS databases the user is allowed to use. Contains one object for each location name with an array of strings containing all allowed mapset names. See example for more information.' }, 'accessible_modules': { 'type': 'array', 'items': {'type': 'string'}, - 'description': 'The GRASS modules the user is allowed to use' + 'description': 'The GRASS GIS modules the user is allowed to use' } }, 'description': 'The names of all users' From 866323876462bf63f53042ed19706930121843e0 Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Tue, 15 Oct 2019 10:45:31 +0200 Subject: [PATCH 19/24] make swagger docs of AsyncPersistentResource reusable --- .../resources/persistent_processing.py | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/src/actinia_core/resources/persistent_processing.py b/src/actinia_core/resources/persistent_processing.py index aa9547a71..9f0eac10a 100644 --- a/src/actinia_core/resources/persistent_processing.py +++ b/src/actinia_core/resources/persistent_processing.py @@ -29,6 +29,7 @@ import os import shutil import subprocess +from copy import deepcopy from flask import jsonify, make_response from flask_restful_swagger_2 import swagger @@ -79,24 +80,19 @@ """ -class AsyncPersistentResource(ResourceBase): - - def __init__(self): - ResourceBase.__init__(self) - - @swagger.doc({ - 'tags': ['Processing'], - 'description': DESCR, - 'consumes':['application/json'], - 'parameters': [ - { - 'name': 'location_name', - 'description': 'The location name', - 'required': True, - 'in': 'path', - 'type': 'string', - 'default': 'nc_spm_08' - }, +SCHEMA_DOC={ + 'tags': ['Processing'], + 'description': DESCR, + 'consumes':['application/json'], + 'parameters': [ + { + 'name': 'location_name', + 'description': 'The location name', + 'required': True, + 'in': 'path', + 'type': 'string', + 'default': 'nc_spm_08' + }, { 'name': 'mapset_name', 'description': 'The name of an existing mapset or a new mapset that should be created', @@ -104,26 +100,34 @@ def __init__(self): 'in': 'path', 'type': 'string' }, - { - 'name': 'process_chain', - 'description': 'The process chain that should be executed', - 'required': True, - 'in': 'body', - 'schema': ProcessChainModel - } - ], - 'responses': { - '200': { - 'description': 'The result of the process chain execution', - 'schema':ProcessingResponseModel - }, - '400': { - 'description':'The error message and a detailed log why process chain execution ' - 'did not succeeded', - 'schema':ProcessingResponseModel - } + { + 'name': 'process_chain', + 'description': 'The process chain that should be executed', + 'required': True, + 'in': 'body', + 'schema': ProcessChainModel + } + ], + 'responses': { + '200': { + 'description': 'The result of the process chain execution', + 'schema':ProcessingResponseModel + }, + '400': { + 'description':'The error message and a detailed log why process chain execution ' + 'did not succeeded', + 'schema':ProcessingResponseModel } - }) + } +} + + +class AsyncPersistentResource(ResourceBase): + + def __init__(self): + ResourceBase.__init__(self) + + @swagger.doc(deepcopy(SCHEMA_DOC)) def post(self, location_name, mapset_name): """Execute a user defined process chain that creates a new mapset or runs in an existing one. From 4cb76fe002f58fdd0ab7e7b6735ee9d057976b6c Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Wed, 16 Oct 2019 18:02:19 +0200 Subject: [PATCH 20/24] fixed typo --- src/actinia_core/resources/persistent_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/actinia_core/resources/persistent_processing.py b/src/actinia_core/resources/persistent_processing.py index 9f0eac10a..3b28ccbb4 100644 --- a/src/actinia_core/resources/persistent_processing.py +++ b/src/actinia_core/resources/persistent_processing.py @@ -115,7 +115,7 @@ }, '400': { 'description':'The error message and a detailed log why process chain execution ' - 'did not succeeded', + 'did not succeed', 'schema':ProcessingResponseModel } } From 96ec2b2f6ab1d5ab9dde2d4883fb5f3c21f5b369 Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Fri, 18 Oct 2019 12:44:31 +0200 Subject: [PATCH 21/24] use copy instead of git pull in Dockerfile --- .dockerignore | 13 ++++++++++ docker/actinia-core/Dockerfile | 21 +++++----------- docker/actinia-core/README.md | 44 ++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..531c76be0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +docker +!docker/actinia-core/snap +!docker/actinia-core/actinia.cfg +!docker/actinia-core/start.sh +!docker/actinia-core/start-dev.sh +.gitignore +.github +.travis +.travis.yml +.eggs +.pytest_cache +build +dist diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index e4c27ad5b..33c88585b 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -3,14 +3,6 @@ FROM mundialis/grass-py3-pdal LABEL authors="Carmen Tawalika,Anika Bettge,Markus Neteler,Sören Gebbert" LABEL maintainer="tawalika@mundialis.de,bettge@mundialis.de,neteler@mundialis.de,soerengebbert@gmail.com" -ARG SOURCE_GIT_URL=https://github.com -ARG SOURCE_GIT_REMOTE=mundialis -ARG SOURCE_GIT_REPO=actinia_core -# can be "tags" (for tag) or "heads" (for) branch -ARG SOURCE_GIT_TYPE=heads -# can be a tag name or branch name -ARG SOURCE_GIT_REF=master - ENV GDAL_CACHEMAX=2000 ENV GRASS_COMPRESSOR=ZSTD @@ -81,14 +73,13 @@ RUN grass --tmp-location EPSG:4326 --exec g.extension -s extension=i.cutlines && RUN apt-get install default-jdk maven -y ENV JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64" RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -COPY snap /src/snap +COPY docker/actinia-core/snap /src/snap RUN sh /src/snap/install.sh RUN update-alternatives --remove python /usr/bin/python3 # Install actinia-core -WORKDIR /src -ADD https://api.github.com/repos/$SOURCE_GIT_REMOTE/$SOURCE_GIT_REPO/git/refs/$SOURCE_GIT_TYPE/$SOURCE_GIT_REF version.json -RUN git clone -b ${SOURCE_GIT_REF} --single-branch ${SOURCE_GIT_URL}/${SOURCE_GIT_REMOTE}/${SOURCE_GIT_REPO}.git actinia_core + +COPY . /src/actinia_core # set link to match actinia default config (needed for tests) RUN ln -s /actinia_core /root/actinia @@ -117,9 +108,9 @@ WORKDIR /grassdb VOLUME /grassdb # Copy actinia config file and start scripts -COPY actinia.cfg /etc/default/actinia -COPY start.sh /src/start.sh -COPY start-dev.sh /src/start-dev.sh +COPY docker/actinia-core/actinia.cfg /etc/default/actinia +COPY docker/actinia-core/start.sh /src/start.sh +COPY docker/actinia-core/start-dev.sh /src/start-dev.sh ENTRYPOINT ["/bin/bash"] CMD ["/src/start.sh"] diff --git a/docker/actinia-core/README.md b/docker/actinia-core/README.md index 2520d8bba..3bd0907f0 100644 --- a/docker/actinia-core/README.md +++ b/docker/actinia-core/README.md @@ -7,3 +7,47 @@ https://hub.docker.com/r/mundialis/actinia-core ## Background info This docker image is based on https://hub.docker.com/r/mundialis/grass-py3-pdal which provides GRASS GIS 7.8 (release branch, grass78) with python3 and PDAL support. + +The Dockerfile contained in this folder is used to build to dockerhub. +If you want to build manually... +## mind the build context! + +Clone this repository and change directory: + +```bash +$ git clone https://github.com/mundialis/actinia_core.git +$ cd actinia_core +``` + +__Build the docker with__: + +```bash +$ docker build \ + --file docker/actinia-core/Dockerfile \ + --tag actinia-core:latest . +``` + +View the images available using `sudo docker images` and open a bash terminal with: + +```bash +$ docker run -it --entrypoint=/bin/bash actinia-core:latest +root@c5e3b72ad8ba:/grassdb# +``` + +__To build a stable version__: + +change to the tag you want to build (only supported from v0.2.3): +```bash +$ git checkout v0.2.3 +``` + +and build and enter with: + +```bash +$ docker build \ + --file docker/actinia-core/Dockerfile \ + --tag actinia-core:stable . + +$ docker run -it --entrypoint=/bin/bash actinia-core:latest +root@c5e3b72ad8ba:/grassdb# +``` From b58c1425cb3fb22a83c346fdc4e75e40942d930d Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Wed, 23 Oct 2019 11:50:58 +0200 Subject: [PATCH 22/24] update build context in docker-compose --- docker/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c0e1483fb..d579b7a45 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,7 +3,8 @@ services: actinia-core: build: - context: actinia-core/ + context: ./.. + dockerfile: docker/actinia-core/Dockerfile args: - SOURCE_GIT_URL=https://github.com - SOURCE_GIT_REMOTE=mundialis From 18118710b83c8c9a3b7d90fd273f6a37fdeab934 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Tue, 29 Oct 2019 12:37:24 +0100 Subject: [PATCH 23/24] Updated create/delete location example --- scripts/curl_commands.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/curl_commands.sh b/scripts/curl_commands.sh index 467223c7f..14a325d74 100644 --- a/scripts/curl_commands.sh +++ b/scripts/curl_commands.sh @@ -15,13 +15,18 @@ export AUTH='-u demouser:gu3st!pa55w0rd' curl ${AUTH} -X GET -i ${actinia}/api/v1/locations # Create a new location based on an EPSG code -# curl -d "epsg=4326" -X POST -i ${actinia}/locations/latlong_wgs84_new # BUG +echo '{ + "epsg": "4326" +}' > /tmp/pc_location_epsg4326.json + +curl ${AUTH} -H "Content-Type: application/json" -X POST "${actinia}/api/v1/locations/latlong_wgs84_new" -d @/tmp/pc_location_epsg4326.json # Check if the location is listed -curl ${AUTH} -X GET -i ${actinia}/api/v1/locations +curl ${AUTH} -X GET ${actinia}/api/v1/locations # Delete the new location -# curl -X DELETE -i ${actinia}/locations/latlong_wgs84_new # BUG +curl ${AUTH} -X DELETE ${actinia}/api/v1/locations/latlong_wgs84_new +rm -f /tmp/pc_location_epsg4326.json # Get information of the NC location curl ${AUTH} -X GET -i ${actinia}/api/v1/locations/nc_spm_08/info From 6be9c9647cab764547e2ff1739eabed535bf8e7d Mon Sep 17 00:00:00 2001 From: Carmen Tawalika Date: Wed, 30 Oct 2019 13:57:11 +0100 Subject: [PATCH 24/24] make base dockerimage explicit --- docker/actinia-core/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/actinia-core/Dockerfile b/docker/actinia-core/Dockerfile index 33c88585b..8e63a8c4c 100644 --- a/docker/actinia-core/Dockerfile +++ b/docker/actinia-core/Dockerfile @@ -1,4 +1,4 @@ -FROM mundialis/grass-py3-pdal +FROM mundialis/grass-py3-pdal:stable-ubuntu LABEL authors="Carmen Tawalika,Anika Bettge,Markus Neteler,Sören Gebbert" LABEL maintainer="tawalika@mundialis.de,bettge@mundialis.de,neteler@mundialis.de,soerengebbert@gmail.com"