Merge branch 'master' into gunicorn-optimization

.dockerignore

@@ -0,0 +1,13 @@
+docker
+!docker/actinia-core/snap
+!docker/actinia-core/actinia.cfg
+!docker/actinia-core/start.sh
+!docker/actinia-core/start-dev.sh
+.gitignore
+.github
+.travis
+.travis.yml
+.eggs
+.pytest_cache
+build
+dist

Makefile

@@ -1,22 +1,22 @@
 # Makefile to run setup.py
 
 clean:
-	python setup.py clean
+	python3 setup.py clean
 
 docs:
-	python setup.py docs
+	python3 setup.py docs
 
 build:
-	python setup.py build
+	python3 setup.py build
 
 install:
-	python setup.py install
+	python3 setup.py install
 
 bdist:
-	python setup.py bdist
+	python3 setup.py bdist
 
 dist:
-	python setup.py dist
+	python3 setup.py dist
 
 test:
-	python setup.py test
+	python3 setup.py test

docker/README.md

@@ -27,6 +27,10 @@ docker-compose build
 # Note: adding -d starts the containers in the background and leaves them
 #       running; without docker-compose will show the logging in the terminal:
 docker-compose up
+
+# or DEV setup
+docker-compose -f docker-compose-dev.yml build
+docker-compose -f docker-compose-dev.yml up
 ```
 
 This will keep logging in the terminal foreground.
@@ -37,8 +41,9 @@ __See below for production deployment__.
 
 For actinia_core development, run and enter the running container (in a separate terminal):
 ```
-cd $HOME/repos/actinia_core/docker/
-docker-compose run --rm --service-ports --entrypoint /bin/bash -v $HOME/repos/actinia_core/src:/src/actinia_core/src actinia-core
+docker-compose run --rm --entrypoint /bin/bash -v $HOME/repos/actinia_core/src:/src/actinia_core/src actinia-core
+
+docker-compose -f docker-compose-dev.yml run --rm --entrypoint /bin/bash -v $HOME/repos/actinia/actinia_core/src:/src/actinia_core/src -v $HOME/repos/actinia/actinia_core/scripts:/src/actinia_core/scripts actinia-core```
 ```
 
 Inside the container, you can run GRASS GIS with:

docker/actinia-core-prod/start.sh

@@ -9,7 +9,7 @@ mkdir -p /actinia_core/resources
 
 # Create default location in mounted (!) directory
 grass -text -e -c 'EPSG:25832' /actinia_core/grassdb/utm32n
-grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong
+grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84
 # TODO: use this location for tests and integrate sample data, see README
 # created here, because set in sample config as default location
 grass -text -e -c 'EPSG:3358' /actinia_core/grassdb/nc_spm_08

docker/actinia-core/Dockerfile

@@ -1,16 +1,8 @@
-FROM mundialis/grass-py3-pdal
+FROM mundialis/grass-py3-pdal:stable-ubuntu
 
 LABEL authors="Carmen Tawalika,Anika Bettge,Markus Neteler,Sören Gebbert"
 LABEL maintainer="[email protected],[email protected],[email protected],[email protected]"
 
-ARG SOURCE_GIT_URL=https://github.com
-ARG SOURCE_GIT_REMOTE=mundialis
-ARG SOURCE_GIT_REPO=actinia_core
-# can be "tags" (for tag) or "heads" (for) branch
-ARG SOURCE_GIT_TYPE=heads
-# can be a tag name or branch name
-ARG SOURCE_GIT_REF=master
-
 ENV GDAL_CACHEMAX=2000
 ENV GRASS_COMPRESSOR=ZSTD
 
@@ -34,6 +26,7 @@ RUN apt-get update && apt-get upgrade -y && \
     python3 \
     python3-dateutil \
     python3-dev \
+    python3-gdal \
     python3-magic \
     python3-pip \
     python3-ply \
@@ -65,12 +58,6 @@ RUN ln -s `grass --config path` /usr/local/grass7
 # show installed version
 RUN grass --tmp-location EPSG:4326 --exec g.version -rge
 
-# Install actinia-core plugins
-RUN git config --global http.sslVerify false
-RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin
-WORKDIR /src/actinia_statistic_plugin
-RUN pip3 install -r requirements.txt && python3 setup.py install
-
 # Install GRASS GIS addons
 WORKDIR /src
 # Install the d.rast.multi module to render several maps at once
@@ -82,26 +69,35 @@ RUN grass --tmp-location EPSG:4326 --exec g.extension -s extension=i.cutlines &&
     grass --tmp-location EPSG:4326 --exec g.extension -s extension=v.out.png && \
     grass --tmp-location EPSG:4326 --exec g.extension -s extension=v.centerpoint
 
-# Set python shebangs to python3, just to be sure
-WORKDIR /root/.grass7/addons/scripts
-RUN find -type f | xargs sed -zi 's,#!/usr/bin/env python\n,#!/usr/bin/env python3\n,'
-
 # install SNAPPY
 RUN apt-get install default-jdk maven -y
 ENV JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64"
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-COPY snap /src/snap
+COPY docker/actinia-core/snap /src/snap
 RUN sh /src/snap/install.sh
 RUN update-alternatives --remove python /usr/bin/python3
 
 # Install actinia-core
-WORKDIR /src
 
-ADD https://api.github.com/repos/$SOURCE_GIT_REMOTE/$SOURCE_GIT_REPO/git/refs/$SOURCE_GIT_TYPE/$SOURCE_GIT_REF version.json
-RUN git clone -b ${SOURCE_GIT_REF} --single-branch ${SOURCE_GIT_URL}/${SOURCE_GIT_REMOTE}/${SOURCE_GIT_REPO}.git actinia_core
+COPY . /src/actinia_core
+
+# set link to match actinia default config (needed for tests)
+RUN ln -s /actinia_core /root/actinia
 
 WORKDIR /src/actinia_core
 RUN pip3 install -r requirements.txt && python3 setup.py install
+## TODO: fix tests
+#\
+#    && python3 setup.py test
+
+# Install actinia-core plugins
+RUN git config --global http.sslVerify false
+RUN git clone https://github.com/mundialis/actinia_statistic_plugin.git /src/actinia_statistic_plugin
+WORKDIR /src/actinia_statistic_plugin
+RUN pip3 install -r requirements.txt && python3 setup.py install
+## TODO: fix tests
+#\
+#    && python3 setup.py test
 
 # Reduce the image size
 RUN apt-get autoremove -y
@@ -112,9 +108,9 @@ WORKDIR /grassdb
 VOLUME /grassdb
 
 # Copy actinia config file and start scripts
-COPY actinia.cfg /etc/default/actinia
-COPY start.sh /src/start.sh
-COPY start-dev.sh /src/start-dev.sh
+COPY docker/actinia-core/actinia.cfg /etc/default/actinia
+COPY docker/actinia-core/start.sh /src/start.sh
+COPY docker/actinia-core/start-dev.sh /src/start-dev.sh
 
 ENTRYPOINT ["/bin/bash"]
 CMD ["/src/start.sh"]

docker/actinia-core/README.md

@@ -7,3 +7,47 @@ https://hub.docker.com/r/mundialis/actinia-core
 ## Background info
 
 This docker image is based on https://hub.docker.com/r/mundialis/grass-py3-pdal which provides GRASS GIS 7.8 (release branch, grass78) with python3 and PDAL support.
+
+The Dockerfile contained in this folder is used to build to dockerhub.
+If you want to build manually...
+## mind the build context!
+
+Clone this repository and change directory:
+
+```bash
+$ git clone https://github.com/mundialis/actinia_core.git
+$ cd actinia_core
+```
+
+__Build the docker with__:
+
+```bash
+$ docker build \
+         --file docker/actinia-core/Dockerfile \
+         --tag actinia-core:latest .
+```
+
+View the images available using `sudo docker images` and open a bash terminal with:
+
+```bash
+$ docker run -it --entrypoint=/bin/bash actinia-core:latest
+root@c5e3b72ad8ba:/grassdb#
+```
+
+__To build a stable version__:
+
+change to the tag you want to build (only supported from v0.2.3):
+```bash
+$ git checkout v0.2.3
+```
+
+and build and enter with:
+
+```bash
+$ docker build \
+        --file docker/actinia-core/Dockerfile \
+        --tag actinia-core:stable .
+
+$ docker run -it --entrypoint=/bin/bash actinia-core:latest
+root@c5e3b72ad8ba:/grassdb#
+```

docker/actinia-core/snap/install.sh

@@ -15,7 +15,7 @@ mkdir -p /root/.snap/snap-python/snappy
 cp /src/snap/jpy/dist/*.whl "/root/.snap/snap-python/snappy"
 
 # install and update snap
-wget -O /src/snap/esa-snap_all_unix_6_0.sh \
+wget -q -O /src/snap/esa-snap_all_unix_6_0.sh \
   "http://step.esa.int/downloads/6.0/installers/esa-snap_all_unix_6_0.sh"
 sh /src/snap/esa-snap_all_unix_6_0.sh -q -varfile /src/snap/response.varfile
 /usr/local/snap/bin/snap --nosplash --nogui --modules --update-all

docker/actinia-core/start-dev.sh

@@ -21,7 +21,7 @@ cp /root/.grass7/dblogin /tmp/:/root/.grass7/
 
 # Create default location in mounted (!) directory
 grass -text -e -c 'EPSG:25832' /actinia_core/grassdb/utm32n
-grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong
+grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84
 # TODO: use this location for tests and integrate sample data, see README
 # created here, because set in sample config as default location
 grass -text -e -c 'EPSG:3358' /actinia_core/grassdb/nc_spm_08

docker/actinia-core/start.sh

@@ -21,7 +21,7 @@ cp /root/.grass7/dblogin /tmp/:/root/.grass7/
 
 # Create default location in mounted (!) directory
 grass -text -e -c 'EPSG:25832' /actinia_core/grassdb/utm32n
-grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong
+grass -text -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84
 # TODO: use this location for tests and integrate sample data, see README
 # created here, because set in sample config as default location
 grass -text -e -c 'EPSG:3358' /actinia_core/grassdb/nc_spm_08

docker/docker-compose-dev.yml

@@ -8,10 +8,11 @@ services:
       - ./actinia-core-data/grassdb:/actinia_core/grassdb:Z
       - ./actinia-core-data/pgpass:/mnt/pgpass:Z
       - ./actinia-core-data/geodata_dir:/mnt/geodata:Z
-      - ./actinia-core-data/userdata:/actinia_core/userdata
-      - ./actinia-core-data/workspace/temp_db:/actinia_core/workspace/temp_db
-      - ./actinia-core-data/workspace/tmp:/actinia_core/workspace/tmp
-      - ./actinia-core-data/resources:/actinia_core/resources
+      - ./actinia-core-data/userdata:/actinia_core/userdata"
+      - ./actinia-core-data/workspace/temp_db:/actinia_core/workspace/temp_db"
+      - ./actinia-core-data/workspace/tmp:/actinia_core/workspace/tmp"
+      - ./actinia-core-data/resources:/actinia_core/resources"
+      - ../scripts:/src/actinia_core/scripts
     ports:
       - "8088:8088"
     depends_on:

docker/docker-compose.yml

@@ -3,7 +3,8 @@ services:
 
   actinia-core:
     build:
-      context: actinia-core/
+      context: ./..
+      dockerfile: docker/actinia-core/Dockerfile
       args:
         - SOURCE_GIT_URL=https://github.com
         - SOURCE_GIT_REMOTE=mundialis

docs/actinia_concepts.rst

@@ -69,6 +69,69 @@ The following user-roles are supported:
         - Has access to persistent databases that were granted by a superadmin
 
 
+Overview table:
+
++------------+------------+------------+-----------+------------+-------------+
+| task       | superadmin | admin      | user      | guest      | notes       |
+|            |            |            |           |            |             |
++============+============+============+===========+============+=============+
+| amount     | y          | y          | limited,  | limited,   | -           |
+| raster     |            |            | selected  | selected   |             |
+| cells is   |            |            | via redis | via redis  |             |
+| unlimited  |            |            |           |            |             |
++------------+------------+------------+-----------+------------+-------------+
+| database   | y          | only to    | limited,  | limited,   | -           |
+| access is  |            | persistent | defined   | defined    |             |
+| unlimited  |            | databases  | in redis  | in redis   |             |
+|            |            | that were  |           |            |             |
+|            |            | granted    |           |            |             |
+|            |            | by a       |           |            |             |
+|            |            | superadmin |           |            |             |
++------------+------------+------------+-----------+------------+-------------+
+| location/  | y          | y          | can       | has        | -           |
+| mapset     |            |            | create    | access to  |             |
+| access is  |            |            | create,   | persistent |             |
+| unlimited  |            |            | modify    | databases  |             |
+|            |            |            | and       | that were  |             |
+|            |            |            | delete    | granted    |             |
+|            |            |            | mapsets   | by         |             |
+|            |            |            | in user   | a          |             |
+|            |            |            | specific  | superadmin,|             |
+|            |            |            | databases | defined    |             |
+|            |            |            | ,         | in redis   |             |
+|            |            |            | defined   |            |             |
+|            |            |            | in redis  |            |             |
++------------+------------+------------+-----------+------------+-------------+
+| module     | y          | y          | can run   | has very   | -           |
+| access is  |            |            | computati | limited    |             |
+| unlimited  |            |            | onal      | access to  |             |
+|            |            |            | tasks in  | API calls  |             |
+|            |            |            | ephemeral |            |             |
+|            |            |            | and user  |            |             |
+|            |            |            | specific  |            |             |
+|            |            |            | databases |            |             |
++------------+------------+------------+-----------+------------+-------------+
+| get,       | y          | users      | n         | n          | Only        |
+| create,    |            | with the   |           |            | normal      |
+| delete a   |            | maximum    |           |            | users       |
+| single     |            | user-role  |           |            | (role=user) |
+| user       |            | user of    |           |            | can be      |
+|            |            | the same   |           |            | created     |
+|            |            | user       |           |            |             |
+|            |            | group      |           |            |             |
++------------+------------+------------+-----------+------------+-------------+
+
+In the file actinia.cfg, limits and more can be defined:
+
+.. code:: bash
+
+   [LIMITS]
+   max_cell_limit = 2000000
+   process_time_limt = 60
+   process_num_limit = 20
+   number_of_workers = 3
+
+
 The Actinia databases
 ---------------------
 

docs/tutorial_data_access.rst

@@ -47,7 +47,7 @@ The output should look similar to this:
 
         {
           "locations": [
-            "latlong",
+            "latlong_wgs84",
             "nc_spm_08"
           ],
           "status": "success"

scripts/README.md

@@ -110,7 +110,7 @@ ace --location nc_spm_08 --list-mapsets
 All following commands can be executed in any active GRASS GIS location, since
 the location name at the actinia server is explicitly provided. 
 In case the location option is not provided, the active location will be used.
-The following command lists mapsets of the provided location **latlong**:
+The following command lists mapsets of the provided location **latlong_wgs84**:
 
 ```bash
 ace --location latlong_wgs84 --list-mapsets
@@ -122,7 +122,7 @@ but only if the actinia user is enabled; otherwise the following message appears
 ```bash
 ace --location latlong_wgs84 --list-mapsets
 {'message': "{'Status': 'error', 'Messages': 'Unauthorized access to location "
-            "<latlong>'}"}
+            "<latlong_wgs84>'}"}
 ```
 
 To list all raster maps available in the specified mapset belonging to the provided location **nc_spm_08**, run:
@@ -601,19 +601,19 @@ acp --location nc_spm_08 r.info slope_elev@test_mapset
 
 ```bash
 # create new location
-ace --create-location latlon 4326
+ace --create-location mylatlon 4326
 # create new mapset within location
-ace --location latlon --create-mapset user1
+ace --location mylatlon --create-mapset user1
 ```
 
 ### Installing of GRASS GIS addons (extensions)
 
 ```bash
 # list existing addons, see also
 # https://grass.osgeo.org/grass7/manuals/addons/
-ace --location latlon g.extension -l
+ace --location latlong_wgs84 g.extension -l
 
 # install machine learning addon r.learn.ml
-ace --location latlon g.extension r.learn.ml
+ace --location latlong_wgs84 g.extension r.learn.ml
 ```