From 767c7d5e1abe43b18f14101df1b03f51d5b8fc0a Mon Sep 17 00:00:00 2001 From: Carmen Date: Mon, 11 Dec 2023 19:57:50 +0100 Subject: [PATCH] cleanup tests --- CHANGELOG.md | 5 + Makefile | 2 +- docker/actinia-core-alpine/Dockerfile | 56 ++++------- docker/actinia-core-tests/Dockerfile | 35 +++---- src/actinia_core/core/redis.conf | 44 --------- src/actinia_core/testsuite.py | 30 ------ tests/README.md | 4 +- tests/test_common_base.py | 132 -------------------------- tests/test_job_resumption.py | 4 +- tests/test_noauth.py | 18 ---- tests/test_resource_base.py | 17 +--- tests_with_redis.sh | 7 +- 12 files changed, 47 insertions(+), 307 deletions(-) delete mode 100644 src/actinia_core/core/redis.conf delete mode 100644 tests/test_common_base.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c79aabef0..1a25460f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,10 +24,15 @@ released from main ... ## [4.12.0](https://github.com/actinia-org/actinia-core/releases/tag/4.12.0) - 2023-11-21 + released from main + ### Added + - Checking pixellimit for r.import commands by @linakrisztian in https://github.com/actinia-org/actinia-core/pull/491 + ### Changed + - chore(deps): update mundialis/actinia-core docker tag to v4.11.0 by @renovate in https://github.com/actinia-org/actinia-core/pull/492 **Full Changelog**: https://github.com/actinia-org/actinia-core/compare/4.11.0...4.12.0 diff --git a/Makefile b/Makefile index 24448f1f5..2e3480f9c 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ devtest: sh ./tests_with_redis.sh dev noauthtest: - sh ./tests_with_redis.sh noauth + sh ./tests_with_redis.sh noauth integrationtest: sh ./tests_with_redis.sh integrationtest diff --git a/docker/actinia-core-alpine/Dockerfile b/docker/actinia-core-alpine/Dockerfile index f0bcc2404..1d913354b 100644 --- a/docker/actinia-core-alpine/Dockerfile +++ b/docker/actinia-core-alpine/Dockerfile @@ -1,58 +1,47 @@ -FROM mundialis/actinia:alpine-dependencies-2023-10-12 as build-base +FROM mundialis/actinia:alpine-dependencies-2023-12-06 as build-base FROM osgeo/grass-gis:releasebranch_8_3-alpine as grass -FROM build-base as build +FROM build-base as requirements -LABEL authors="Carmen Tawalika,Anika Bettge,Markus Neteler,Sören Gebbert" +LABEL authors="Carmen Tawalika,Anika Weinmann,Markus Neteler,Sören Gebbert" LABEL maintainer="tawalika@mundialis.de,weinmann@mundialis.de,neteler@mundialis.de" -COPY . /src/actinia_core -WORKDIR /src/actinia_core -RUN pip3 install build && python3 -m build --outdir /build . -RUN if [[ -f /build/UNKNOWN* ]];then echo "ERROR - Check actinia-core build";exit 1;fi - - -FROM mundialis/actinia:alpine-dependencies-2023-06-03 as actinia_installation - ENV LC_ALL "en_US.UTF-8" ENV GDAL_CACHEMAX=2000 ENV GRASS_COMPRESSOR=ZSTD ENV GRASS_SKIP_MAPSET_OWNER_CHECK 1 ENV GISBASE "" -ENV ACTINIA_API_VERSION 3.4.0 - -USER root # GRASS GIS SETUP COPY --from=grass /usr/local/bin/grass /usr/local/bin/grass COPY --from=grass /usr/local/grass* /usr/local/grass/ -RUN pip3 install --upgrade pip six grass-session --ignore-installed six +RUN pip install --upgrade pip six grass-session --ignore-installed six RUN ln -s /usr/local/grass `grass --config path` RUN grass --tmp-location EPSG:4326 --exec g.version -rge && \ pdal --version && \ - python3 --version + python --version -# install GRASS GIS addon d.rast.multi, because it is needed for STRDS render -# endpoint +# Install GRASS GIS addon d.rast.multi (needed for STRDS render endpoint) RUN grass --tmp-location EPSG:4326 --exec g.extension -s \ - extension=d.rast.multi url=https://github.com/mundialis/d_rast_multi + extension=d.rast.multi url=https://github.com/mundialis/d_rast_multi -# actinia-core and actinia libs BUILD -WORKDIR /build -ARG WHEEL_NAME=actinia_api-${ACTINIA_API_VERSION}-py3-none-any.whl -RUN curl -L --output /build/${WHEEL_NAME} \ - https://github.com/actinia-org/actinia-api/releases/download/${ACTINIA_API_VERSION}/${WHEEL_NAME} -# Install actinia-core and libs -COPY --from=build /build/*.whl /build/ -RUN pip3 install /build/* +FROM build-base as build -# Duplicate install actinia_core requirements. They are already included -# in alpine-build / alpine-runtime images, but check for updates here. -COPY requirements.txt /src/requirements.txt -RUN pip3 install -r /src/requirements.txt +COPY . /src/actinia_core +WORKDIR /src/actinia_core +RUN pip install build && python -m build --outdir /build . +RUN if [[ -f /build/UNKNOWN* ]];then echo "ERROR - Check actinia-core build";exit 1;fi + + +FROM requirements as actinia -# Copy actinia config file and start scripts + set needed envs +# actinia-core and requirements installation +WORKDIR /build +COPY --from=build /build/*.whl /build/ +RUN pip install /build/* + +# Copy actinia config file and start script COPY docker/actinia-core-alpine/actinia.cfg /etc/default/actinia COPY docker/actinia-core-alpine/start.sh /src/start.sh @@ -66,9 +55,6 @@ RUN mkdir -p /actinia_core/grassdb && \ mkdir -p /actinia_core/userdata && \ ln -s /actinia_core /root/actinia - -FROM actinia_installation as actinia - VOLUME /grassdb WORKDIR /src/actinia_core diff --git a/docker/actinia-core-tests/Dockerfile b/docker/actinia-core-tests/Dockerfile index 74c09c4e3..0c1826d84 100644 --- a/docker/actinia-core-tests/Dockerfile +++ b/docker/actinia-core-tests/Dockerfile @@ -3,20 +3,9 @@ FROM mundialis/actinia-core:latest as actinia_test LABEL authors="Carmen Tawalika,Anika Weinmann" LABEL maintainer="tawalika@mundialis.de,weinmann@mundialis.de" -ENV ACTINIA_API_VERSION 3.4.0 - -ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 -ENV ACTINIA_CUSTOM_TEST_CFG /etc/default/actinia_test -# TODO do not set DEFAULT_CONFIG_PATH if this is fixed ENV DEFAULT_CONFIG_PATH /etc/default/actinia_test -ENV GRASS_DATABASE=/tmp/actinia_core/grassdb - -# install things only for tests -RUN apk add redis -RUN pip3 install iniconfig colorlog - -# uninstall actinia core from FROM-image -RUN pip3 uninstall actinia-core -y +# Path must be equal to actinia config "GRASS_DATABASE" +ARG GRASS_DATABASE=/tmp/actinia_core/grassdb # add data for tests WORKDIR ${GRASS_DATABASE} @@ -31,6 +20,16 @@ RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_mapset mv modis_lst nc_spm_08/modis_lst RUN chown -R 1001:1001 nc_spm_08/modis_lst && chmod -R g+w nc_spm_08/modis_lst +# install GRASS addons required for tests +RUN grass --tmp-location EPSG:4326 --exec g.extension -s extension=r.colors.out_sld + +# install things only for tests +RUN apk add redis +RUN pip install pytest pytest-cov iniconfig + +# uninstall actinia core from FROM-image +RUN pip uninstall actinia-core -y + RUN rmdir /actinia_core/grassdb RUN rmdir /actinia_core/userdata RUN rmdir /actinia_core/resources @@ -41,21 +40,11 @@ RUN rmdir /actinia_core/workspace/tmp RUN rmdir /actinia_core/workspace RUN rmdir /actinia_core -# install actinia-api -RUN pip3 install actinia-api@https://github.com/actinia-org/actinia-api/releases/download/${ACTINIA_API_VERSION}/actinia_api-${ACTINIA_API_VERSION}-py3-none-any.whl - -# install GRASS addons required for tests -RUN grass --tmp-location EPSG:4326 --exec g.extension -s extension=r.colors.out_sld -RUN grass --tmp-location EPSG:4326 --exec g.extension -s \ - extension=d.rast.multi url=https://github.com/mundialis/d_rast_multi - # copy needed files and configs for test COPY docker/actinia-core-alpine/actinia.cfg /etc/default/actinia COPY docker/actinia-core-tests/actinia-test.cfg /etc/default/actinia_test COPY docker/actinia-core-tests/actinia-test-noauth.cfg /etc/default/actinia_test_noauth -RUN pip3 install pytest pytest-cov - # TODO: Postgres for tests # using tests/data/poly.gpkg diff --git a/src/actinia_core/core/redis.conf b/src/actinia_core/core/redis.conf deleted file mode 100644 index dfffff0c6..000000000 --- a/src/actinia_core/core/redis.conf +++ /dev/null @@ -1,44 +0,0 @@ -daemonize no -pidfile ./redis-server.pid -port 7000 -bind 127.0.0.1 -timeout 0 -tcp-keepalive 0 -loglevel debug -logfile ./redis-server.log -databases 16 -save 900 1 -save 300 10 -save 60 10000 -stop-writes-on-bgsave-error yes -rdbcompression yes -rdbchecksum yes -dbfilename dump.rdb -dir ./ -slave-serve-stale-data yes -slave-read-only yes -repl-disable-tcp-nodelay no -slave-priority 100 -appendonly no -appendfilename "appendonly.aof" -appendfsync everysec -no-appendfsync-on-rewrite no -auto-aof-rewrite-percentage 100 -auto-aof-rewrite-min-size 64mb -lua-time-limit 5000 -slowlog-log-slower-than 10000 -slowlog-max-len 128 -notify-keyspace-events "" -hash-max-ziplist-entries 512 -hash-max-ziplist-value 64 -list-max-ziplist-entries 512 -list-max-ziplist-value 64 -set-max-intset-entries 512 -zset-max-ziplist-entries 128 -zset-max-ziplist-value 64 -activerehashing yes -client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 -client-output-buffer-limit pubsub 32mb 8mb 60 -hz 10 -aof-rewrite-incremental-fsync yes diff --git a/src/actinia_core/testsuite.py b/src/actinia_core/testsuite.py index 09d5f93e0..e3f1f99e2 100644 --- a/src/actinia_core/testsuite.py +++ b/src/actinia_core/testsuite.py @@ -130,7 +130,6 @@ class ActiniaTestCaseBase(unittest.TestCase): """Base class for GRASS GIS REST API tests""" server_test = False - custom_actinia_cfg = False guest = None user = None admin = None @@ -141,37 +140,8 @@ class ActiniaTestCaseBase(unittest.TestCase): if "ACTINIA_SERVER_TEST" in os.environ: server_test = bool(os.environ["ACTINIA_SERVER_TEST"]) - if "ACTINIA_CUSTOM_TEST_CFG" in os.environ: - custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) - @classmethod def setUpClass(cls): - if cls.server_test is False and cls.custom_actinia_cfg is False: - global_config.REDIS_SERVER_SERVER = "localhost" - global_config.REDIS_SERVER_PORT = 7000 - global_config.GRASS_RESOURCE_DIR = "/tmp" - global_config.DOWNLOAD_CACHE = "/tmp/download_cache" - global_config.REDIS_QUEUE_SERVER_URL = "localhost" - global_config.REDIS_QUEUE_SERVER_PORT = 6379 - global_config.NUMBER_OF_WORKERS = 3 - # Create the job queue - # redis_interface.create_job_queues( - # global_config.REDIS_QUEUE_SERVER_URL, - # global_config.REDIS_QUEUE_SERVER_PORT, - # global_config.NUMBER_OF_WORKERS) - - # If the custom_actinia_cfg variable is set, then the actinia config - # file will be read to configure Redis queue - if cls.server_test is False and cls.custom_actinia_cfg is not False: - global_config.read(cls.custom_actinia_cfg) - - # # Create the job queue - # redis_interface.create_job_queues( - # global_config.REDIS_QUEUE_SERVER_URL, - # global_config.REDIS_QUEUE_SERVER_PORT, - # global_config.NUMBER_OF_WORKERS - # ) - # create configured folders os.makedirs(global_config.GRASS_USER_DATABASE, exist_ok=True) os.makedirs(global_config.GRASS_TMP_DATABASE, exist_ok=True) diff --git a/tests/README.md b/tests/README.md index 042aecf4d..5a81185ac 100644 --- a/tests/README.md +++ b/tests/README.md @@ -57,7 +57,7 @@ Redis authorization is set by `from actinia_core.core.common import redis_interf from actinia_core.core.common import redis_interface # PROBLEM!!!! from actinia_core.core.common.config import global_config - custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) + custom_actinia_cfg = str(os.environ["DEFAULT_CONFIG_PATH"]) global_config.read(custom_actinia_cfg) global_config.REDIS_SERVER_PW ``` @@ -66,7 +66,7 @@ Redis authorization is set by `from actinia_core.core.common import redis_interf import os from actinia_core.core.common.config import global_config - custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) + custom_actinia_cfg = str(os.environ["DEFAULT_CONFIG_PATH"]) global_config.read(custom_actinia_cfg) global_config.REDIS_SERVER_PW ``` diff --git a/tests/test_common_base.py b/tests/test_common_base.py deleted file mode 100644 index 839eeeadd..000000000 --- a/tests/test_common_base.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -####### -# actinia-core - an open source REST API for scalable, distributed, high -# performance processing of geographical data that uses GRASS GIS for -# computational tasks. For details, see https://actinia.mundialis.de/ -# -# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -####### - -""" -Tests: Common test case base -""" -import atexit -import unittest -import signal -import time -import os -from actinia_core.core.common.config import global_config -import actinia_core.core.common.redis_interface as redis_interface -from actinia_core.core.common.app import flask_app - -__license__ = "GPLv3" -__author__ = "Sören Gebbert" -__copyright__ = ( - "Copyright 2016-2019, Sören Gebbert and mundialis GmbH & Co. KG" -) -__maintainer__ = "Sören Gebbert" -__email__ = "soerengebbert@googlemail.com" - -redis_pid = None -custom_actinia_cfg = False - -# Set this variable to use a actinia config file in a docker container -if "ACTINIA_CUSTOM_TEST_CFG" in os.environ: - custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) - - -def setup_environment(): - # If docker config - if custom_actinia_cfg is not False: - global_config.read(custom_actinia_cfg) - return - - global redis_pid - # Set the port to the test redis server - global_config.REDIS_SERVER_SERVER = "localhost" - global_config.REDIS_SERVER_PORT = 7000 - - # home = os.getenv("HOME") - - # GRASS - - # Setup the test environment - global_config.GRASS_GIS_BASE = "/usr/local/grass/" - global_config.GRASS_GIS_START_SCRIPT = "/usr/local/bin/grass" - # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" - # global_config.GRASS_DATABASE = "%s/actinia/grass_test_db" % home - - # Start the redis server for user and logging management - redis_pid = os.spawnl( - os.P_NOWAIT, - "/usr/bin/redis-server", - "common/redis.conf", - "--port %i" % global_config.REDIS_SERVER_PORT, - ) - time.sleep(1) - - -def stop_redis(): - global redis_pid - # Kill th redis server - if custom_actinia_cfg is not False: - return - os.kill(redis_pid, signal.SIGTERM) - - -# Register the redis stop function -atexit.register(stop_redis) -# Setup the environment -setup_environment() - - -class CommonTestCaseBase(unittest.TestCase): - """ - This is the base class for the common testing - """ - - @classmethod - def setUpClass(cls): - if custom_actinia_cfg is not False: - global_config.read(custom_actinia_cfg) - else: - global_config.REDIS_SERVER_URL = "localhost" - global_config.REDIS_SERVER_PORT = 7000 - - args = ( - global_config.REDIS_SERVER_URL, - global_config.REDIS_SERVER_PORT, - ) - if ( - global_config.REDIS_SERVER_PW - and global_config.REDIS_SERVER_PW is not None - ): - args = (*args, global_config.REDIS_SERVER_PW) - - redis_interface.connect(*args) - - @classmethod - def tearDownClass(cls): - redis_interface.disconnect() - - def setUp(self): - # We need to set the application context - self.app_context = flask_app.app_context() - self.app_context.push() - - def tearDown(self): - self.app_context.pop() diff --git a/tests/test_job_resumption.py b/tests/test_job_resumption.py index bb9d564b7..c3d623168 100644 --- a/tests/test_job_resumption.py +++ b/tests/test_job_resumption.py @@ -286,9 +286,7 @@ class JobResumptionProcessingTestCase(ActiniaResourceTestCaseBase): - cfg_file = os.environ.get( - "ACTINIA_CUSTOM_TEST_CFG", "/etc/default/actinia" - ) + cfg_file = os.environ.get("DEFAULT_CONFIG_PATH", "/etc/default/actinia") tmp_cfg_file = "%s_tmp" % cfg_file save_interim_results_value = None endpoint = "/locations/nc_spm_08/processing_async" diff --git a/tests/test_noauth.py b/tests/test_noauth.py index c9a010029..5d3f04294 100644 --- a/tests/test_noauth.py +++ b/tests/test_noauth.py @@ -46,10 +46,6 @@ __email__ = "info@mundialis.de" -# Set this variable to use a actinia config file in a docker container -if "ACTINIA_CUSTOM_TEST_CFG" in os.environ: - custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) - PC = { "version": 1, "list": [ @@ -106,20 +102,6 @@ class ActiniaWithoutAuthentication(ActiniaTestCaseBase): @classmethod def setUpClass(cls): - if cls.server_test is False and cls.custom_actinia_cfg is False: - global_config.REDIS_SERVER_SERVER = "localhost" - global_config.REDIS_SERVER_PORT = 7000 - global_config.GRASS_RESOURCE_DIR = "/tmp" - global_config.DOWNLOAD_CACHE = "/tmp/download_cache" - global_config.REDIS_QUEUE_SERVER_URL = "localhost" - global_config.REDIS_QUEUE_SERVER_PORT = 6379 - global_config.NUMBER_OF_WORKERS = 3 - - # If the custom_actinia_cfg variable is set, then the actinia config - # file will be read to configure Redis queue - if cls.server_test is False and cls.custom_actinia_cfg is not False: - global_config.read(cls.custom_actinia_cfg) - # global_config.read(cls.custom_actinia_cfg) # Start the redis interface redis_args = ( global_config.REDIS_SERVER_URL, diff --git a/tests/test_resource_base.py b/tests/test_resource_base.py index 1058065f2..0f7befb7a 100644 --- a/tests/test_resource_base.py +++ b/tests/test_resource_base.py @@ -44,15 +44,11 @@ redis_pid = None server_test = False -custom_actinia_cfg = False # If this environmental variable is set, then a real http request will be send # instead of using the flask test_client. if "ACTINIA_SERVER_TEST" in os.environ: server_test = bool(os.environ["ACTINIA_SERVER_TEST"]) -# Set this variable to use a actinia config file in a docker container -if "ACTINIA_CUSTOM_TEST_CFG" in os.environ: - custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) base_url_data = "https://apps.mundialis.de/actinia_test_datasets" additional_external_data = { @@ -73,13 +69,6 @@ def setup_environment(): global redis_pid - # Set the port to the test redis server - global_config.REDIS_SERVER_SERVER = "localhost" - global_config.REDIS_SERVER_PORT = 7000 - # Set the path to redis WORKER_LOGFILE - # global_config.WORKER_LOGFILE = "/var/log/redis/redis" - - # home = os.getenv("HOME") # GRASS GIS # Setup the test environment @@ -88,7 +77,8 @@ def setup_environment(): # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" # global_config.GRASS_DATABASE = "%s/actinia/grass_test_db" % home global_config.GRASS_TMP_DATABASE = "/tmp" - + # TODO: never secretly overwrite config parameters + custom_actinia_cfg = True if server_test is False and custom_actinia_cfg is False: # Start the redis server for user and logging management redis_pid = os.spawnl( @@ -99,9 +89,6 @@ def setup_environment(): ) time.sleep(1) - if server_test is False and custom_actinia_cfg is not False: - global_config.read(custom_actinia_cfg) - def stop_redis(): if server_test is False: diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 42e4b668d..f294e7a96 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -2,10 +2,10 @@ run_tests_noauth (){ # change config run tests and change config bag - mv ${ACTINIA_CUSTOM_TEST_CFG} ${ACTINIA_CUSTOM_TEST_CFG}_tmp - cp /etc/default/actinia_test_noauth ${ACTINIA_CUSTOM_TEST_CFG} + mv ${DEFAULT_CONFIG_PATH} ${DEFAULT_CONFIG_PATH}_tmp + cp /etc/default/actinia_test_noauth ${DEFAULT_CONFIG_PATH} pytest -m 'noauth' - mv ${ACTINIA_CUSTOM_TEST_CFG}_tmp ${ACTINIA_CUSTOM_TEST_CFG} + mv ${DEFAULT_CONFIG_PATH}_tmp ${DEFAULT_CONFIG_PATH} } @@ -19,7 +19,6 @@ webhook-server --host "0.0.0.0" --port "5005" & sleep 10 # run tests -echo $ACTINIA_CUSTOM_TEST_CFG echo $DEFAULT_CONFIG_PATH TEST_RES=1