Merge branch 'develop'

dbca-wa · Jul 9, 2024 · d84a1aa · d84a1aa
2 parents 6a98e25 + f4bbd7a
commit d84a1aa
Show file tree

Hide file tree

Showing 35 changed files with 1,029 additions and 202 deletions.
diff --git a/.ahoy.yml b/.ahoy.yml
@@ -124,10 +124,54 @@ commands:
     usage: Pipe in a postgres dump file.  `ahoy db-import local.dump`
     cmd: |
       if [ -e "$@" ] ; then
+        if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
+          CKAN_CONTAINER_NAME=ckan-dev
+        fi
+        SERVICE=$CKAN_CONTAINER_NAME
+        # Clean the database
+        ahoy run "ckan db clean --yes"
+        # Import the database
         docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists -v --no-owner --no-privileges' < "$@"
-      else echo "Provided sql file" "$@" "does not exist"
+        # Initialise plugin tables
+        docker compose -f $DOCKER_COMPOSE exec -T $CKAN_CONTAINER_NAME sh -c 'rm  /tmp/container_ready && export CKAN__PLUGINS=$(grep "^ckan\.plugins" $APP_DIR/config/dbca.ini | cut -d"=" -f2) && bash /docker-entrypoint.d/02_setup_dbca.sh'
+        # Rebuild the Solr index
+        ahoy reindex-solr
+      else
+        echo "Provided sql file" "$@" "does not exist"
+      fi
+
+  import-db:
+    usage: Import the DB from an environment into your local dev environment | ahoy import-db ENVIRONMENT
+    cmd: |    
+      source ./scripts/messages.sh
+      #
+      # Check the correct arguments are supplied
+      #
+      if [[ "$1" != "staging" ]] && [[ "$1" != "production" ]]; then
+        error "Invalid environment. Please use 'staging' or 'production'. eg ahoy import-db ENVIRONMENT"
+        exit 1
       fi
 
+      ENVIRONMENT=$1  
+      info "Dumping DB from $ENVIRONMENT"
+      if [[ "$1" == "staging" ]]; then
+        docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'cd /tmp && pg_dump "postgresql://'"$AZURE_POSTGRES_STAGING_USER"':'"$AZURE_POSTGRES_STAGING_PASSWORD"'@'"$AZURE_POSTGRES_STAGING_HOST"'/'"$AZURE_POSTGRES_STAGING_DB"'" --format=custom -v > latest.dump'
+      else
+        docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'cd /tmp && pg_dump "postgresql://'"$AZURE_POSTGRES_PROD_USER"':'"$AZURE_POSTGRES_PROD_PASSWORD"'@'"$AZURE_POSTGRES_PROD_HOST"'/'"$AZURE_POSTGRES_PROD_DB"'" --exclude-table=dbca_spatial --format=custom -v > latest.dump'
+      fi
+        
+      info "Importing DB from $ENVIRONMENT"
+      docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_restore -U $CKAN_DB_USER -d $CKAN_DB --clean --if-exists --no-owner --no-privileges -v < /tmp/latest.dump'
+      info "Rebuilding Solr Index"
+      ahoy reindex-solr
+
+      success "Database import successful"
+
+
+  reindex-solr:
+    usage: Re-runs the Solr index, e.g. after running `db-import` command
+    cmd: ahoy run "ckan search-index rebuild"
+
   db-dump:
     usage: Dump data out into a file. `ahoy db-dump local.dump`
     cmd: docker compose -f $DOCKER_COMPOSE exec -T $POSTGRESQL_CONTAINER_NAME sh -c 'pg_dump -U $CKAN_DB_USER -d $CKAN_DB --format=custom -v' > "$@"
@@ -146,7 +190,48 @@ commands:
       if [ "$DOCKER_COMPOSE" = "docker-compose.dev.yml" ]; then
         SITE_URL="http://localhost:${CKAN_PORT_HOST}"
       else
-        SITE_URL="http://localhost:${NGINX_PORT_HOST}"
+        SITE_URL="http://localhost:${NGINX_SSLPORT_HOST}"
       fi
       open $SITE_URL
 
+  download-ckan-logs:
+    usage: Download CKAN logs
+    cmd: |
+      if [ -z "$AZURE_FILE_SHARE_URL" ] || [ -z "$AZURE_FILE_SHARE_SAS_TOKEN" ]; then
+        echo "AZURE_FILE_SHARE_URL or AZURE_FILE_SHARE_SAS_TOKEN is not set"
+        exit 1
+      fi
+      
+      echo "Downloading ckan_logs from:$AZURE_FILE_SHARE_URL"
+      
+      LOG_FILE="ckan.log"
+      echo "Downloading $LOG_FILE"
+      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
+      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
+
+      LOG_FILE="ckan-worker.log"
+      echo "Downloading $LOG_FILE"
+      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
+      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
+
+      LOG_FILE="ckan-cron-jobs.log"
+      echo "Downloading $LOG_FILE"
+      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
+      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
+      
+      LOG_FILE="nginx_access.log"
+      echo "Downloading $LOG_FILE"      
+      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
+      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
+      
+      LOG_FILE="nginx_error.log"
+      echo "Downloading $LOG_FILE"
+      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
+      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
+
+      LOG_FILE="supervisord.log"
+      echo "Downloading $LOG_FILE"
+      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
+      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
+      
+      echo "CKAN logs downloaded to dbca/ckan_logs"
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -21,40 +21,46 @@
 		"vscode": {
 			"extensions": [
 				"ms-python.python",
-				"ms-python.vscode-pylance",
+				"ms-python.debugpy",
 				"ms-python.autopep8",
 				"ms-python.flake8",
+				"ms-python.vscode-pylance",
 				"VisualStudioExptTeam.vscodeintellicode",
 				"GitHub.vscode-pull-request-github",
 				"eamodio.gitlens",
 				"streetsidesoftware.code-spell-checker",
+				"streetsidesoftware.code-spell-checker-australian-english",
 				"wholroyd.jinja",
 				"GitHub.copilot",
 				"GitHub.copilot-chat"
 			],
 			// Set *default* container specific settings.json values on container create.
 			"settings": {
-				"terminal.integrated.profiles.linux": {
-					"bash": {
-						"path": "/bin/bash"
-					}
-				},
+				"python.defaultInterpreterPath": "python",
 				"python.languageServer": "Pylance",
 				"python.linting.enabled": true,
-				"python.linting.flake8Enabled": true,
+				"python.analysis.extraPaths": [
+					"./app/src/ckan"
+				],
+				"[python]": {
+					"editor.defaultFormatter": "ms-python.autopep8"
+				},
+				"flake8.interpreter": [
+					"python"
+				],
 				"flake8.args": [
 					"--max-line-length",
-					"120"
+					"160"
+				],
+				"autopep8.interpreter": [
+					"python"
 				],
-				"python.formatting.provider": "autopep8",
 				"autopep8.args": [
 					"--max-line-length",
-					"120"
+					"160"
 				],
-				"cSpell.language": "en-GB",
-				"python.analysis.extraPaths": [
-					"./app/src"
-				]
+				"cSpell.language": "en-AU",
+				"extensions.verifySignature": false
 			}
 		}
 	},

diff --git a/.env.dbca b/.env.dbca
@@ -37,7 +37,10 @@ USE_HTTPS_FOR_DEV=false
 # CKAN core
 CKAN_VERSION=2.10.0
 CKAN_SITE_ID=default
-CKAN_SITE_URL=http://localhost:5000
+# CKAN Dev
+CKAN_SITE_URL=http://localhost:$CKAN_PORT_HOST
+# CKAN Nginx
+#CKAN_SITE_URL=https://localhost:$NGINX_SSLPORT_HOST
 CKAN_PORT=5000
 
 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
@@ -53,10 +56,10 @@ CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
-TZ=UTC
+TZ=Australia/Perth
 
 # Solr
-SOLR_IMAGE_VERSION=2.10-solr9
+SOLR_IMAGE_VERSION=2.10-solr9-spatial
 CKAN_SOLR_URL=http://solr:8983/solr/ckan
 TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
 
@@ -71,12 +74,33 @@ CKANEXT__XLOADER__JOBS_DB__URI=$CKAN_SQLALCHEMY_URL
 # NGINX
 NGINX_PORT=80
 NGINX_SSLPORT=443
+CKAN_SERVER_NAME=localhost
 
 # Extensions
 
+## ckanext-doi ##
+CKANEXT__DOI__ACCOUNT_NAME=
+CKANEXT__DOI__ACCOUNT_PASSWORD=
+
+## ckanext-spatial ##
+CKANEXT__SPATIAL__COMMON_MAP__APIKEY=
 
 ## WA DBCA Config ##
 # Docker compose project name
 COMPOSE_PROJECT_NAME=dbca
 # The docker compose file to use. Options are docker-compose.dev.yml (The default for local development) or docker-compose.prod.yml (To test production builds)
 DOCKER_COMPOSE=docker-compose.dev.yml
+
+# Azure File Share Storage
+AZURE_FILE_SHARE_URL=
+AZURE_FILE_SHARE_SAS_TOKEN=
+
+# Azure Postgres
+AZURE_POSTGRES_STAGING_USER=
+AZURE_POSTGRES_STAGING_PASSWORD=
+AZURE_POSTGRES_STAGING_HOST=
+AZURE_POSTGRES_STAGING_DB=
+AZURE_POSTGRES_PROD_USER=
+AZURE_POSTGRES_PROD_PASSWORD=
+AZURE_POSTGRES_PROD_HOST=
+AZURE_POSTGRES_PROD_DB=
diff --git a/.env.example b/.env.example
@@ -1,12 +1,3 @@
-# Container names
-NGINX_CONTAINER_NAME=nginx
-REDIS_CONTAINER_NAME=redis
-POSTGRESQL_CONTAINER_NAME=db
-SOLR_CONTAINER_NAME=solr
-DATAPUSHER_CONTAINER_NAME=datapusher
-CKAN_CONTAINER_NAME=ckan
-WORKER_CONTAINER_NAME=ckan-worker
-
 # Host Ports
 CKAN_PORT_HOST=5000
 NGINX_PORT_HOST=81
@@ -78,7 +69,7 @@ NGINX_PORT=80
 NGINX_SSLPORT=443
 
 # Extensions
-CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher"
+CKAN__PLUGINS="envvars image_view text_view datatables_view datastore datapusher"
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
 CKAN__HARVEST__MQ__PORT=6379

diff --git a/.github/workflows/dbca_build.yml b/.github/workflows/dbca_build.yml
@@ -7,6 +7,8 @@ on:
       - master
       - develop
       - tags/*
+  # Trigger the workflow manually
+  workflow_dispatch:
 
 env:
    REGISTRY: ghcr.io

diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,9 @@ _solr/schema.xml
 _src/*
 local/*
 .env
+
+# DBCA
+src/
+dbca/
+spatial_data/*.geojson
+logs/
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "Python: CKAN Run & Debug",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "pdb",
             "args": [
@@ -23,7 +23,7 @@
         },
         {
             "name": "Python: CKAN Jobs Worker Run & Debug",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "pdb",
             "args": [
@@ -32,21 +32,20 @@
                 "--config",
                 "/srv/app/config/dbca.ini",
                 "jobs",
-                "worker",
-                "priority"
+                "worker"
             ],
             "justMyCode": true
         },
         {
             "name": "Python: CKAN CLI Command Run & Debug",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "pdb",
             "args": [
                 "-c continue",
-                "/srv/app/config/dbca.ini",
+                "/usr/bin/ckan",
                 "--config",
-                "/srv/app/dbca.ini",
+                "/srv/app/config/dbca.ini",
                 "<CLI_COMMAND>"
             ],
             "justMyCode": true
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,6 +7,8 @@ on: @@
           - master
           - develop
           - tags/*
+      # Trigger the workflow manually
+      workflow_dispatch:
     env:
        REGISTRY: ghcr.io
@@ Expand Down @@