Rewrite database tooling and support for postgres passwords

.circleci/config.yml

@@ -47,12 +47,12 @@ jobs:
           keys:
             - yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }}
             - yarn-cache-{{ checksum ".circleci/cache_version" }}-
-      - run:
-          name: Assert unique evolution numbers
-          command: tools/postgres/assert_unique_evolution_numbers.sh
       - run:
           name: Install frontend dependencies
           command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --frozen-lockfile
+      - run:
+          name: Assert unique evolution numbers
+          command: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
       - restore_cache:
           name: Restore webpack cache
           keys:
@@ -63,7 +63,7 @@ jobs:
           command: |
             docker-compose up -d postgres
             sleep 3
-            docker-compose run compile tools/postgres/diff_schema.js tools/postgres/schema.sql "conf/evolutions/*.sql"
+            docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema
       - run:
           name: Build frontend documentation
           command: docker-compose run base yarn run docs

CHANGELOG.unreleased.md

@@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Changed
 - Limit paid team sharing features to respective organization plans. [#6767](https://github.com/scalableminds/webknossos/pull/6776)
+- Rewrite the database tools in `tools/postgres` to JavaScript and adding support for non-default Postgres username-password combinations. [#6803](https://github.com/scalableminds/webknossos/pull/6803)
 
 ### Fixed
 - Fixed a benign error message which briefly appeared after logging in. [#6810](https://github.com/scalableminds/webknossos/pull/6810)

Dockerfile

@@ -1,6 +1,8 @@
 FROM eclipse-temurin:11
-RUN apt-get update \
-  && apt-get -y install libblosc1 postgresql-client git \
+ARG VERSION_NODE="16.x"
+
+RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \
+  && apt-get -y install libblosc1 postgresql-client git nodejs \
   && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /webknossos

MIGRATIONS.unreleased.md

@@ -8,4 +8,6 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 ## Unreleased
 [Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...HEAD)
 
+- WEBKNOSSOS now requires Node.js not only for development and building, but also for execution. The prebuilt Docker images already contain this dependency. If you're using these, nothing needs to be changed. [#6803](https://github.com/scalableminds/webknossos/pull/6803)
+
 ### Postgres Evolutions:

app/Startup.scala

@@ -1,13 +1,10 @@
-import java.io.{ByteArrayOutputStream, File}
-
 import akka.actor.{ActorSystem, Props}
 import com.typesafe.scalalogging.LazyLogging
 import controllers.InitialDataService
-import io.apigee.trireme.core.{NodeEnvironment, Sandbox}
-import javax.inject._
 import models.annotation.AnnotationDAO
 import models.user.InviteService
 import net.liftweb.common.{Failure, Full}
+import org.apache.http.client.utils.URIBuilder
 import oxalis.cleanup.CleanUpService
 import oxalis.files.TempFileService
 import oxalis.mail.{Mailer, MailerConfig}
@@ -17,6 +14,8 @@ import play.api.inject.ApplicationLifecycle
 import utils.WkConf
 import utils.sql.SqlClient
 
+import javax.inject._
+import scala.collection.mutable
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 import scala.concurrent.duration._
@@ -69,8 +68,19 @@ class Startup @Inject()(actorSystem: ActorSystem,
     }
   }
 
+  private lazy val postgresUrl = {
+    val slickUrl =
+      if (conf.Slick.Db.url.startsWith("jdbc:"))
+        conf.Slick.Db.url.substring(5)
+      else conf.Slick.Db.url
+    val uri = new URIBuilder(slickUrl)
+    uri.setUserInfo(conf.Slick.Db.user, conf.Slick.Db.password)
+    uri.build().toString
+  }
+
   if (conf.Slick.checkSchemaOnStartup) {
     ensurePostgresDatabase()
+    ensurePostgresSchema()
   }
 
   initialDataService.insert.futureBox.map {
@@ -81,38 +91,31 @@ class Startup @Inject()(actorSystem: ActorSystem,
     case _ => ()
   }
 
-  private def ensurePostgresDatabase(): Unit = {
-    logger.info("Running ensure_db.sh with POSTGRES_URL " + sys.env.get("POSTGRES_URL"))
-
-    val processLogger = ProcessLogger((o: String) => logger.info(o), (e: String) => logger.error(e))
-
-    // this script is copied to the stage directory in AssetCompilation
-    val result = "./tools/postgres/ensure_db.sh" ! processLogger
+  private def ensurePostgresSchema(): Unit = {
+    logger.info("Checking database schema…")
 
-    if (result != 0)
-      throw new Exception("Could not ensure Postgres database. Is postgres installed?")
+    val errorMessageBuilder = mutable.ListBuffer[String]()
+    val capturingProcessLogger =
+      ProcessLogger((o: String) => errorMessageBuilder.append(o), (e: String) => logger.error(e))
 
-    // diffing the actual DB schema against schema.sql:
-    logger.info("Running diff_schema.js tools/postgres/schema.sql DB")
-    val nodeEnv = new NodeEnvironment()
-    nodeEnv.setDefaultNodeVersion("0.12")
-    val nodeOutput = new ByteArrayOutputStream()
-    nodeEnv.setSandbox(new Sandbox().setStdout(nodeOutput).setStderr(nodeOutput))
-    val script = nodeEnv.createScript(
-      "diff_schema.js",
-      new File("tools/postgres/diff_schema.js"),
-      Array("tools/postgres/schema.sql", "DB")
-    )
-    val status = script.execute().get()
-    if (status.getExitCode == 0) {
-      logger.info("Schema is up to date.")
+    val result = Process("./tools/postgres/dbtool.js check-db-schema", None, "POSTGRES_URL" -> postgresUrl) ! capturingProcessLogger
+    if (result == 0) {
+      logger.info("Database schema is up to date.")
     } else {
-      val nodeOut = new String(nodeOutput.toByteArray, "UTF-8")
-      val errorMessage = s"Database schema does not fit to schema.sql! \n $nodeOut"
+      val errorMessage = errorMessageBuilder.toList.mkString("\n")
       logger.error(errorMessage)
       slackNotificationService.warn("SQL schema mismatch", errorMessage)
     }
+  }
+
+  private def ensurePostgresDatabase(): Unit = {
+    logger.info(s"Ensuring Postgres database…")
+    val processLogger = ProcessLogger((o: String) => logger.info(o), (e: String) => logger.error(e))
 
+    // this script is copied to the stage directory in AssetCompilation
+    val result = Process("./tools/postgres/dbtool.js ensure-db", None, "POSTGRES_URL" -> postgresUrl) ! processLogger
+    if (result != 0)
+      throw new Exception("Could not ensure Postgres database. Is postgres installed?")
   }
 
   private def startActors(actorSystem: ActorSystem) = {

app/utils/WkConf.scala

@@ -203,6 +203,14 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L
 
   object Slick {
     val checkSchemaOnStartup: Boolean = get[Boolean]("slick.checkSchemaOnStartup")
+
+    object Db {
+      val url: String = get[String]("slick.db.url")
+      val user: String = get[String]("slick.db.user")
+      val password: String = get[String]("slick.db.password")
+    }
+
+    val children = List(Db)
   }
 
   object Voxelytics {

conf/slick.conf

@@ -8,7 +8,9 @@ slick = {
     driver = org.postgresql.Driver
     keepAliveConnection = true
     user = "postgres"
+    user = ${?POSTGRES_USER}
     password = "postgres"
+    password = ${?POSTGRES_PASSWORD}
     queueSize = 5000
   }
   checkSchemaOnStartup = true

docker-compose.yml

@@ -35,7 +35,9 @@ services:
     volumes:
       - ./binaryData:/webknossos/binaryData
     environment:
-      - POSTGRES_URL=jdbc:postgresql://postgres/webknossos
+      POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+      POSTGRES_USER: webknossos_user
+      POSTGRES_PASSWORD: secret_password
     user: ${USER_UID:-1000}:${USER_GID:-1000}
 
   webknossos-datastore:
@@ -95,6 +97,8 @@ services:
       - COVERALLS_REPO_TOKEN
       - TZ=${TZ:-Europe/Berlin}
       - POSTGRES_URL=jdbc:postgresql://postgres/webknossos
+      - POSTGRES_USER=webknossos_user
+      - POSTGRES_PASSWORD=secret_password
       - HOME=/root
       - CIRCLE_TAG=${CIRCLE_TAG}
       - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM}
@@ -133,7 +137,9 @@ services:
       fossildb-dev:
         condition: service_healthy
     environment:
-      - POSTGRES_URL=jdbc:postgresql://postgres/webknossos
+      POSTGRES_URL: jdbc:postgresql://postgres/webknossos
+      POSTGRES_USER: webknossos_user
+      POSTGRES_PASSWORD: secret_password
     command:
       - bash
       - -c
@@ -175,7 +181,9 @@ services:
       fossildb:
         condition: service_healthy
     environment:
-      - POSTGRES_URL=jdbc:postgresql://postgres/webknossos_testing
+      POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing
+      POSTGRES_USER: webknossos_user
+      POSTGRES_PASSWORD: secret_password
     command:
       - bash
       - -c
@@ -212,8 +220,8 @@ services:
     image: postgres:10-alpine
     environment:
       POSTGRES_DB: webknossos
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
+      POSTGRES_USER: webknossos_user
+      POSTGRES_PASSWORD: secret_password
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres -h 127.0.0.1 -p 5432"]
       interval: 2s
@@ -238,27 +246,27 @@ services:
 
   psql:
     extends: postgres
-    command: psql -h postgres -U postgres webknossos
+    command: psql -h postgres -U webknossos_user webknossos
     links:
       - "postgres-persisted:postgres"
     depends_on:
       postgres-persisted:
         condition: service_healthy
     environment:
-      PGPASSWORD: postgres
+      PGPASSWORD: secret_password
     volumes:
       - ./conf/evolutions/:/evolutions/
 
   drop-db:
     extends: postgres
-    command: psql -h postgres -U postgres postgres -c "DROP DATABASE webknossos"
+    command: psql -h postgres -U webknossos_user postgres -c "DROP DATABASE webknossos"
     links:
       - "postgres-persisted:postgres"
     depends_on:
       postgres-dev:
         condition: service_healthy
     environment:
-      PGPASSWORD: postgres
+      PGPASSWORD: secret_password
 
   # FossilDB
   fossildb:

frontend/javascripts/test/enzyme/e2e-setup.ts

@@ -140,6 +140,6 @@ export function resetDatabase() {
   // The parameter needs to be set globally here.
   // See https://github.com/shelljs/shelljs/issues/981#issuecomment-626840798
   shell.config.fatal = true;
-  shell.exec("tools/postgres/prepareTestDB.sh", { silent: true });
+  shell.exec("tools/postgres/dbtool.js prepare-test-db", { silent: true });
 }
 export { tokenUserA, tokenUserB, tokenUserC, tokenUserD, tokenUserE, setCurrToken };

package.json

@@ -32,7 +32,7 @@
     "abort-controller": "^3.0.0",
     "ava": "^3.13.0",
     "browserslist-to-esbuild": "^1.2.0",
-    "commander": "^2.17.1",
+    "commander": "^10.0.0",
     "coveralls": "^3.0.2",
     "css-loader": "^6.5.1",
     "documentation": "^13.2.5",
@@ -43,7 +43,6 @@
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint_d": "^12.2.1",
     "espree": "^3.5.4",
-    "glob": "^7.1.3",
     "husky": "^0.14.3",
     "jsdoc": "^3.5.5",
     "jsdom": "^11.5.1",
@@ -62,12 +61,9 @@
     "prettier": "2.7.1",
     "proto-loader6": "^0.4.0",
     "puppeteer": "^13.3.2",
-    "randomstring": "^1.1.5",
     "react-test-renderer": "^16.8.0",
     "redux-mock-store": "^1.2.2",
-    "replace-in-file": "1.1.3",
     "retoggle": "^0.3.0",
-    "rimraf": "^2.6.2",
     "shelljs": "^0.8.5",
     "sinon": "^12.0.1",
     "tmp": "0.0.33",
@@ -95,7 +91,7 @@
     "test-watch": "tools/test.sh test --watch",
     "test-prepare": "tools/test.sh prepare",
     "test-prepare-watch": "tools/test.sh prepare -w --verbose",
-    "test-e2e": "tools/postgres/prepareTestDB.sh && tools/test.sh test-e2e --timeout=60s --verbose",
+    "test-e2e": "tools/postgres/dbtool.js prepare-test-db && tools/test.sh test-e2e --timeout=60s --verbose",
     "test-screenshot": "tools/test.sh test-screenshot --timeout=5m",
     "refresh-screenshots": "rm -rf frontend/javascripts/test/screenshots/** && docker-compose up screenshot-tests",
     "test-help": "echo For development it is recommended to run yarn test-prepare-watch in one terminal and yarn test-watch in another. This is the fastest way to perform incremental testing. If you are only interested in running test once, use yarn test.",
@@ -112,13 +108,13 @@
     "licenses-backend": "sbt dumpLicenseReport",
     "am-i-pretty": "node_modules/.bin/prettier --list-different --config .prettierrc \"frontend/javascripts/**/*.ts\" \"frontend/javascripts/**/*.tsx\" \"tools/**/*.js\"",
     "docs": "node_modules/.bin/documentation build --shallow frontend/javascripts/oxalis/api/api_loader.ts frontend/javascripts/oxalis/api/api_latest.ts --github --project-name \"WEBKNOSSOS Frontend API\" --format html --output public/docs/frontend-api",
-    "refresh-schema": "./tools/postgres/refresh_schema.sh && rm -f target/scala-2.12/src_managed/schema/com/scalableminds/webknossos/schema/Tables.scala",
-    "enable-jobs": "sed -i -e 's/jobsEnabled = false/jobsEnabled = true/g' ./conf/application.conf; ./tools/postgres/set_jobs.sh true",
-    "disable-jobs": "sed -i -e 's/jobsEnabled = true/jobsEnabled = false/g' ./conf/application.conf; ./tools/postgres/set_jobs.sh false",
+    "refresh-schema": "./tools/postgres/dbtool.js refresh-schema && rm -f target/scala-2.12/src_managed/schema/com/scalableminds/webknossos/schema/Tables.scala",
+    "enable-jobs": "sed -i -e 's/jobsEnabled = false/jobsEnabled = true/g' ./conf/application.conf; ./tools/postgres/dbtool.js enable-jobs",
+    "disable-jobs": "sed -i -e 's/jobsEnabled = true/jobsEnabled = false/g' ./conf/application.conf; ./tools/postgres/dbtool.js disable-jobs",
     "coverage-local": "c8 report --reporter=html && echo Success! Open coverage/index.html",
     "coverage": "c8 report --reporter=text-lcov | coveralls",
     "postcheckout": "echo 'Deleting auto-generated typescript files...' && rm -f frontend/javascripts/test/snapshots/type-check/*.ts",
-    "apply-evolutions": "tools/postgres/apply_evolutions.sh",
+    "apply-evolutions": "tools/postgres/dbtool.js apply-evolutions",
     "postinstall": "cd tools/proxy && yarn install",
     "typecheck": "yarn tsc",
     "ts": "yarn tsc",
@@ -257,11 +253,11 @@
   "madge": {
     "detectiveOptions": {
       "ts": {
-            "skipTypeImports": true
-          },
-          "tsx": {
-            "skipTypeImports": true
-          }
+        "skipTypeImports": true
+      },
+      "tsx": {
+        "skipTypeImports": true
+      }
     }
   }
 }