From 79d21cbf269426375fb5862ced24f247079ab949 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Mon, 6 Feb 2023 16:03:55 +0100 Subject: [PATCH] Rewrite database tooling and support for postgres passwords (#6803) * adds Ok to health check * replace bash scripts with node script * fixes * fixes * fixes * fixes * fixes * logging * fixes * fixes * fixes * try other user/password * use env vars * fixes * changelog * check 'jdbc:' prefix * fix * Apply suggestions from code review Co-authored-by: Florian M Co-authored-by: Philipp Otto * pr feedback * fix * fix * fixes * pr feedback * fix * fix * fix * fix * fix --------- Co-authored-by: Florian M Co-authored-by: Philipp Otto --- .circleci/config.yml | 8 +- CHANGELOG.unreleased.md | 1 + Dockerfile | 6 +- MIGRATIONS.unreleased.md | 2 + app/Startup.scala | 61 +-- app/utils/WkConf.scala | 8 + conf/slick.conf | 2 + docker-compose.yml | 26 +- frontend/javascripts/test/enzyme/e2e-setup.ts | 2 +- package.json | 26 +- project/AssetCompilation.scala | 20 +- project/Dependencies.scala | 6 +- tools/dev_deployment/refresh_schema.sh | 2 +- tools/postgres/apply_evolutions.sh | 35 -- .../assert_unique_evolution_numbers.sh | 9 - tools/postgres/db_host.sh | 11 - tools/postgres/db_name.sh | 11 - tools/postgres/dbtool.js | 450 ++++++++++++++++++ tools/postgres/diff_schema.js | 186 -------- tools/postgres/drop_db.sh | 9 - tools/postgres/dump_schema.sh | 43 -- tools/postgres/ensure_db.sh | 17 - tools/postgres/ensure_schema.sh | 18 - tools/postgres/prepareTestDB.sh | 18 - tools/postgres/refresh_db.sh | 6 - tools/postgres/refresh_schema.sh | 9 - tools/postgres/set_jobs.sh | 17 - yarn.lock | 31 +- 28 files changed, 546 insertions(+), 494 deletions(-) delete mode 100755 tools/postgres/apply_evolutions.sh delete mode 100755 tools/postgres/assert_unique_evolution_numbers.sh delete mode 100755 tools/postgres/db_host.sh delete mode 100755 tools/postgres/db_name.sh create mode 100755 tools/postgres/dbtool.js delete mode 100755 tools/postgres/diff_schema.js delete mode 100755 tools/postgres/drop_db.sh delete mode 100755 tools/postgres/dump_schema.sh delete mode 100755 tools/postgres/ensure_db.sh delete mode 100755 tools/postgres/ensure_schema.sh delete mode 100755 tools/postgres/prepareTestDB.sh delete mode 100755 tools/postgres/refresh_db.sh delete mode 100755 tools/postgres/refresh_schema.sh delete mode 100755 tools/postgres/set_jobs.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a1c79c9577..a775abeb708 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,12 +47,12 @@ jobs: keys: - yarn-cache-{{ checksum ".circleci/cache_version" }}-{{ checksum "yarn.lock" }} - yarn-cache-{{ checksum ".circleci/cache_version" }}- - - run: - name: Assert unique evolution numbers - command: tools/postgres/assert_unique_evolution_numbers.sh - run: name: Install frontend dependencies command: docker-compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --frozen-lockfile + - run: + name: Assert unique evolution numbers + command: docker-compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers - restore_cache: name: Restore webpack cache keys: @@ -63,7 +63,7 @@ jobs: command: | docker-compose up -d postgres sleep 3 - docker-compose run compile tools/postgres/diff_schema.js tools/postgres/schema.sql "conf/evolutions/*.sql" + docker-compose run compile tools/postgres/dbtool.js check-evolutions-schema - run: name: Build frontend documentation command: docker-compose run base yarn run docs diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index f9d0e97c961..adc49e3a9f3 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -14,6 +14,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Changed - Limit paid team sharing features to respective organization plans. [#6767](https://github.com/scalableminds/webknossos/pull/6776) +- Rewrite the database tools in `tools/postgres` to JavaScript and adding support for non-default Postgres username-password combinations. [#6803](https://github.com/scalableminds/webknossos/pull/6803) ### Fixed - Fixed a benign error message which briefly appeared after logging in. [#6810](https://github.com/scalableminds/webknossos/pull/6810) diff --git a/Dockerfile b/Dockerfile index d20e6160571..3e8a54a23a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM eclipse-temurin:11 -RUN apt-get update \ - && apt-get -y install libblosc1 postgresql-client git \ +ARG VERSION_NODE="16.x" + +RUN curl -sL "https://deb.nodesource.com/setup_${VERSION_NODE}" | bash - \ + && apt-get -y install libblosc1 postgresql-client git nodejs \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /webknossos diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md index 957ece2b00d..1cb642c190a 100644 --- a/MIGRATIONS.unreleased.md +++ b/MIGRATIONS.unreleased.md @@ -8,4 +8,6 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md). ## Unreleased [Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...HEAD) +- WEBKNOSSOS now requires Node.js not only for development and building, but also for execution. The prebuilt Docker images already contain this dependency. If you're using these, nothing needs to be changed. [#6803](https://github.com/scalableminds/webknossos/pull/6803) + ### Postgres Evolutions: diff --git a/app/Startup.scala b/app/Startup.scala index 77413110053..5ba2f5f5a2e 100755 --- a/app/Startup.scala +++ b/app/Startup.scala @@ -1,13 +1,10 @@ -import java.io.{ByteArrayOutputStream, File} - import akka.actor.{ActorSystem, Props} import com.typesafe.scalalogging.LazyLogging import controllers.InitialDataService -import io.apigee.trireme.core.{NodeEnvironment, Sandbox} -import javax.inject._ import models.annotation.AnnotationDAO import models.user.InviteService import net.liftweb.common.{Failure, Full} +import org.apache.http.client.utils.URIBuilder import oxalis.cleanup.CleanUpService import oxalis.files.TempFileService import oxalis.mail.{Mailer, MailerConfig} @@ -17,6 +14,8 @@ import play.api.inject.ApplicationLifecycle import utils.WkConf import utils.sql.SqlClient +import javax.inject._ +import scala.collection.mutable import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.concurrent.duration._ @@ -69,8 +68,19 @@ class Startup @Inject()(actorSystem: ActorSystem, } } + private lazy val postgresUrl = { + val slickUrl = + if (conf.Slick.Db.url.startsWith("jdbc:")) + conf.Slick.Db.url.substring(5) + else conf.Slick.Db.url + val uri = new URIBuilder(slickUrl) + uri.setUserInfo(conf.Slick.Db.user, conf.Slick.Db.password) + uri.build().toString + } + if (conf.Slick.checkSchemaOnStartup) { ensurePostgresDatabase() + ensurePostgresSchema() } initialDataService.insert.futureBox.map { @@ -81,38 +91,31 @@ class Startup @Inject()(actorSystem: ActorSystem, case _ => () } - private def ensurePostgresDatabase(): Unit = { - logger.info("Running ensure_db.sh with POSTGRES_URL " + sys.env.get("POSTGRES_URL")) - - val processLogger = ProcessLogger((o: String) => logger.info(o), (e: String) => logger.error(e)) - - // this script is copied to the stage directory in AssetCompilation - val result = "./tools/postgres/ensure_db.sh" ! processLogger + private def ensurePostgresSchema(): Unit = { + logger.info("Checking database schema…") - if (result != 0) - throw new Exception("Could not ensure Postgres database. Is postgres installed?") + val errorMessageBuilder = mutable.ListBuffer[String]() + val capturingProcessLogger = + ProcessLogger((o: String) => errorMessageBuilder.append(o), (e: String) => logger.error(e)) - // diffing the actual DB schema against schema.sql: - logger.info("Running diff_schema.js tools/postgres/schema.sql DB") - val nodeEnv = new NodeEnvironment() - nodeEnv.setDefaultNodeVersion("0.12") - val nodeOutput = new ByteArrayOutputStream() - nodeEnv.setSandbox(new Sandbox().setStdout(nodeOutput).setStderr(nodeOutput)) - val script = nodeEnv.createScript( - "diff_schema.js", - new File("tools/postgres/diff_schema.js"), - Array("tools/postgres/schema.sql", "DB") - ) - val status = script.execute().get() - if (status.getExitCode == 0) { - logger.info("Schema is up to date.") + val result = Process("./tools/postgres/dbtool.js check-db-schema", None, "POSTGRES_URL" -> postgresUrl) ! capturingProcessLogger + if (result == 0) { + logger.info("Database schema is up to date.") } else { - val nodeOut = new String(nodeOutput.toByteArray, "UTF-8") - val errorMessage = s"Database schema does not fit to schema.sql! \n $nodeOut" + val errorMessage = errorMessageBuilder.toList.mkString("\n") logger.error(errorMessage) slackNotificationService.warn("SQL schema mismatch", errorMessage) } + } + + private def ensurePostgresDatabase(): Unit = { + logger.info(s"Ensuring Postgres database…") + val processLogger = ProcessLogger((o: String) => logger.info(o), (e: String) => logger.error(e)) + // this script is copied to the stage directory in AssetCompilation + val result = Process("./tools/postgres/dbtool.js ensure-db", None, "POSTGRES_URL" -> postgresUrl) ! processLogger + if (result != 0) + throw new Exception("Could not ensure Postgres database. Is postgres installed?") } private def startActors(actorSystem: ActorSystem) = { diff --git a/app/utils/WkConf.scala b/app/utils/WkConf.scala index d1469712595..31b307d9241 100644 --- a/app/utils/WkConf.scala +++ b/app/utils/WkConf.scala @@ -203,6 +203,14 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L object Slick { val checkSchemaOnStartup: Boolean = get[Boolean]("slick.checkSchemaOnStartup") + + object Db { + val url: String = get[String]("slick.db.url") + val user: String = get[String]("slick.db.user") + val password: String = get[String]("slick.db.password") + } + + val children = List(Db) } object Voxelytics { diff --git a/conf/slick.conf b/conf/slick.conf index bc1324b1abf..033e18d8647 100644 --- a/conf/slick.conf +++ b/conf/slick.conf @@ -8,7 +8,9 @@ slick = { driver = org.postgresql.Driver keepAliveConnection = true user = "postgres" + user = ${?POSTGRES_USER} password = "postgres" + password = ${?POSTGRES_PASSWORD} queueSize = 5000 } checkSchemaOnStartup = true diff --git a/docker-compose.yml b/docker-compose.yml index 906b97477d2..f98908d7ee1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,9 @@ services: volumes: - ./binaryData:/webknossos/binaryData environment: - - POSTGRES_URL=jdbc:postgresql://postgres/webknossos + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password user: ${USER_UID:-1000}:${USER_GID:-1000} webknossos-datastore: @@ -95,6 +97,8 @@ services: - COVERALLS_REPO_TOKEN - TZ=${TZ:-Europe/Berlin} - POSTGRES_URL=jdbc:postgresql://postgres/webknossos + - POSTGRES_USER=webknossos_user + - POSTGRES_PASSWORD=secret_password - HOME=/root - CIRCLE_TAG=${CIRCLE_TAG} - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM} @@ -133,7 +137,9 @@ services: fossildb-dev: condition: service_healthy environment: - - POSTGRES_URL=jdbc:postgresql://postgres/webknossos + POSTGRES_URL: jdbc:postgresql://postgres/webknossos + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password command: - bash - -c @@ -175,7 +181,9 @@ services: fossildb: condition: service_healthy environment: - - POSTGRES_URL=jdbc:postgresql://postgres/webknossos_testing + POSTGRES_URL: jdbc:postgresql://postgres/webknossos_testing + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password command: - bash - -c @@ -212,8 +220,8 @@ services: image: postgres:10-alpine environment: POSTGRES_DB: webknossos - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_USER: webknossos_user + POSTGRES_PASSWORD: secret_password healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -h 127.0.0.1 -p 5432"] interval: 2s @@ -238,27 +246,27 @@ services: psql: extends: postgres - command: psql -h postgres -U postgres webknossos + command: psql -h postgres -U webknossos_user webknossos links: - "postgres-persisted:postgres" depends_on: postgres-persisted: condition: service_healthy environment: - PGPASSWORD: postgres + PGPASSWORD: secret_password volumes: - ./conf/evolutions/:/evolutions/ drop-db: extends: postgres - command: psql -h postgres -U postgres postgres -c "DROP DATABASE webknossos" + command: psql -h postgres -U webknossos_user postgres -c "DROP DATABASE webknossos" links: - "postgres-persisted:postgres" depends_on: postgres-dev: condition: service_healthy environment: - PGPASSWORD: postgres + PGPASSWORD: secret_password # FossilDB fossildb: diff --git a/frontend/javascripts/test/enzyme/e2e-setup.ts b/frontend/javascripts/test/enzyme/e2e-setup.ts index b9ea5757e89..66a45426f53 100644 --- a/frontend/javascripts/test/enzyme/e2e-setup.ts +++ b/frontend/javascripts/test/enzyme/e2e-setup.ts @@ -140,6 +140,6 @@ export function resetDatabase() { // The parameter needs to be set globally here. // See https://github.com/shelljs/shelljs/issues/981#issuecomment-626840798 shell.config.fatal = true; - shell.exec("tools/postgres/prepareTestDB.sh", { silent: true }); + shell.exec("tools/postgres/dbtool.js prepare-test-db", { silent: true }); } export { tokenUserA, tokenUserB, tokenUserC, tokenUserD, tokenUserE, setCurrToken }; diff --git a/package.json b/package.json index b1ba119fc51..e000d0846f7 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "abort-controller": "^3.0.0", "ava": "^3.13.0", "browserslist-to-esbuild": "^1.2.0", - "commander": "^2.17.1", + "commander": "^10.0.0", "coveralls": "^3.0.2", "css-loader": "^6.5.1", "documentation": "^13.2.5", @@ -43,7 +43,6 @@ "eslint-plugin-react-hooks": "^4.6.0", "eslint_d": "^12.2.1", "espree": "^3.5.4", - "glob": "^7.1.3", "husky": "^0.14.3", "jsdoc": "^3.5.5", "jsdom": "^11.5.1", @@ -62,12 +61,9 @@ "prettier": "2.7.1", "proto-loader6": "^0.4.0", "puppeteer": "^13.3.2", - "randomstring": "^1.1.5", "react-test-renderer": "^16.8.0", "redux-mock-store": "^1.2.2", - "replace-in-file": "1.1.3", "retoggle": "^0.3.0", - "rimraf": "^2.6.2", "shelljs": "^0.8.5", "sinon": "^12.0.1", "tmp": "0.0.33", @@ -95,7 +91,7 @@ "test-watch": "tools/test.sh test --watch", "test-prepare": "tools/test.sh prepare", "test-prepare-watch": "tools/test.sh prepare -w --verbose", - "test-e2e": "tools/postgres/prepareTestDB.sh && tools/test.sh test-e2e --timeout=60s --verbose", + "test-e2e": "tools/postgres/dbtool.js prepare-test-db && tools/test.sh test-e2e --timeout=60s --verbose", "test-screenshot": "tools/test.sh test-screenshot --timeout=5m", "refresh-screenshots": "rm -rf frontend/javascripts/test/screenshots/** && docker-compose up screenshot-tests", "test-help": "echo For development it is recommended to run yarn test-prepare-watch in one terminal and yarn test-watch in another. This is the fastest way to perform incremental testing. If you are only interested in running test once, use yarn test.", @@ -112,13 +108,13 @@ "licenses-backend": "sbt dumpLicenseReport", "am-i-pretty": "node_modules/.bin/prettier --list-different --config .prettierrc \"frontend/javascripts/**/*.ts\" \"frontend/javascripts/**/*.tsx\" \"tools/**/*.js\"", "docs": "node_modules/.bin/documentation build --shallow frontend/javascripts/oxalis/api/api_loader.ts frontend/javascripts/oxalis/api/api_latest.ts --github --project-name \"WEBKNOSSOS Frontend API\" --format html --output public/docs/frontend-api", - "refresh-schema": "./tools/postgres/refresh_schema.sh && rm -f target/scala-2.12/src_managed/schema/com/scalableminds/webknossos/schema/Tables.scala", - "enable-jobs": "sed -i -e 's/jobsEnabled = false/jobsEnabled = true/g' ./conf/application.conf; ./tools/postgres/set_jobs.sh true", - "disable-jobs": "sed -i -e 's/jobsEnabled = true/jobsEnabled = false/g' ./conf/application.conf; ./tools/postgres/set_jobs.sh false", + "refresh-schema": "./tools/postgres/dbtool.js refresh-schema && rm -f target/scala-2.12/src_managed/schema/com/scalableminds/webknossos/schema/Tables.scala", + "enable-jobs": "sed -i -e 's/jobsEnabled = false/jobsEnabled = true/g' ./conf/application.conf; ./tools/postgres/dbtool.js enable-jobs", + "disable-jobs": "sed -i -e 's/jobsEnabled = true/jobsEnabled = false/g' ./conf/application.conf; ./tools/postgres/dbtool.js disable-jobs", "coverage-local": "c8 report --reporter=html && echo Success! Open coverage/index.html", "coverage": "c8 report --reporter=text-lcov | coveralls", "postcheckout": "echo 'Deleting auto-generated typescript files...' && rm -f frontend/javascripts/test/snapshots/type-check/*.ts", - "apply-evolutions": "tools/postgres/apply_evolutions.sh", + "apply-evolutions": "tools/postgres/dbtool.js apply-evolutions", "postinstall": "cd tools/proxy && yarn install", "typecheck": "yarn tsc", "ts": "yarn tsc", @@ -257,11 +253,11 @@ "madge": { "detectiveOptions": { "ts": { - "skipTypeImports": true - }, - "tsx": { - "skipTypeImports": true - } + "skipTypeImports": true + }, + "tsx": { + "skipTypeImports": true + } } } } diff --git a/project/AssetCompilation.scala b/project/AssetCompilation.scala index 787321d36d2..2ca0374f25a 100644 --- a/project/AssetCompilation.scala +++ b/project/AssetCompilation.scala @@ -5,7 +5,6 @@ import sbt.Keys._ import sbt._ import sys.process.Process - object AssetCompilation { object SettingsKeys { val yarnPath = SettingKey[String]("npm-path", "where npm is installed") @@ -16,12 +15,11 @@ object AssetCompilation { private def isWindowsSystem = System.getProperty("os.name").startsWith("Windows") - private def startProcess(app: String, params: List[String], base: File) = { + private def startProcess(app: String, params: List[String], base: File) = if (isWindowsSystem) Process("cmd" :: "/c" :: app :: params, base) else Process(app :: params, base) - } private def yarnInstall: Def.Initialize[Task[Seq[File]]] = Def task { try { @@ -72,8 +70,7 @@ object AssetCompilation { copyRecursively(baseDirectory.value / "tools" / "postgres", destination) } catch { case e: Exception => - streams.value.log - .error("Could not copy SQL schema to stage dir: " + e.getMessage) + streams.value.log.error("Could not copy SQL schema to stage dir: " + e.getMessage) } // copy test/db @@ -84,14 +81,11 @@ object AssetCompilation { copyRecursively(baseDirectory.value / "test" / "db", destination) } catch { case e: Exception => - streams.value.log - .error("Could not test database entries to stage dir: " + e.getMessage) + streams.value.log.error("Could not test database entries to stage dir: " + e.getMessage) } // copy node_modules for diff_schema.js { - val nodeModules = - Seq("commander", "randomstring", "glob", "rimraf", "replace-in-file") val nodeSrc = baseDirectory.value / "node_modules" val nodeDest = target.value / "universal" / "stage" / "node_modules" val tmpPath = baseDirectory.value / "tmp" @@ -99,9 +93,7 @@ object AssetCompilation { tmpPath.mkdirs startProcess(yarnPath.value, List("init", "-y"), tmpPath) ! streamsValue - nodeModules.foreach(nodeModule => { - startProcess(yarnPath.value, List("add", (nodeSrc / nodeModule).getAbsolutePath), tmpPath) ! streamsValue - }) + startProcess(yarnPath.value, List("add", (nodeSrc / "commander").getAbsolutePath), tmpPath) ! streamsValue deleteRecursively(nodeDest) copyRecursively(tmpPath / "node_modules", nodeDest) deleteRecursively(tmpPath) @@ -127,8 +119,8 @@ object AssetCompilation { "Ensuring Postgres DB is running for Slick code generation..." ) startProcess( - (baseDirectoryValue / "tools" / "postgres" / "ensure_db.sh").toString, - List(), + (baseDirectoryValue / "tools" / "postgres" / "dbtool.js").toString, + List("ensure-db"), baseDirectoryValue ) ! streamsValue.log diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 077787dbe74..4ed7d9f176b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -36,8 +36,6 @@ object Dependencies { private val silhouette = "io.github.honeycomb-cheesecake" %% "play-silhouette" % silhouetteVersion private val silhouetteTestkit = "io.github.honeycomb-cheesecake" %% "play-silhouette-testkit" % silhouetteVersion % "test" private val silhouetteCrypto = "io.github.honeycomb-cheesecake" %% "play-silhouette-crypto-jca" % silhouetteVersion - private val trireme = "io.apigee.trireme" % "trireme-core" % "0.9.3" - private val triremeNode = "io.apigee.trireme" % "trireme-node12src" % "0.9.3" private val webknossosWrap = "com.scalableminds" %% "webknossos-wrap" % webknossosWrapVersion private val xmlWriter = "org.glassfish.jaxb" % "txw2" % "2.2.11" private val woodstoxXml = "org.codehaus.woodstox" % "wstx-asl" % "3.2.3" @@ -60,7 +58,7 @@ object Dependencies { "com.typesafe.slick" %% "slick" % "3.3.3", "com.typesafe.slick" %% "slick-hikaricp" % "3.3.3", "com.typesafe.slick" %% "slick-codegen" % "3.3.3", - "org.postgresql" % "postgresql" % "42.3.6" + "org.postgresql" % "postgresql" % "42.5.2" ) val utilDependencies: Seq[ModuleID] = Seq( @@ -120,8 +118,6 @@ object Dependencies { silhouetteTestkit, silhouetteCrypto, specs2 % Test, - trireme, - triremeNode, xmlWriter, woodstoxXml, jwt diff --git a/tools/dev_deployment/refresh_schema.sh b/tools/dev_deployment/refresh_schema.sh index 4d52a293436..7fea7b1de27 100755 --- a/tools/dev_deployment/refresh_schema.sh +++ b/tools/dev_deployment/refresh_schema.sh @@ -20,5 +20,5 @@ POD=$( echo "Going to run 'refresh_schema.sh' in pod $POD" ssh -t kube.scm.io " - sudo kubectl exec $POD -n $NAMESPACE -- /webknossos/tools/postgres/refresh_schema.sh + sudo kubectl exec $POD -n $NAMESPACE -- /webknossos/tools/postgres/dbtool.js refresh-schema " diff --git a/tools/postgres/apply_evolutions.sh b/tools/postgres/apply_evolutions.sh deleted file mode 100755 index b26a3daca4c..00000000000 --- a/tools/postgres/apply_evolutions.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -# find schema version -schema_version=`psql -A -U postgres -h localhost --dbname="webknossos" -t -c "SELECT * FROM webknossos.releaseinformation"` -echo "Schema version: ${schema_version}" - -# assert that the found schema version is a number -re='^[0-9]+$' -if ! [[ $schema_version =~ $re ]] ; then - echo "Error: Schema version is not a number" >&2; exit 1 -fi - -# get list of evolutions to apply -files="" -for entry in "${scriptdir}/../../conf/evolutions"/*.sql -do - tmp_number=`grep -oP "[0-9]{3}" <<< $entry` - evolution_number=$((10#$tmp_number)) # force the number to be decimal - if (( evolution_number > schema_version )); then - files="$files -f $entry" - fi; -done - -# apply evolutions -if [ -n "$files" ]; then - echo "Applying following evolutions: $files" - `PGPASSWORD=postgres psql -U postgres -h localhost --dbname="webknossos" -v ON_ERROR_STOP=ON -q $files` - echo "Successfully applied the evolutions after $schema_version" -else - echo "There are no evolutions that can be applied." -fi; - diff --git a/tools/postgres/assert_unique_evolution_numbers.sh b/tools/postgres/assert_unique_evolution_numbers.sh deleted file mode 100755 index 0f3d8ff9ff2..00000000000 --- a/tools/postgres/assert_unique_evolution_numbers.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -all="$(basename -a $(echo "$scriptdir/../../conf/evolutions/"*.sql) | cut -d'-' -f1 | sort)" -unique="$(echo "$all" | uniq)" - -diff <(echo "$all") <(echo "$unique") diff --git a/tools/postgres/db_host.sh b/tools/postgres/db_host.sh deleted file mode 100755 index c2132d2af2a..00000000000 --- a/tools/postgres/db_host.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -SED=sed -if [ -x "$(command -v gsed)" ]; then - SED=gsed -fi - -url=${POSTGRES_URL:-"jdbc:postgresql://localhost/webknossos"} - -echo $url | $SED -r 's#.*\/\/([^/]+).*#\1#' diff --git a/tools/postgres/db_name.sh b/tools/postgres/db_name.sh deleted file mode 100755 index 17669273158..00000000000 --- a/tools/postgres/db_name.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -SED=sed -if [ -x "$(command -v gsed)" ]; then - SED=gsed -fi - -url=${POSTGRES_URL:-"jdbc:postgresql://localhost/webknossos"} - -echo $url | $SED -r 's#.*\/(.+)#\1#' diff --git a/tools/postgres/dbtool.js b/tools/postgres/dbtool.js new file mode 100755 index 00000000000..2ca29afcf4f --- /dev/null +++ b/tools/postgres/dbtool.js @@ -0,0 +1,450 @@ +#!/usr/bin/env node +const { spawnSync } = require("child_process"); +const path = require("path"); +const fs = require("fs"); +const { Command } = require("commander"); + +const schemaPath = path.join(__dirname, "schema.sql"); +const evolutionsPath = path.resolve(path.join(__dirname, "..", "..", "conf", "evolutions")); + +const PG_CONFIG = (() => { + let rawUrl = process.env.POSTGRES_URL || "postgres://postgres:postgres@127.0.0.1:5432/webknossos"; + if (rawUrl.startsWith("jdbc:")) { + rawUrl = rawUrl.substring(5); + } + const url = new URL(rawUrl); + url.username = url.username + ? url.username + : process.env.POSTGRES_USER ?? process.env.PGUSER ?? "postgres"; + url.password = url.password + ? url.password + : process.env.POSTGRES_PASSWORD ?? process.env.PGPASSWORD ?? "postgres"; + url.port = url.port ? url.port : 5432; + + const urlWithDefaultDatabase = new URL(url); + urlWithDefaultDatabase.pathname = "/postgres"; + + return { + url: url.toString(), + urlWithDefaultDatabase: urlWithDefaultDatabase.toString(), + username: url.username, + password: url.password, + hostname: url.hostname, + port: url.port, + database: + url.pathname.length > 1 + ? url.pathname.substring(1) // remove leading '/'` + : "webknossos", + }; +})(); + +const program = new Command(); + +function safeSpawn(command, args, options = {}) { + const prc = spawnSync(command, args, { ...options, encoding: "utf-8" }); + if (prc.status !== 0) { + const error = new Error(`Exit code: ${prc.status}\Stderr:\n${prc.stderr}`); + error.status = prc.status; + error.stderr = prc.stderr; + error.stdout = prc.stdout; + throw error; + } + return prc.stdout; +} + +function safePsqlSpawn(args, options = {}) { + try { + return safeSpawn("psql", args, options); + } catch (err) { + throw new Error(`PSQL exit code: ${err.status}\nPSQL stderr:\n${err.stderr}`); + } +} + +function callPsql(sql) { + return safePsqlSpawn([PG_CONFIG.url, "-tAc", sql]); +} + +function dropDb() { + console.log( + safePsqlSpawn([PG_CONFIG.urlWithDefaultDatabase, "-c", `DROP DATABASE ${PG_CONFIG.database}`]), + ); +} + +function ensureDb() { + const doesDbExist = safePsqlSpawn([ + PG_CONFIG.urlWithDefaultDatabase, + "-tAc", + `SELECT 1 FROM pg_database WHERE datname='${PG_CONFIG.database}'`, + ]).trim(); + if (doesDbExist === "1") { + console.log("Database already exists"); + } else { + console.log( + safePsqlSpawn([ + PG_CONFIG.urlWithDefaultDatabase, + "-tAc", + `CREATE DATABASE ${PG_CONFIG.database};`, + ]), + ); + } + + const existingSchemaName = callPsql( + "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'webknossos';", + ).trim(); + if (existingSchemaName === "webknossos") { + console.log("Schema already exists"); + } else { + refreshSchema(); + } +} + +function prepareTestDb() { + ensureDb(); + refreshSchema(); + + const csvFolder = path.join(__dirname, "..", "..", "test", "db"); + for (const filename of fs.readdirSync(csvFolder)) { + if (filename.endsWith(".csv")) { + console.log(`IMPORT ${filename}`); + safePsqlSpawn( + [ + PG_CONFIG.url, + "-c", + `SET session_replication_role = replica; COPY webknossos.${filename.slice( + 0, + -4, + )} FROM STDOUT WITH CSV HEADER QUOTE ''''`, + ], + { + input: fs.readFileSync(path.join(csvFolder, filename)), + }, + ); + } + } + console.log("✨✨ Done preparing test database"); +} + +function refreshSchema() { + console.log(safePsqlSpawn([PG_CONFIG.url, "-v", "ON_ERROR_STOP=ON", "-f", schemaPath])); +} + +function dumpCurrentSchema(databaseUrl, schemaDir, silent = false) { + if (!fs.existsSync(schemaDir) || !fs.statSync(schemaDir).isDirectory) { + console.error("Schema directory $schemadir does not exist, aborting!"); + process.exit(1); + } + + for (const filename of fs.readdirSync(schemaDir)) { + fs.rmSync(path.join(schemaDir, filename), { recursive: true, force: true }); + } + + if (!silent) console.log(`Dumping database to ${schemaDir}.`); + + const items = safePsqlSpawn([databaseUrl, "-c", "\\d+ webknossos.*"]).trimEnd(); + for (const block of items.split("\n\n")) { + const [type, identifier] = block + .split("\n")[0] + .trim() + .split('"') + .map((s) => s.trim().replace(/"/g, "").replace(/ /g, "")); + if (!silent) console.log(type, identifier); + fs.writeFileSync(path.join(schemaDir, `${type}__${identifier}`), block + "\n"); + } + + const functions = safePsqlSpawn([databaseUrl, "-c", "\\df+ webknossos.*"]); + fs.writeFileSync(path.join(schemaDir, "Functions"), functions); + + const schemaVersion = safePsqlSpawn([ + databaseUrl, + "-tAc", + "SELECT schemaVersion FROM webknossos.releaseInformation;", + ]); + fs.writeFileSync(path.join(schemaDir, "schemaVersion"), schemaVersion); +} + +function cleanSchemaDump(dumpDir) { + for (const filename of fs.readdirSync(dumpDir)) { + fs.writeFileSync( + path.join(dumpDir, filename), + fs + .readFileSync(path.join(dumpDir, filename), { encoding: "utf-8" }) + .replace(/,$/gm, "") + .replace(/\\r/gm, " ") + .split("\n") + .sort() + .join("\n"), + ); + } +} + +function dumpExpectedSchema(sqlFilePaths) { + const tmpDbName = `webknossos_tmp_${Date.now()}`; + const tmpSchemaDir = fs.mkdtempSync("temp-webknossos-schema-"); + + try { + // Create tmp database + safePsqlSpawn([PG_CONFIG.urlWithDefaultDatabase, "-c", `CREATE DATABASE ${tmpDbName}`]); + + const urlWithDatabase = new URL(PG_CONFIG.urlWithDefaultDatabase); + urlWithDatabase.pathname = "/" + tmpDbName; + // Load schema into tmp database + safePsqlSpawn([ + urlWithDatabase.toString(), + "-v", + "ON_ERROR_STOP=ON", + "-q", + ...sqlFilePaths.flatMap((filePath) => ["-f", filePath]), + ]); + + // Dump schema into diffable files + dumpCurrentSchema(urlWithDatabase.toString(), tmpSchemaDir, true); + return tmpSchemaDir; + } finally { + safePsqlSpawn([PG_CONFIG.urlWithDefaultDatabase, "-c", `DROP DATABASE ${tmpDbName}`]); + } +} + +function checkDbSchema() { + const dbDumpDir = fs.mkdtempSync("temp-webknossos-schema-"); + let schemaDumpDir = null; + try { + dumpCurrentSchema(PG_CONFIG.url, dbDumpDir, true); + schemaDumpDir = dumpExpectedSchema([schemaPath]); + + cleanSchemaDump(dbDumpDir); + cleanSchemaDump(schemaDumpDir); + + try { + safeSpawn("diff", ["--strip-trailing-cr", "-r", dbDumpDir, schemaDumpDir]); + } catch (err) { + throw new Error(`Database schema is not up-to-date:\n${err.stdout}`); + } + } finally { + if (fs.existsSync(dbDumpDir)) { + fs.rmSync(dbDumpDir, { recursive: true, force: true }); + } + if (fs.existsSync(schemaDumpDir)) { + fs.rmSync(schemaDumpDir, { recursive: true, force: true }); + } + } + console.log("✨✨ Database schema is up-to-date"); +} + +function findEvolutionFiles() { + return fs + .readdirSync(evolutionsPath) + .filter((filename) => filename.endsWith(".sql")) + .map((filename) => { + const num = parseInt(filename.split("-")[0], 10); + return [num, filename]; + }) + .sort((a, b) => a[0] - b[0]); +} + +function checkEvolutionsSchema() { + let evolutionsDumpDir = null; + let schemaDumpDir = null; + try { + evolutionsDumpDir = dumpExpectedSchema( + findEvolutionFiles().map(([, evolutionFilename]) => + path.join(evolutionsPath, evolutionFilename), + ), + ); + schemaDumpDir = dumpExpectedSchema([schemaPath]); + + cleanSchemaDump(evolutionsDumpDir); + cleanSchemaDump(schemaDumpDir); + + try { + safeSpawn("diff", ["--strip-trailing-cr", "-r", evolutionsDumpDir, schemaDumpDir]); + } catch (err) { + throw new Error(`Evolutions do not match with schema file:\n${err.stdout}`); + } + } finally { + if (fs.existsSync(evolutionsDumpDir)) { + fs.rmSync(evolutionsDumpDir, { recursive: true, force: true }); + } + if (fs.existsSync(schemaDumpDir)) { + fs.rmSync(schemaDumpDir, { recursive: true, force: true }); + } + } + console.log("✨✨ Evolutions match with schema file"); +} + +function applyEvolutions() { + const schemaVersion = parseInt( + callPsql("SELECT schemaVersion FROM webknossos.releaseInformation;").trim(), + 10, + ); + if (isNaN(schemaVersion)) { + console.error("Error: Schema version is not a number"); + process.exit(1); + } + console.log(`Schema version: ${schemaVersion}`); + + // get list of evolutions to apply + const evolutions = findEvolutionFiles() + .filter(([num]) => num > schemaVersion) + .map(([, evolutionFilename]) => evolutionFilename); + + // apply evolutions + if (evolutions.length > 0) { + console.log(`Applying evolutions: ${evolutions}`); + safePsqlSpawn([ + PG_CONFIG.url, + "-v", + "ON_ERROR_STOP=ON", + "-q", + ...evolutions.flatMap((evolutionFilename) => [ + "-f", + path.join(evolutionsPath, evolutionFilename), + ]), + ]); + console.log("✨✨ Successfully applied the evolutions"); + } else { + console.log("There are no evolutions that can be applied."); + } +} + +function assertUniqueEvolutionNumbers() { + const groupedEvolutions = new Map(); + for (const filename of fs.readdirSync(evolutionsPath)) { + const num = parseInt(filename.split("-")[0], 10); + if (isNaN(num)) { + console.log("Found invalid evolution filename:", filename); + } + if (groupedEvolutions.has(num)) { + groupedEvolutions.get(num).push(filename); + } else { + groupedEvolutions.set(num, [filename]); + } + } + + if (Array.from(groupedEvolutions.values()).some((group) => group.length > 1)) { + console.log("Duplicate evolutions found:"); + for (const [num, filenames] of groupedEvolutions.entries()) { + if (filenames.length > 1) { + console.log(num, filenames); + } + } + + process.exit(1); + } + console.log("✨✨ All evolution numbers are unique"); +} + +program.name("dbtool").description("Tool for managing the WEBKNOSSOS database"); + +program + .command("drop-db") + .description("Drop the current database") + .action(() => { + dropDb(); + console.log("✨✨ Done"); + }); + +program + .command("drop-and-refresh-db") + .description("Drop the current database and initializes a new one") + .action(() => { + try { + dropDb(); + } catch (err) { + if (!err.message.includes("does not exist")) { + throw err; + } + } + ensureDb(); + console.log("✨✨ Done"); + }); + +program + .command("ensure-db") + .description("Make sure that database is initialized, creates one if necessary") + .action(() => { + ensureDb(); + console.log("✨✨ Done"); + }); + +program + .command("prepare-test-db") + .description("Sets up database for testing") + .action(() => { + prepareTestDb(); + }); + +program + .command("refresh-schema") + .description("Drop the current schema and initializes a new one") + .action(() => { + refreshSchema(); + console.log("✨✨ Done"); + }); + +program + .command("enable-jobs") + .description("Activates jobs in WEBKNOSSOS by registering a worker") + .action(() => { + console.log("Enabling jobs in the local database by inserting a worker."); + console.log( + callPsql( + `INSERT INTO webknossos.workers(_id, _dataStore, key) VALUES('6194dc03040200b0027f28a1', 'localhost', 'secretWorkerKey') ON CONFLICT DO NOTHING;`, + ), + ); + console.log("✨✨ Done"); + }); + +program + .command("disable-jobs") + .description("Deactivates jobs in WEBKNOSSOS by unregistering a worker") + .action(() => { + console.log(callPsql(`DELETE FROM webknossos.workers WHERE _id = '6194dc03040200b0027f28a1';`)); + console.log( + "If existing jobs prevent the delete, use yarn refresh-schema to reset the db or remove the existing jobs manually.", + ); + console.log("✨✨ Done"); + }); + +program + .command("dump-schema ") + .description("Dumps current schema into a folder") + .action((schemaDir) => { + dumpCurrentSchema(PG_CONFIG.url, schemaDir); + console.log("✨✨ Done"); + }); + +program + .command("check-db-schema") + .description("Compares the schema of SQL files against the database") + .action(() => { + try { + checkDbSchema(); + } catch (err) { + console.error(err.message); + process.exit(1); + } + }); + +program + .command("check-evolutions-schema") + .description("Compares the schema of SQL files against the evolutions") + .action(() => { + checkEvolutionsSchema(); + }); + +program + .command("apply-evolutions") + .description("Applies all necessary evolutions") + .action(() => { + applyEvolutions(); + }); + +program + .command("assert-unique-evolution-numbers") + .description("Checks that all evolution numbers are unique") + .action(() => { + assertUniqueEvolutionNumbers(); + }); + +program.showHelpAfterError(); +program.parse(process.argv); diff --git a/tools/postgres/diff_schema.js b/tools/postgres/diff_schema.js deleted file mode 100755 index 57b9ab2fb8c..00000000000 --- a/tools/postgres/diff_schema.js +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env node -// @noflow -/* eslint-disable import/no-extraneous-dependencies, prefer-template, prefer-arrow-callback */ -const program = require("commander"); -const randomstring = require("randomstring"); -const execSync = require("child_process").execSync; -const path = require("path"); -const fs = require("fs"); -const glob = require("glob"); -const rimraf = require("rimraf"); -const replace = require("replace-in-file"); - -const POSTGRES_URL = - typeof process.env.POSTGRES_URL !== "undefined" - ? process.env.POSTGRES_URL - : "jdbc:postgresql://localhost/webknossos"; -const envWithPostgresPwd = Object.assign({}, process.env, { - PGPASSWORD: "postgres", -}); -const scriptdir = __dirname; -const scriptName = __filename; - -let p1; -let p2; -let dir1; -let dir2; -let exitCode; - -function dump(parameter) { - if (parameter === "DB") { - return dumpToFolder(POSTGRES_URL); - } else { - const tmpdb = initTmpDB(); - const dbName = tmpdb[0]; - const dbHost = tmpdb[1]; - const postgresUrl = tmpdb[2]; - - console.log("Creating DB " + dbName); - execSync("psql -U postgres -h " + dbHost + " -c 'CREATE DATABASE " + dbName + ";'", { - env: envWithPostgresPwd, - }); - try { - loadDataIntoDB(parameter, dbHost, dbName); - return dumpToFolder(postgresUrl); - } catch (err) { - console.log(err); - process.exit(1); - } finally { - console.log("CLEANUP: DROP DATABASE " + dbName); - execSync("psql -U postgres -h " + dbHost + " -c 'DROP DATABASE " + dbName + ";'", { - env: envWithPostgresPwd, - }); - } - } - return null; -} - -function getEnvWithPostgresUrl(postgresUrl) { - return Object.assign({}, process.env, { - POSTGRES_URL: postgresUrl, - }); -} - -function initTmpDB() { - const tempDbName = generateRandomName(); - const postgresDirname = path.dirname(POSTGRES_URL); - const postgresUrl = postgresDirname + "/" + tempDbName; - - const dbName = execSync(scriptdir + "/db_name.sh", { - env: getEnvWithPostgresUrl(postgresUrl), - }) - .toString() - .trim(); // "trim" to remove the line break - if (dbName !== tempDbName) { - console.log("Wrong temporary dbName, got", dbName, "expected", tempDbName); - process.exit(1); - } - const dbHost = execSync(scriptdir + "/db_host.sh", { - env: getEnvWithPostgresUrl(postgresUrl), - }) - .toString() - .trim(); - return [dbName, dbHost, postgresUrl]; -} - -function loadDataIntoDB(parameter, dbHost, dbName) { - const fileNames = glob.sync(parameter); - const concatenateFileNames = fileNames.map((name) => "-f " + name).join(" "); - // prettier-ignore - execSync( - "psql -U postgres -h " + dbHost + " --dbname='" + dbName + "' -v ON_ERROR_STOP=ON -q " + concatenateFileNames, - { env: envWithPostgresPwd } - ); -} - -function dumpToFolder(postgresUrl) { - const tmpDir = execSync("mktemp -d").toString().trim(); - try { - execSync(scriptdir + "/dump_schema.sh " + tmpDir, { - env: getEnvWithPostgresUrl(postgresUrl), - }); - } catch (err) { - console.log("CLEANUP: remove " + tmpDir); - rimraf.sync(tmpDir); - process.exit(1); - } - return tmpDir; -} - -function generateRandomName() { - const random = randomstring.generate({ - length: 8, - charset: "alphanumeric", - capitalization: "lowercase", - }); - return "wk_tmp_" + random; -} - -function sortAndClean(dumpedDir) { - glob.sync(dumpedDir + "/**", { nodir: true }).forEach(function (fileName) { - replace({ files: fileName, replace: /,$/gm, with: "" }); - replace({ files: fileName, replace: /\\r/gm, with: " " }); - sortFile(fileName); - }); -} - -function sortFile(fileName) { - try { - const content = fs.readFileSync(fileName); - let lines = content.toString().split("\n"); - lines = lines.sort(); - fs.writeFileSync(fileName, lines.join("\n")); - } catch (err) { - console.log("FAILED to sort file " + fileName); - process.exit(1); - } -} - -program - .version("0.1.0", "-v, --version") - .arguments(" ") - .action(function (parameter1, parameter2) { - p1 = parameter1; - p2 = parameter2; - }); - -if (process.argv.length !== 4) { - // 2 "real" parameter - console.log("Usage: $0 "); - console.log("Examples:"); - console.log(" node ", scriptName, ' tools/postgres/schema.sql "conf/evolutions/*.sql"'); - console.log(" node ", scriptName, " tools/postgres/schema.sql DB"); - process.exit(1); -} - -try { - program.parse(process.argv); - dir1 = dump(p1); - dir2 = dump(p2); - // sort and remove commas and occurences of "\r" - sortAndClean(dir1); - sortAndClean(dir2); - // diff - try { - execSync("diff --strip-trailing-cr -r " + dir1 + " " + dir2); - exitCode = 0; - console.log("[SUCCESS] Schemas do match"); - } catch (err) { - exitCode = 1; - console.log(err.stdout.toString("utf8")); - console.log("[FAILED] Schemas do not match"); - } -} catch (err) { - console.log(err); - exitCode = 2; -} finally { - if (typeof dir1 !== "undefined" && dir1) { - console.log("CLEANUP: remove " + dir1); - rimraf.sync(dir1); - } - if (typeof dir2 !== "undefined" && dir2) { - console.log("CLEANUP: remove " + dir2); - rimraf.sync(dir2); - } - process.exit(exitCode); -} diff --git a/tools/postgres/drop_db.sh b/tools/postgres/drop_db.sh deleted file mode 100755 index d66684feb87..00000000000 --- a/tools/postgres/drop_db.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -PGPASSWORD=postgres psql -U postgres -h $dbHost -c "DROP DATABASE $dbName;" diff --git a/tools/postgres/dump_schema.sh b/tools/postgres/dump_schema.sh deleted file mode 100755 index 3faf3293b27..00000000000 --- a/tools/postgres/dump_schema.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -FORMATING=$(cat <<-"EOM" -{ - folder=(schemadir "/" $1); - file=$2; - gsub(/ /, "", folder); - gsub(/ /, "", file); - system("mkdir -p " folder); - print > (folder "/" file) -} -EOM -) - -if [ "$#" -ne 1 ] -then - echo "Usage: $0 " - exit 1 -fi - -schemadir="$1" - -if ! [ -d "$schemadir" ]; then - echo "Schema directory $schemadir does not exist, aborting!" - exit 1 -fi -rm -rf "$schemadir"/* - -echo "dumping $dbName to $schemadir" 1>&2 - -PGPASSWORD=postgres psql -U postgres -h "$dbHost" --dbname "$dbName" -c "\d+ webknossos.*" | \ - awk -v RS= -v FS='"' -v schemadir="$schemadir" "$FORMATING" - -PGPASSWORD=postgres psql -U postgres -h "$dbHost" --dbname "$dbName" -c "\df+ webknossos.*" > "$schemadir/Functions" - -PGPASSWORD=postgres psql -U postgres -h "$dbHost" --dbname "$dbName" \ - -At -c " SELECT schemaVersion FROM webknossos.releaseInformation;" > "$schemadir/schemaVersion" diff --git a/tools/postgres/ensure_db.sh b/tools/postgres/ensure_db.sh deleted file mode 100755 index 9df99037101..00000000000 --- a/tools/postgres/ensure_db.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -dbExistence="$(PGPASSWORD=postgres psql -U postgres -h $dbHost -tAc "SELECT 1 FROM pg_database WHERE datname='$dbName'" )" -if [ "$dbExistence" = '1' ] -then - echo "Database already exists" -else - PGPASSWORD=postgres psql -U postgres -h $dbHost -c "CREATE DATABASE $dbName;" -fi - -"$(dirname "$0")"/ensure_schema.sh diff --git a/tools/postgres/ensure_schema.sh b/tools/postgres/ensure_schema.sh deleted file mode 100755 index 7bc9ed34c0b..00000000000 --- a/tools/postgres/ensure_schema.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -schemaPath="$scriptdir/schema.sql" - -schemaName="$(PGPASSWORD=postgres psql -U postgres -h $dbHost --dbname=$dbName -tAc "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'webknossos';")" -if [ "$schemaName" = 'webknossos' ] -then - echo "Schema already exists" - exit -fi - -PGPASSWORD=postgres psql -U postgres -h $dbHost --dbname=$dbName -f "$schemaPath" diff --git a/tools/postgres/prepareTestDB.sh b/tools/postgres/prepareTestDB.sh deleted file mode 100755 index 0fa80ea0228..00000000000 --- a/tools/postgres/prepareTestDB.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -"$scriptdir"/ensure_db.sh -"$scriptdir"/refresh_schema.sh - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -for file in $(find "$scriptdir"/../../test/db -name "*.csv") -do - echo $file - PGPASSWORD=postgres psql -U postgres -h $dbHost --dbname=$dbName -c "SET session_replication_role = replica;COPY webknossos.$(basename $file .csv) FROM STDOUT WITH CSV HEADER QUOTE ''''" < $file -done; - -echo "Done preparing test db (host=$dbHost, name=$dbName)" diff --git a/tools/postgres/refresh_db.sh b/tools/postgres/refresh_db.sh deleted file mode 100755 index cc410c9ccba..00000000000 --- a/tools/postgres/refresh_db.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -"$(dirname "$0")"/drop_db.sh || true - -"$(dirname "$0")"/ensure_db.sh diff --git a/tools/postgres/refresh_schema.sh b/tools/postgres/refresh_schema.sh deleted file mode 100755 index 0c73b60422b..00000000000 --- a/tools/postgres/refresh_schema.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -PGPASSWORD=postgres psql -U postgres -h $dbHost --dbname=$dbName -f "$scriptdir"/schema.sql diff --git a/tools/postgres/set_jobs.sh b/tools/postgres/set_jobs.sh deleted file mode 100755 index 8799f21e4c9..00000000000 --- a/tools/postgres/set_jobs.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -scriptdir="$(dirname "$0")" - -dbName="$("$scriptdir"/db_name.sh)" -dbHost="$("$scriptdir"/db_host.sh)" - -if [[ $1 == "true" ]]; then - echo "Enabling jobs in the local database by inserting a worker." - PGPASSWORD=postgres psql -U postgres -h "$dbHost" --dbname="$dbName" -c "insert into webknossos.workers(_id, _dataStore, key) values('6194dc03040200b0027f28a1', 'localhost', 'secretWorkerKey') on conflict do nothing" -elif [[ $1 == "false" ]]; then - echo "Disabling jobs in the local database by removing the worker. If existing jobs prevent the delete, use yarn refresh-schema to reset the db or remove the existing jobs manually." - PGPASSWORD=postgres psql -U postgres -h "$dbHost" --dbname="$dbName" -c "delete from webknossos.workers where _id = '6194dc03040200b0027f28a1'" -else - echo "set_jobs was called without valid parameter. Either supply 'true' or 'false' "; -fi diff --git a/yarn.lock b/yarn.lock index 78405e570fb..98fd6651c9b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2359,11 +2359,6 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -array-uniq@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.2.tgz#5fcc373920775723cfd64d65c64bef53bf9eba6d" - integrity sha1-X8w3OSB3VyPP1k1lxkvvU7+eum0= - array-unique@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" @@ -3451,7 +3446,12 @@ comma-separated-tokens@^1.0.0: resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== -commander@^2.14.1, commander@^2.16.0, commander@^2.17.1, commander@^2.19.0, commander@^2.20.0, commander@^2.20.3, commander@^2.8.1, commander@^2.9.0: +commander@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1" + integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA== + +commander@^2.14.1, commander@^2.16.0, commander@^2.19.0, commander@^2.20.0, commander@^2.20.3, commander@^2.8.1, commander@^2.9.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== @@ -10427,13 +10427,6 @@ randombytes@^2.1.0: dependencies: safe-buffer "^5.1.0" -randomstring@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/randomstring/-/randomstring-1.1.5.tgz#6df0628f75cbd5932930d9fe3ab4e956a18518c3" - integrity sha1-bfBij3XL1ZMpMNn+OrTpVqGFGMM= - dependencies: - array-uniq "1.0.2" - range-parser@^1.2.1, range-parser@~1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" @@ -11564,11 +11557,6 @@ replace-ext@^1.0.0: resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.1.tgz#2d6d996d04a15855d967443631dd5f77825b016a" integrity sha512-yD5BHCe7quCgBph4rMQ+0KkIRKwWCrHDOX1p1Gp6HwjPM5kVoCdKGNhN7ydqqsX6lJEnQDKZ/tFMiEdQ1dvPEw== -replace-in-file@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/replace-in-file/-/replace-in-file-1.1.3.tgz#0e76076a408a0cded7990774904a34ae84c2bec7" - integrity sha1-DnYHakCKDN7XmQd0kEo0roTCvsc= - request-promise-core@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.4.tgz#3eedd4223208d419867b78ce815167d10593a22f" @@ -11801,13 +11789,6 @@ rimraf@3.0.2, rimraf@^3.0.2: dependencies: glob "^7.1.3" -rimraf@^2.6.2: - version "2.7.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" - integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== - dependencies: - glob "^7.1.3" - rome@^11.0.0: version "11.0.0" resolved "https://registry.yarnpkg.com/rome/-/rome-11.0.0.tgz#cd2f00fadfd3611399eba9a2f87612e1f3299a23"