From c8d3c2d1b6a9a82fe3eb6d2cedc4ba81bf63057c Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 19 Mar 2019 11:54:00 +0100 Subject: [PATCH 1/7] add wk-connect to the setup --- docker-compose.yml | 16 ++++++++++++++++ tools/proxy/proxy.js | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 6623d558908..0baf5e96164 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,11 +12,14 @@ services: - "fossildb-persisted:fossildb" - "postgres-persisted:postgres" - redis + - webknossos-connect depends_on: postgres-persisted: condition: service_healthy fossildb-persisted: condition: service_healthy + webknossos-connect: + condition: service_healthy command: - -Dconfig.file=conf/application.conf - -Djava.net.preferIPv4Stack=true @@ -122,11 +125,14 @@ services: - "fossildb-dev:fossildb" - "postgres-dev:postgres" - redis + - webknossos-connect depends_on: postgres-dev: condition: service_healthy fossildb-dev: condition: service_healthy + webknossos-connect: + condition: service_healthy environment: - POSTGRES_URL=jdbc:postgresql://postgres/webknossos command: @@ -166,11 +172,14 @@ services: - postgres - fossildb - redis + - webknossos-connect depends_on: postgres: condition: service_healthy fossildb: condition: service_healthy + webknossos-connect: + condition: service_healthy environment: - POSTGRES_URL=jdbc:postgresql://postgres/webknossos_testing command: @@ -195,6 +204,13 @@ services: - ".:/home/pptruser/webknossos" user: ${USER_UID:-1000}:${USER_GID:-1000} + # webKnossos-connect + webknossos-connect: + image: scalableminds/webknossos-connect:master__188 + volumes: + - "./conf/connect:/app/data" + network_mode: host + # Postgres postgres: image: postgres:10-alpine diff --git a/tools/proxy/proxy.js b/tools/proxy/proxy.js index e20ecafb9b0..7294f255c47 100644 --- a/tools/proxy/proxy.js +++ b/tools/proxy/proxy.js @@ -52,6 +52,10 @@ const processes = { shell: true, }, ), + connect: spawnIfNotSpecified("noConnect", "docker-compose run --rm webknossos-connect", [], { + cwd: ROOT, + shell: true, + }), }; function spawnIfNotSpecified(keyword, command, args, options) { From 9405f22d78bb956483d499263c04b83db632f2ea Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 19 Mar 2019 17:11:06 +0100 Subject: [PATCH 2/7] add connect datastore in initialData and test db --- app/controllers/InitialDataController.scala | 9 +++++++++ app/utils/WkConf.scala | 2 +- test/db/dataStores.csv | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/controllers/InitialDataController.scala b/app/controllers/InitialDataController.scala index d835bc670bb..1b015b962c9 100644 --- a/app/controllers/InitialDataController.scala +++ b/app/controllers/InitialDataController.scala @@ -94,6 +94,7 @@ Samplecountry for { _ <- insertLocalDataStoreIfEnabled _ <- insertLocalTracingStoreIfEnabled + _ <- insertConnectDataStore _ <- assertInitialDataEnabled _ <- assertNoOrganizationsPresent _ <- insertOrganization @@ -211,6 +212,14 @@ Samplecountry } } else Fox.successful(()) + def insertConnectDataStore: Fox[Any] = + dataStoreDAO.findOneByName("connect").futureBox.map { maybeStore => + if (maybeStore.isEmpty) { + logger.info("inserting connect datastore") + dataStoreDAO.insertOne(DataStore("connect", "http://localhost:8000", "secret-key", isConnector=true)) + } + } + def insertLocalTracingStoreIfEnabled: Fox[Any] = if (conf.Tracingstore.enabled) { tracingStoreDAO.findOneByName("localhost").futureBox.map { maybeStore => diff --git a/app/utils/WkConf.scala b/app/utils/WkConf.scala index 8da45400f06..6e0cbefd92f 100644 --- a/app/utils/WkConf.scala +++ b/app/utils/WkConf.scala @@ -125,5 +125,5 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader { } val children = - List(Application, Http, Mail, WebKnossos, Datastore, User, Braintracing, Features, Silhouette, Airbrake, Google) + List(Application, Http, Mail, WebKnossos, Datastore, Tracingstore, User, Braintracing, Features, Silhouette, Airbrake, Google) } diff --git a/test/db/dataStores.csv b/test/db/dataStores.csv index b3b56bfa6d8..0d4f93383fd 100644 --- a/test/db/dataStores.csv +++ b/test/db/dataStores.csv @@ -1,2 +1,3 @@ name,url,key,isScratch,isDeleted,isForeign,isConnector 'localhost','http://localhost:9000','something-secure',f,f,f,f +'connect','http://localhost:8000','secret-key',f,f,f,t From 88d479b2d30564481e293b797e081b5c125eb91d Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 19 Mar 2019 18:35:50 +0100 Subject: [PATCH 3/7] pretty --- app/controllers/InitialDataController.scala | 2 +- app/utils/WkConf.scala | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/controllers/InitialDataController.scala b/app/controllers/InitialDataController.scala index 1b015b962c9..d19164822b5 100644 --- a/app/controllers/InitialDataController.scala +++ b/app/controllers/InitialDataController.scala @@ -216,7 +216,7 @@ Samplecountry dataStoreDAO.findOneByName("connect").futureBox.map { maybeStore => if (maybeStore.isEmpty) { logger.info("inserting connect datastore") - dataStoreDAO.insertOne(DataStore("connect", "http://localhost:8000", "secret-key", isConnector=true)) + dataStoreDAO.insertOne(DataStore("connect", "http://localhost:8000", "secret-key", isConnector = true)) } } diff --git a/app/utils/WkConf.scala b/app/utils/WkConf.scala index 6e0cbefd92f..92c05b6bd49 100644 --- a/app/utils/WkConf.scala +++ b/app/utils/WkConf.scala @@ -125,5 +125,16 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader { } val children = - List(Application, Http, Mail, WebKnossos, Datastore, Tracingstore, User, Braintracing, Features, Silhouette, Airbrake, Google) + List(Application, + Http, + Mail, + WebKnossos, + Datastore, + Tracingstore, + User, + Braintracing, + Features, + Silhouette, + Airbrake, + Google) } From fffb630b0e688a4350d7cdd52e3c2f14be35804d Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Wed, 20 Mar 2019 12:13:30 +0100 Subject: [PATCH 4/7] upgrade wk-connect --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0baf5e96164..cf0b0d13f3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,7 +206,7 @@ services: # webKnossos-connect webknossos-connect: - image: scalableminds/webknossos-connect:master__188 + image: scalableminds/webknossos-connect:master__190 volumes: - "./conf/connect:/app/data" network_mode: host From 7647cd8994adac2b37f4b3940d1aa085c6466703 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Wed, 20 Mar 2019 12:45:05 +0100 Subject: [PATCH 5/7] update backend snapshots --- .../test/backend-snapshot-tests/misc.e2e.js.md | 7 +++++++ .../backend-snapshot-tests/misc.e2e.js.snap | Bin 952 -> 998 bytes 2 files changed, 7 insertions(+) diff --git a/frontend/javascripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/misc.e2e.js.md b/frontend/javascripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/misc.e2e.js.md index 3ed875ad62b..1182edc01fb 100644 --- a/frontend/javascripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/misc.e2e.js.md +++ b/frontend/javascripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/misc.e2e.js.md @@ -44,6 +44,13 @@ Generated by [AVA](https://ava.li). ## misc-datastores [ + { + isConnector: true, + isForeign: false, + isScratch: false, + name: 'connect', + url: 'http://localhost:8000', + }, { isConnector: false, isForeign: false, diff --git a/frontend/javascripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/misc.e2e.js.snap b/frontend/javascripts/test/snapshots/public/test-bundle/test/backend-snapshot-tests/misc.e2e.js.snap index f166f529e9697cd6ef2a1f7b52c3468fc6c277c2..12ad6b9d058e383d963b427160e2d8067cc10f92 100644 GIT binary patch literal 998 zcmVN8LvYjXZgyuj1Tvd!Kx1};4kS6P)!j8S<@QvKRW&;q zQ9;F%l9LAoi713XARd)faKVd$hme~HMuHa)URYMkf zzOEsJY;^nYt)t%_-TAb1>HVGMWA6nRg});fLbtw{{;T`tS8spx!MQ_spWVKJquFzL zgsvU?;g8D4+cz$4NPKqt>n$A3K0OiU%SZ9JfNck!3xMN*X`GwF*{YPJhw)%kEEW;o zgRq70UW`x2fiQYEeBv-iZVQInfZOpDdJ3?D`}uNf?rYT3*cuK?rBW2O36wVhmjT}a zZUHiu5oV8vO(L8{xCk8UF>J%j6yD3!3BfrqgpB5_qQRJ8>XeX_HF&&UO0K`+Nus@G z`g(?w79G6gy?8T&V|QdQ81HXt-CjC;I1nBQnnLNNvg0_r6rUpQ;fbFZO8Q^&DbQG2A)#D1(6E~nO#(g#8i54WsxC;`7#itkBR5Hpvs?4g^nN)gp zq;$DBt--UHKiv1oqT^QT^ZqPZ^c*_xJ5Jdr3$$*86qu@dnr+hb!A2g@0!g;B3 z!Fw&yQR)5O!_Y)?SbJ{?bz^O*bawQwC86qyaM(w0{2i`wOU)%d_s>qT`p#6}LR^%sRC0&zaXFO~8K}ax7r(0R=4VM!;?=FhAmu z6EDK^EZ`#GI^b780i19Z@DAXr`FTt_VZqpQiuiv2OK_5CqNq+U&OzXvVM;OH=<}94 zS_{6yLr?J2gB~YAAiASB@RH3NQOWO>5K@-ltB!z3N`N>gKzM*sMf6kNBVm-k>bnNL^A Urc2~YqA|?%FMUDo>39hM08j4sbN~PV literal 952 zcmV;p14sNpRzV zRZWN#M;NW@o*f=Y_3M~b+qc~lbl5{ZcxAOTs!fO%62(QC< zSASAQ!M6jr40r^18TR8|kj%XM9rZM};%1>xKzIb-g86B-*PdkEb|k(3l(uj|6q0w864lGU&HIs%zbx*}gsI&XgR- z8E`D`!nY1M2Y3wl6_AC^&j5}C+JH-dhk)M!ISZlB0pD7NL3qfZ4Ti9+Cn?d?kN&4J zPxn>R6>M|kD8$wcrsKrXB zNe_0-ut&@#g{%?Ieju6&9?zT~=v+7!ozB&P(@ASe8Y;@rXVCq#d3TKD%HhJ7XC!4boct@C0 zjMsX+*_PIV@8@An@RKKfLV`fF2QzYSSaaCIX_d4BJwKe-$e`wZro0yD5qwq< z8K1|-@R?EY&Zt(iDWo26Q+JUIrGy%XdSOKQ)QIvG{l3K{9juk6#ypL7d_W4Wn{bwI aVbz&W=g0|{$it!*$NCq+_O;4&2mk;}D9gYA From f62d5e42f59c7fe96e45a4f268256d0a88de6523 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 21 Mar 2019 16:18:16 +0100 Subject: [PATCH 6/7] insert connect-datastore based on insertLocalConnectDatastore conf --- app/controllers/InitialDataController.scala | 16 +++++++++------- app/utils/WkConf.scala | 1 + conf/application.conf | 1 + conf/connect/.gitignore | 1 + conf/connect/config.json | 7 +++++++ 5 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 conf/connect/.gitignore create mode 100644 conf/connect/config.json diff --git a/app/controllers/InitialDataController.scala b/app/controllers/InitialDataController.scala index d19164822b5..bc19efd576e 100644 --- a/app/controllers/InitialDataController.scala +++ b/app/controllers/InitialDataController.scala @@ -94,7 +94,7 @@ Samplecountry for { _ <- insertLocalDataStoreIfEnabled _ <- insertLocalTracingStoreIfEnabled - _ <- insertConnectDataStore + _ <- insertConnectDataStoreIfEnabled _ <- assertInitialDataEnabled _ <- assertNoOrganizationsPresent _ <- insertOrganization @@ -212,13 +212,15 @@ Samplecountry } } else Fox.successful(()) - def insertConnectDataStore: Fox[Any] = - dataStoreDAO.findOneByName("connect").futureBox.map { maybeStore => - if (maybeStore.isEmpty) { - logger.info("inserting connect datastore") - dataStoreDAO.insertOne(DataStore("connect", "http://localhost:8000", "secret-key", isConnector = true)) + def insertConnectDataStoreIfEnabled: Fox[Any] = + if (conf.Application.insertLocalConnectDatastore) { + dataStoreDAO.findOneByName("connect").futureBox.map { maybeStore => + if (maybeStore.isEmpty) { + logger.info("inserting connect datastore") + dataStoreDAO.insertOne(DataStore("connect", "http://localhost:8000", "secret-key", isConnector = true)) + } } - } + } else Fox.successful(()) def insertLocalTracingStoreIfEnabled: Fox[Any] = if (conf.Tracingstore.enabled) { diff --git a/app/utils/WkConf.scala b/app/utils/WkConf.scala index 92c05b6bd49..88abe2be36c 100644 --- a/app/utils/WkConf.scala +++ b/app/utils/WkConf.scala @@ -12,6 +12,7 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader { object Application { val insertInitialData = get[Boolean]("application.insertInitialData") + val insertLocalConnectDatastore = get[Boolean]("application.insertLocalConnectDatastore") val title = get[String]("application.title") object Authentication { diff --git a/conf/application.conf b/conf/application.conf index adfc452ccfa..b6897125780 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -64,6 +64,7 @@ braintracing { application { insertInitialData = true + insertLocalConnectDatastore = true authentication { enableDevAutoLogin = false enableDevAutoAdmin = false diff --git a/conf/connect/.gitignore b/conf/connect/.gitignore new file mode 100644 index 00000000000..0ca687ac137 --- /dev/null +++ b/conf/connect/.gitignore @@ -0,0 +1 @@ +datasets.json diff --git a/conf/connect/config.json b/conf/connect/config.json new file mode 100644 index 00000000000..0b758441137 --- /dev/null +++ b/conf/connect/config.json @@ -0,0 +1,7 @@ +{ + "server": {"host": "0.0.0.0", "port": 8000, "url": "http://localhost:8000"}, + "datastore": {"name": "connect", "key": "secret-key"}, + "webknossos": {"url": "http://localhost:9000"}, + "backends": {"neuroglancer": {}}, + "datasets_path": "data/datasets.json" +} From bdfd512bed873c40ae055cc7eeeb05bf0aeeb553 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 21 Mar 2019 16:20:40 +0100 Subject: [PATCH 7/7] update Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cee12807d6..8b1617becce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md). - The dataset settings within the tracing view allow to select between different loading strategies now ("best quality first" and "progressive quality"). Additionally, the rendering can use different magnifications as a fallback (instead of only one magnification). [#3801](https://github.com/scalableminds/webknossos/pull/3801) - The mapping selection dropbown is now sorted alphabetically. [#3864](https://github.com/scalableminds/webknossos/pull/3864) - The HTML template now includes SEO tags for demo instances and hides internal instances from search engines. +- [webknossos-connect](https://github.com/scalableminds/webknossos-connect) now starts with webknossos on local and development instances by default. ### Changed - Improved the flight mode performance for tracings with very large trees (>80.000 nodes). [#3880](https://github.com/scalableminds/webknossos/pull/3880)