From 78bdea47f1a660aaea9b67130fdf0a5c985d286b Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Thu, 14 Jan 2021 15:40:11 +0100 Subject: [PATCH 001/137] LinkButton supports icon --- .../desktop/preferences/pages/ExperimentalPage.js | 2 +- .../osparc/desktop/preferences/pages/SecurityPage.js | 2 +- .../client/source/class/osparc/ui/form/LinkButton.js | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js index 8d5ac5682a9..3c854486f2a 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js @@ -62,7 +62,7 @@ qx.Class.define("osparc.desktop.preferences.pages.ExperimentalPage", { )); box.add(label); - const linkBtn = new osparc.ui.form.LinkButton(this.tr("To qx-osparc-theme"), "https://github.com/ITISFoundation/qx-osparc-theme"); + const linkBtn = new osparc.ui.form.LinkButton(this.tr("To qx-osparc-theme"), null, "https://github.com/ITISFoundation/qx-osparc-theme"); box.add(linkBtn); const select = new qx.ui.form.SelectBox("Theme"); diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js index f5f1bb9b879..029102c691f 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js @@ -220,7 +220,7 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { )); box.add(label); - const linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), "https://app.blackfynn.io"); + const linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), null, "https://app.blackfynn.io"); box.add(linkBtn); const tokensList = this.__tokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); diff --git a/services/web/client/source/class/osparc/ui/form/LinkButton.js b/services/web/client/source/class/osparc/ui/form/LinkButton.js index 09db9e37b89..03e13475531 100644 --- a/services/web/client/source/class/osparc/ui/form/LinkButton.js +++ b/services/web/client/source/class/osparc/ui/form/LinkButton.js @@ -29,14 +29,13 @@ */ qx.Class.define("osparc.ui.form.LinkButton", { - extend: osparc.ui.form.FetchButton, + extend: qx.ui.form.Button, /** * @param label {String} Label to use * @param url {String} Url to point to - * @param height {Integer?12} Height of the link icon */ - construct: function(label, url, height = 12) { + construct: function(label, icon, url) { this.base(arguments, label); this.set({ @@ -45,7 +44,11 @@ qx.Class.define("osparc.ui.form.LinkButton", { }); if (url) { - this.setIcon("@FontAwesome5Solid/external-link-alt/" + height); + if (icon) { + this.setIcon(icon); + } else { + this.setIcon("@FontAwesome5Solid/external-link-alt/12"); + } this.addListener("execute", () => { window.open(url); }, this); From 87c32cf8ce632ea4edaf1516d26c21eb9332454c Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Thu, 14 Jan 2021 16:04:24 +0100 Subject: [PATCH 002/137] Revert "LinkButton supports icon" This reverts commit 78bdea47f1a660aaea9b67130fdf0a5c985d286b. --- .../desktop/preferences/pages/ExperimentalPage.js | 2 +- .../osparc/desktop/preferences/pages/SecurityPage.js | 2 +- .../client/source/class/osparc/ui/form/LinkButton.js | 11 ++++------- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js index 3c854486f2a..8d5ac5682a9 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/ExperimentalPage.js @@ -62,7 +62,7 @@ qx.Class.define("osparc.desktop.preferences.pages.ExperimentalPage", { )); box.add(label); - const linkBtn = new osparc.ui.form.LinkButton(this.tr("To qx-osparc-theme"), null, "https://github.com/ITISFoundation/qx-osparc-theme"); + const linkBtn = new osparc.ui.form.LinkButton(this.tr("To qx-osparc-theme"), "https://github.com/ITISFoundation/qx-osparc-theme"); box.add(linkBtn); const select = new qx.ui.form.SelectBox("Theme"); diff --git a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js index 029102c691f..f5f1bb9b879 100644 --- a/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js +++ b/services/web/client/source/class/osparc/desktop/preferences/pages/SecurityPage.js @@ -220,7 +220,7 @@ qx.Class.define("osparc.desktop.preferences.pages.SecurityPage", { )); box.add(label); - const linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), null, "https://app.blackfynn.io"); + const linkBtn = new osparc.ui.form.LinkButton(this.tr("To DAT-Core"), "https://app.blackfynn.io"); box.add(linkBtn); const tokensList = this.__tokensList = new qx.ui.container.Composite(new qx.ui.layout.VBox(8)); diff --git a/services/web/client/source/class/osparc/ui/form/LinkButton.js b/services/web/client/source/class/osparc/ui/form/LinkButton.js index 03e13475531..09db9e37b89 100644 --- a/services/web/client/source/class/osparc/ui/form/LinkButton.js +++ b/services/web/client/source/class/osparc/ui/form/LinkButton.js @@ -29,13 +29,14 @@ */ qx.Class.define("osparc.ui.form.LinkButton", { - extend: qx.ui.form.Button, + extend: osparc.ui.form.FetchButton, /** * @param label {String} Label to use * @param url {String} Url to point to + * @param height {Integer?12} Height of the link icon */ - construct: function(label, icon, url) { + construct: function(label, url, height = 12) { this.base(arguments, label); this.set({ @@ -44,11 +45,7 @@ qx.Class.define("osparc.ui.form.LinkButton", { }); if (url) { - if (icon) { - this.setIcon(icon); - } else { - this.setIcon("@FontAwesome5Solid/external-link-alt/12"); - } + this.setIcon("@FontAwesome5Solid/external-link-alt/" + height); this.addListener("execute", () => { window.open(url); }, this); From 4ea3d45a9d8328c2974dfbd6c60f5e16dd45be17 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:22:15 +0200 Subject: [PATCH 003/137] Iterations -> Snapshots and table improved --- .../Iterations.js => snapshots/Snapshots.js} | 92 ++++++++++--------- 1 file changed, 48 insertions(+), 44 deletions(-) rename services/web/client/source/class/osparc/component/{sweeper/Iterations.js => snapshots/Snapshots.js} (53%) diff --git a/services/web/client/source/class/osparc/component/sweeper/Iterations.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js similarity index 53% rename from services/web/client/source/class/osparc/component/sweeper/Iterations.js rename to services/web/client/source/class/osparc/component/snapshots/Snapshots.js index 21678e8afc8..54716f7edd7 100644 --- a/services/web/client/source/class/osparc/component/sweeper/Iterations.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -15,62 +15,66 @@ ************************************************************************ */ -qx.Class.define("osparc.component.sweeper.Iterations", { +qx.Class.define("osparc.component.snapshots.Snapshots", { extend: osparc.ui.table.Table, construct: function(primaryStudy) { this.__primaryStudy = primaryStudy; - this.__initModel(); + this.__cols = {}; + const model = this.__initModel(); - this.base(arguments, this.__model, { - tableColumnModel: obj => new qx.ui.table.columnmodel.Resize(obj), - statusBarVisible: false, - initiallyHiddenColumns: [0] + this.base(arguments, model, { + initiallyHiddenColumns: [this.self().T_POS.ID.col], + statusBarVisible: false }); - this.__updateTable(); - }, + this.setColumnWidth(this.self().T_POS.NAME.col, 220); + this.setColumnWidth(this.self().T_POS.DATE.col, 130); - members: { // eslint-disable-line qx-rules/no-refs-in-members - __primaryStudy: null, - __model: null, + this.__populateTable(); + }, - __cols: { - "id": { + statics: { + T_POS: { + ID: { col: 0, label: qx.locale.Manager.tr("StudyId") }, - "name": { + NAME: { col: 1, - label: qx.locale.Manager.tr("Iteration") + label: qx.locale.Manager.tr("Snapshot Name") + }, + DATE: { + col: 2, + label: qx.locale.Manager.tr("Created At") } - }, + } + }, + + members: { + __primaryStudy: null, + __cols: null, getRowData: function(rowIdx) { - return this.__model.getRowDataAsMap(rowIdx); + return this.getTableModel().getRowDataAsMap(rowIdx); }, - __cleanupCols: function() { - Object.keys(this.__cols).forEach(key => { - if (!["id", "name"].includes(key)) { - delete this.__cols[key]; - } + __initModel: function() { + const model = new qx.ui.table.model.Simple(); + + Object.keys(this.self().T_POS).forEach(colKey => { + this.__cols[colKey] = this.self().T_POS[colKey]; }); - }, - __initModel: function() { - const model = this.__model = new qx.ui.table.model.Simple(); - - // add variables in columns - const parameters = this.__primaryStudy.getSweeper().getParameters(); - this.__cleanupCols(); - const nextCol = this.__cols["name"].col + 1; - for (let i=0; i { - for (const [key, value] of Object.entries(paramValue)) { - row[this.__cols[key].col] = value; - } - }); + row[this.self().T_POS.ID.col] = secondaryStudy.uuid; + row[this.self().T_POS.NAME.col] = secondaryStudy.name; + const date = new Date(secondaryStudy.creationDate); + row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); + // OM: hack for demo for + row[Object.keys(this.self().T_POS).length] = i+1; rows.push(row); } this.getTableModel().setData(rows, false); From 37568f051a2817c441b2a90d9169a976278ac5a2 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:22:39 +0200 Subject: [PATCH 004/137] snapshots resources --- .../source/class/osparc/data/Resources.js | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/services/web/client/source/class/osparc/data/Resources.js b/services/web/client/source/class/osparc/data/Resources.js index 34b6fdb1ab2..42ffc1a79e1 100644 --- a/services/web/client/source/class/osparc/data/Resources.js +++ b/services/web/client/source/class/osparc/data/Resources.js @@ -150,6 +150,37 @@ qx.Class.define("osparc.data.Resources", { } } }, + /* + * SNAPSHOTS + */ + "snapshots": { + useCache: true, + idField: "uuid", + endpoints: { + get: { + method: "GET", + url: statics.API + "/projects/{studyId}/snapshots" + }, + getPage: { + method: "GET", + url: statics.API + "/projects/{studyId}/snapshots?offset={offset}&limit={limit}" + }, + getOne: { + useCache: false, + method: "GET", + url: statics.API + "/projects/{studyId}/snapshots/{snapshotId}" + }, + getParameters: { + useCache: false, + method: "GET", + url: statics.API + "/projects/{studyId}/snapshots/{snapshotId}/parameters" + }, + takeSnapshot: { + method: "POST", + url: statics.API + "/projects/{studyId}/snapshots" + } + } + }, /* * TEMPLATES (actually studies flagged as templates) */ From c1c93d01d687d5e7925ee2d8a07d2b3d328f80a9 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:23:04 +0200 Subject: [PATCH 005/137] Snapshots view --- .../component/snapshots/SnapshotsView.js | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js diff --git a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js new file mode 100644 index 00000000000..6d5bead62e8 --- /dev/null +++ b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js @@ -0,0 +1,168 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2021 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.component.snapshots.SnapshotsView", { + extend: qx.ui.core.Widget, + + construct: function(study) { + this.base(arguments); + + this._setLayout(new qx.ui.layout.VBox(10)); + + if (study.isSnapshot()) { + const primaryStudyId = study.getSweeper().getPrimaryStudyId(); + const openPrimaryStudyParamBtn = new qx.ui.form.Button(this.tr("Open Main Study")).set({ + allowGrowX: false + }); + openPrimaryStudyParamBtn.addListener("execute", () => { + this.fireDataEvent("openPrimaryStudy", primaryStudyId); + }); + this._add(openPrimaryStudyParamBtn); + } else { + this.__primaryStudy = study; + const snapshotsSection = this.__buildSnapshotsSection(); + this._add(snapshotsSection, { + flex: 1 + }); + } + }, + + events: { + "openPrimaryStudy": "qx.event.type.Data", + "openSnapshot": "qx.event.type.Data" + }, + + members: { + __snapshotsSection: null, + __snapshotsTable: null, + __selectedSnapshot: null, + __openSnapshotBtn: null, + + __buildSnapshotsSection: function() { + const snapshotsSection = this.__snapshotsSection = new qx.ui.groupbox.GroupBox(this.tr("Snapshots")).set({ + layout: new qx.ui.layout.VBox(5) + }); + + const snapshotBtns = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)); + const deleteSnapshotsBtn = this.__deleteSnapshotsBtn(); + snapshotBtns.add(deleteSnapshotsBtn); + const recreateSnapshotsBtn = this.__recreateSnapshotsBtn(); + snapshotBtns.add(recreateSnapshotsBtn); + snapshotsSection.addAt(snapshotBtns, 0); + + this.__rebuildSnapshotsTable(); + + const openSnapshotBtn = this.__openSnapshotBtn = this.__createOpenSnapshotBtn(); + openSnapshotBtn.setEnabled(false); + snapshotsSection.addAt(openSnapshotBtn, 2); + openSnapshotBtn.addListener("execute", () => { + if (this.__selectedSnapshot) { + this.fireDataEvent("openSnapshot", this.__selectedSnapshot); + } + }); + + return snapshotsSection; + }, + + __rebuildSnapshotsTable: function() { + if (this.__snapshotsTable) { + this.__snapshotsSection.remove(this.__snapshotsTable); + } + + const snapshotsTable = this.__snapshotsTable = new osparc.component.snapshots.Snapshots(this.__primaryStudy); + snapshotsTable.addListener("cellTap", e => { + if (this.__openSnapshotBtn) { + this.__openSnapshotBtn.setEnabled(true); + } + const selectedRow = e.getRow(); + this.__selectedSnapshot = snapshotsTable.getRowData(selectedRow)["StudyId"]; + }); + + this.__snapshotsSection.addAt(snapshotsTable, 1, { + flex: 1 + }); + + return snapshotsTable; + }, + + __deleteSnapshotsBtn: function() { + const deleteSnapshotsBtn = new osparc.ui.form.FetchButton(this.tr("Delete Snapshots")).set({ + alignX: "left", + allowGrowX: false + }); + deleteSnapshotsBtn.addListener("execute", () => { + deleteSnapshotsBtn.setFetching(true); + this.__deleteSnapshots(deleteSnapshotsBtn) + .then(() => { + this.__rebuildSnapshotsTable(); + }) + .finally(() => { + deleteSnapshotsBtn.setFetching(false); + }); + }, this); + return deleteSnapshotsBtn; + }, + + __recreateSnapshotsBtn: function() { + const recreateSnapshotsBtn = new osparc.ui.form.FetchButton(this.tr("Recreate Snapshots")).set({ + alignX: "right", + allowGrowX: false + }); + recreateSnapshotsBtn.addListener("execute", () => { + recreateSnapshotsBtn.setFetching(true); + this.__recreateSnapshots(recreateSnapshotsBtn) + .then(() => { + this.__rebuildSnapshotsTable(); + }) + .finally(() => { + recreateSnapshotsBtn.setFetching(false); + }); + }, this); + return recreateSnapshotsBtn; + }, + + __deleteSnapshots: function() { + return new Promise((resolve, reject) => { + this.__primaryStudy.getSweeper().removeSecondaryStudies() + .then(() => { + const msg = this.tr("Snapshots Deleted"); + osparc.component.message.FlashMessenger.getInstance().logAs(msg); + resolve(); + }); + }); + }, + + __recreateSnapshots: function() { + return new Promise((resolve, reject) => { + const primaryStudyData = this.__primaryStudy.serialize(); + this.__primaryStudy.getSweeper().recreateSnapshots(primaryStudyData, this.__primaryStudy.getParameters()) + .then(secondaryStudyIds => { + const msg = secondaryStudyIds.length + this.tr(" Snapshots Created"); + osparc.component.message.FlashMessenger.getInstance().logAs(msg); + resolve(); + }); + }); + }, + + __createOpenSnapshotBtn: function() { + const openSnapshotBtn = new qx.ui.form.Button(this.tr("Open Snapshot")).set({ + allowGrowX: false + }); + return openSnapshotBtn; + } + } +}); From 27b973de4a00d292de037494782f2b777dcadc5d Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:23:10 +0200 Subject: [PATCH 006/137] TakeSnapshotView --- .../component/snapshots/TakeSnapshotView.js | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js diff --git a/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js b/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js new file mode 100644 index 00000000000..c4176439535 --- /dev/null +++ b/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js @@ -0,0 +1,116 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2021 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + +qx.Class.define("osparc.component.snapshots.TakeSnapshotView", { + extend: qx.ui.core.Widget, + + construct: function(study) { + this.base(arguments); + + this._setLayout(new qx.ui.layout.VBox(10)); + + this.set({ + study + }); + + this.__buildForm(); + }, + + events: { + "takeSnapshot": "qx.event.type.Event", + "cancel": "qx.event.type.Event" + }, + + properties: { + study: { + check: "osparc.data.model.Study", + init: null, + nullable: false + } + }, + + members: { + __form: null, + + _createChildControlImpl: function(id) { + let control; + switch (id) { + case "label": + control = new qx.ui.form.TextField(); + break; + case "save-data": + control = new qx.ui.form.CheckBox().set({ + value: false, + enabled: false + }); + break; + case "cancel-button": { + control = new qx.ui.form.Button(this.tr("Cancel")).set({ + allowGrowX: false + }); + const commandEsc = new qx.ui.command.Command("Enter"); + control.setCommand(commandEsc); + control.addListener("execute", () => this.fireEvent("cancel")); + break; + } + case "ok-button": { + control = new qx.ui.form.Button(this.tr("OK")).set({ + allowGrowX: false + }); + const commandEnter = new qx.ui.command.Command("Enter"); + control.setCommand(commandEnter); + control.addListener("execute", () => { + // releaseCapture to make sure all changes are applied + this.__renderer.releaseCapture(); + this.fireEvent("takeSnapshot"); + }); + break; + } + } + return control || this.base(arguments, id); + }, + + __buildForm: function() { + const form = this.__form = new qx.ui.form.Form(); + const renderer = this.__renderer = new qx.ui.form.renderer.Single(form); + this._add(renderer); + + const study = this.getStudy(); + + const label = this.getChildControl("label"); + form.add(label, "Label", null, "label"); + label.setValue(study.getName()); + + const saveWData = this.getChildControl("save-data"); + form.add(saveWData, "Save with Data", null, "save-data"); + + // buttons + const cancelButton = this.getChildControl("cancel-button"); + form.addButton(cancelButton); + const okButton = this.getChildControl("ok-button"); + form.addButton(okButton); + }, + + getLabel: function() { + return this.__form.getItem("label").getValue(); + }, + + getSaveData: function() { + return this.__form.getItem("save-data").getValue(); + } + } +}); From c857d09d1e5fc7204ce119bacc1b8339882acac3 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:23:49 +0200 Subject: [PATCH 007/137] Sweeper only shows Parameters --- .../class/osparc/component/sweeper/Sweeper.js | 157 ++---------------- 1 file changed, 14 insertions(+), 143 deletions(-) diff --git a/services/web/client/source/class/osparc/component/sweeper/Sweeper.js b/services/web/client/source/class/osparc/component/sweeper/Sweeper.js index c882ecbbe9a..6df7d952e4d 100644 --- a/services/web/client/source/class/osparc/component/sweeper/Sweeper.js +++ b/services/web/client/source/class/osparc/component/sweeper/Sweeper.js @@ -23,47 +23,31 @@ qx.Class.define("osparc.component.sweeper.Sweeper", { this._setLayout(new qx.ui.layout.VBox(10)); - if (study.getSweeper().getPrimaryStudyId()) { - this.__buildSecondaryLayout(study); + if (study.isSnapshot()) { + const primaryStudyId = study.getSweeper().getPrimaryStudyId(); + const openPrimaryStudyParamBtn = new qx.ui.form.Button(this.tr("Open Main Study")).set({ + allowGrowX: false + }); + openPrimaryStudyParamBtn.addListener("execute", () => { + this.fireDataEvent("openPrimaryStudy", primaryStudyId); + }); + this._add(openPrimaryStudyParamBtn); } else { this.__primaryStudy = study; - this.__buildPrimaryLayout(); + const parametersSection = this.__buildParametersSection(); + this._add(parametersSection, { + flex: 1 + }); } }, events: { - "iterationSelected": "qx.event.type.Data" + "openPrimaryStudy": "qx.event.type.Data" }, members: { __primaryStudy: null, __parametersTable: null, - __iterationsSection: null, - __iterationsTable: null, - __selectedIteration: null, - - __buildSecondaryLayout: function(secondaryStudy) { - const newParamBtn = new qx.ui.form.Button(this.tr("Open primary study")).set({ - allowGrowX: false - }); - newParamBtn.addListener("execute", () => { - const primaryStudyId = secondaryStudy.getSweeper().getPrimaryStudyId(); - this.fireDataEvent("iterationSelected", primaryStudyId); - }); - this._add(newParamBtn); - }, - - __buildPrimaryLayout: function() { - const parametersSection = this.__buildParametersSection(); - this._add(parametersSection, { - flex: 2 - }); - - const iterationsSection = this.__buildIterationsSection(); - this._add(iterationsSection, { - flex: 3 - }); - }, __buildParametersSection: function() { const parametersSection = new qx.ui.groupbox.GroupBox(this.tr("Parameters")).set({ @@ -79,53 +63,6 @@ qx.Class.define("osparc.component.sweeper.Sweeper", { return parametersSection; }, - __buildIterationsSection: function() { - const iterationsSection = this.__iterationsSection = new qx.ui.groupbox.GroupBox(this.tr("Iterations")).set({ - layout: new qx.ui.layout.VBox(5) - }); - - const iterationBtns = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)); - const deleteIterationsBtn = this.__deleteIterationsBtn(); - iterationBtns.add(deleteIterationsBtn); - const recreateIterationsBtn = this.__recreateIterationsBtn(); - iterationBtns.add(recreateIterationsBtn); - iterationsSection.addAt(iterationBtns, 0); - - this.__rebuildIterationsTable(); - - const openIterationsBtn = this.__openIterationsBtn = this.__createOpenIterationsBtn(); - openIterationsBtn.setEnabled(false); - iterationsSection.addAt(openIterationsBtn, 2); - openIterationsBtn.addListener("execute", () => { - if (this.__selectedIteration) { - this.fireDataEvent("iterationSelected", this.__selectedIteration); - } - }); - - return iterationsSection; - }, - - __rebuildIterationsTable: function() { - if (this.__iterationsTable) { - this.__iterationsSection.remove(this.__iterationsTable); - } - - const iterationsTable = this.__iterationsTable = new osparc.component.sweeper.Iterations(this.__primaryStudy); - iterationsTable.addListener("cellTap", e => { - if (this.__openIterationsBtn) { - this.__openIterationsBtn.setEnabled(true); - } - const selectedRow = e.getRow(); - this.__selectedIteration = iterationsTable.getRowData(selectedRow)["StudyId"]; - }); - - this.__iterationsSection.addAt(iterationsTable, 1, { - flex: 1 - }); - - return iterationsTable; - }, - __createNewParamBtn: function() { const label = this.tr("Create new Parameter"); const newParamBtn = new qx.ui.form.Button(label).set({ @@ -150,72 +87,6 @@ qx.Class.define("osparc.component.sweeper.Sweeper", { newParamName.open(); }, this); return newParamBtn; - }, - - __deleteIterationsBtn: function() { - const deleteIterationsBtn = new osparc.ui.form.FetchButton(this.tr("Delete Iterations")).set({ - alignX: "left", - allowGrowX: false - }); - deleteIterationsBtn.addListener("execute", () => { - deleteIterationsBtn.setFetching(true); - this.__deleteIterations(deleteIterationsBtn) - .then(() => { - this.__rebuildIterationsTable(); - }) - .finally(() => { - deleteIterationsBtn.setFetching(false); - }); - }, this); - return deleteIterationsBtn; - }, - - __recreateIterationsBtn: function() { - const recreateIterationsBtn = new osparc.ui.form.FetchButton(this.tr("Recreate Iterations")).set({ - alignX: "right", - allowGrowX: false - }); - recreateIterationsBtn.addListener("execute", () => { - recreateIterationsBtn.setFetching(true); - this.__recreateIterations(recreateIterationsBtn) - .then(() => { - this.__rebuildIterationsTable(); - }) - .finally(() => { - recreateIterationsBtn.setFetching(false); - }); - }, this); - return recreateIterationsBtn; - }, - - __deleteIterations: function() { - return new Promise((resolve, reject) => { - this.__primaryStudy.getSweeper().removeSecondaryStudies() - .then(() => { - const msg = this.tr("Iterations deleted"); - osparc.component.message.FlashMessenger.getInstance().logAs(msg); - resolve(); - }); - }); - }, - - __recreateIterations: function() { - return new Promise((resolve, reject) => { - const primaryStudyData = this.__primaryStudy.serialize(); - this.__primaryStudy.getSweeper().recreateIterations(primaryStudyData) - .then(secondaryStudyIds => { - const msg = secondaryStudyIds.length + this.tr(" Iterations created"); - osparc.component.message.FlashMessenger.getInstance().logAs(msg); - resolve(); - }); - }); - }, - - __createOpenIterationsBtn: function() { - const openIterationBtn = new qx.ui.form.Button(this.tr("Open Iteration")).set({ - allowGrowX: false - }); - return openIterationBtn; } } }); From a27d5903585fe27b357c6b25d451804492b8fcc0 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:24:06 +0200 Subject: [PATCH 008/137] create snapshots only for testers --- services/web/client/source/class/osparc/data/Permissions.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/web/client/source/class/osparc/data/Permissions.js b/services/web/client/source/class/osparc/data/Permissions.js index 5d1534c0ed8..26a750f6161 100644 --- a/services/web/client/source/class/osparc/data/Permissions.js +++ b/services/web/client/source/class/osparc/data/Permissions.js @@ -164,6 +164,7 @@ qx.Class.define("osparc.data.Permissions", { "services.all.read", "user.role.update", "study.service.update", + "study.snapshot.create", "study.nodestree.uuid.read", "study.filestree.uuid.read", "study.logger.debug.read", From 6157462b6e6c3ac254d3e55b9b16caf7a5c0f566 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:24:19 +0200 Subject: [PATCH 009/137] minor --- .../web/client/source/class/osparc/data/StudyParametrizer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/client/source/class/osparc/data/StudyParametrizer.js b/services/web/client/source/class/osparc/data/StudyParametrizer.js index c2ea2627962..99d0db73fcd 100644 --- a/services/web/client/source/class/osparc/data/StudyParametrizer.js +++ b/services/web/client/source/class/osparc/data/StudyParametrizer.js @@ -82,7 +82,7 @@ qx.Class.define("osparc.data.StudyParametrizer", { return activeParams; }, - recreateIterations: function(primaryStudyData, parameters, combinations) { + recreateSnaphots: function(primaryStudyData, parameters, combinations) { return new Promise((resolve, reject) => { const store = osparc.store.Store.getInstance(); store.getGroupsMe() From 6dd2e3a95f25e3706af11d26e7fcad02a07e9861 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:24:33 +0200 Subject: [PATCH 010/137] snapshot helper functions --- .../source/class/osparc/data/model/Study.js | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 2c0d8fab08b..89897dc662d 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -244,12 +244,34 @@ qx.Class.define("osparc.data.model.Study", { this.getWorkbench().initWorkbench(); }, + isSnapshot: function() { + if (this.getSweeper()) { + const primaryStudyId = this.getSweeper().getPrimaryStudyId(); + return primaryStudyId !== null; + } + return false; + }, + + hasSnapshots: function() { + return new Promise((resolve, reject) => { + const params = { + url: { + "studyId": this.getUuid() + } + }; + osparc.data.Resources.get("snapshots", params) + .then(snapshots => { + resolve(snapshots.length); + }); + }); + }, + __applyAccessRights: function(value) { const myGid = osparc.auth.Data.getInstance().getGroupId(); const orgIDs = osparc.auth.Data.getInstance().getOrgIds(); orgIDs.push(myGid); - if (myGid) { + if (myGid && !this.isSnapshot()) { const canIWrite = osparc.component.permissions.Study.canGroupsWrite(value, orgIDs); this.setReadOnly(!canIWrite); } else { From 2c7037ba0a881a019dd3aafc072cabc46397ce3e Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:24:41 +0200 Subject: [PATCH 011/137] minor --- services/web/client/source/class/osparc/data/model/Sweeper.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/client/source/class/osparc/data/model/Sweeper.js b/services/web/client/source/class/osparc/data/model/Sweeper.js index 0eca2316b89..36e2e7b99b5 100644 --- a/services/web/client/source/class/osparc/data/model/Sweeper.js +++ b/services/web/client/source/class/osparc/data/model/Sweeper.js @@ -196,7 +196,7 @@ qx.Class.define("osparc.data.model.Sweeper", { }, /* /PRIMARY STUDY */ - recreateIterations: function(primaryStudyData) { + recreateSnaphots: function(primaryStudyData) { return new Promise((resolve, reject) => { // delete previous iterations this.removeSecondaryStudies() @@ -212,7 +212,7 @@ qx.Class.define("osparc.data.model.Sweeper", { const combinations = osparc.data.StudyParametrizer.calculateCombinations(steps); this.__setCombinations(combinations); - osparc.data.StudyParametrizer.recreateIterations(primaryStudyData, usedParams, combinations) + osparc.data.StudyParametrizer.recreateSnaphots(primaryStudyData, usedParams, combinations) .then(secondaryStudiesData => { secondaryStudiesData.forEach(secondaryStudyData => { this.__secondaryStudyIds.push(secondaryStudyData.uuid); From 3204fe33f2348987e468ae7cd691ddc5d494efe9 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:24:55 +0200 Subject: [PATCH 012/137] StartPipelineView (with cache) --- .../class/osparc/desktop/StartPipelineView.js | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 services/web/client/source/class/osparc/desktop/StartPipelineView.js diff --git a/services/web/client/source/class/osparc/desktop/StartPipelineView.js b/services/web/client/source/class/osparc/desktop/StartPipelineView.js new file mode 100644 index 00000000000..c155851bbdb --- /dev/null +++ b/services/web/client/source/class/osparc/desktop/StartPipelineView.js @@ -0,0 +1,93 @@ +/* ************************************************************************ + + osparc - the simcore frontend + + https://osparc.io + + Copyright: + 2021 IT'IS Foundation, https://itis.swiss + + License: + MIT: https://opensource.org/licenses/MIT + + Authors: + * Odei Maiz (odeimaiz) + +************************************************************************ */ + + +qx.Class.define("osparc.desktop.StartPipelineView", { + extend: qx.ui.core.Widget, + + construct: function(partialPipeline = [], forceRestart = false) { + this.base(arguments); + + this._setLayout(new qx.ui.layout.VBox()); + + this.__buildOptions(partialPipeline, forceRestart); + }, + + events: { + "startPipeline": "qx.event.type.Data", + "cancel": "qx.event.type.Event" + }, + + members: { + __buildOptions: function(partialPipeline, forceRestart) { + const partialBox = new qx.ui.groupbox.GroupBox(this.tr("Partial Running")); + partialBox.set({ + layout: new qx.ui.layout.VBox(), + enabled: false + }); + const runPipeline = new qx.ui.form.RadioButton(this.tr("Run Entire pipeline")); + const runPartialPipeline = new qx.ui.form.RadioButton(this.tr("Run Partial pipeline")); + const rbManager = new qx.ui.form.RadioGroup(runPipeline, runPartialPipeline).set({ + allowEmptySelection: false + }); + partialBox.add(runPipeline); + partialBox.add(runPartialPipeline); + rbManager.add(runPipeline); + rbManager.add(runPartialPipeline); + rbManager.setSelection(partialPipeline.length ? [runPartialPipeline] : [runPipeline]); + this._add(partialBox); + + const reRunBox = new qx.ui.groupbox.GroupBox(this.tr("Re-Run")); + reRunBox.set({ + layout: new qx.ui.layout.VBox(), + enabled: false + }); + const reRunCB = new qx.ui.form.CheckBox(this.tr("Re-run")).set({ + value: forceRestart + }); + reRunBox.add(reRunCB); + this._add(reRunBox); + + const cacheBox = new qx.ui.groupbox.GroupBox(this.tr("Caching")); + cacheBox.setLayout(new qx.ui.layout.VBox()); + const useCacheCB = new qx.ui.form.CheckBox(this.tr("Use cache")).set({ + value: true + }); + cacheBox.add(useCacheCB); + this._add(cacheBox); + + const btnsLayout = new qx.ui.container.Composite(new qx.ui.layout.HBox(5).set({ + alignX: "right" + })); + const cancelBtn = new qx.ui.form.Button(this.tr("Cancel")).set({ + allowGrowX: false + }); + cancelBtn.addListener("execute", () => this.fireEvent("cancel")); + btnsLayout.add(cancelBtn); + const startBtn = new qx.ui.form.Button(this.tr("Start")).set({ + allowGrowX: false + }); + btnsLayout.add(startBtn); + startBtn.addListener("execute", () => { + this.fireDataEvent("startPipeline", { + "useCache": useCacheCB.getValue() + }); + }); + this._add(btnsLayout); + } + } +}); From 9b73b59c1c2bf47b8faddc131efb7f2dc7574113 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:25:27 +0200 Subject: [PATCH 013/137] call StartPipelineView with cache --- .../source/class/osparc/desktop/StudyEditor.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/services/web/client/source/class/osparc/desktop/StudyEditor.js b/services/web/client/source/class/osparc/desktop/StudyEditor.js index b611f01c840..6b309e28be8 100644 --- a/services/web/client/source/class/osparc/desktop/StudyEditor.js +++ b/services/web/client/source/class/osparc/desktop/StudyEditor.js @@ -235,6 +235,20 @@ qx.Class.define("osparc.desktop.StudyEditor", { }, __requestStartPipeline: function(studyId, partialPipeline = [], forceRestart = false) { + const startPipelineView = new osparc.desktop.StartPipelineView(partialPipeline, forceRestart); + const win = osparc.ui.window.Window.popUpInWindow(startPipelineView, "Start Pipeline", 250, 290); + startPipelineView.addListener("startPipeline", e => { + const data = e.getData(); + const useCache = data["useCache"]; + this.__reallyRequestStartPipeline(studyId, partialPipeline, forceRestart, useCache); + win.close(); + }, this); + startPipelineView.addListener("cancel", () => { + win.close(); + }, this); + }, + + __reallyRequestStartPipeline: function(studyId, partialPipeline = [], forceRestart = false, useCache = false) { const url = "/computation/pipeline/" + encodeURIComponent(studyId) + ":start"; const req = new osparc.io.request.ApiRequest(url, "POST"); const startStopButtonsWB = this.__workbenchView.getStartStopButtons(); @@ -268,7 +282,8 @@ qx.Class.define("osparc.desktop.StudyEditor", { req.setRequestData({ "subgraph": partialPipeline, - "force_restart": forceRestart + "force_restart": forceRestart, + "use_cache": useCache }); req.send(); if (partialPipeline.length) { From e724dcf935bb1022924fd64bed56e2ef2826917a Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:25:39 +0200 Subject: [PATCH 014/137] snapshot buttons --- .../class/osparc/desktop/WorkbenchToolbar.js | 106 +++++++++++++++--- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js index 02e62c29a89..13e76605f10 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js @@ -25,7 +25,11 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { }, events: { - "showSweeper": "qx.event.type.Event" + "takeSnapshot": "qx.event.type.Event", + "convertToStudy": "qx.event.type.Event", + "showParameters": "qx.event.type.Event", + "showSnapshots": "qx.event.type.Event", + "openPrimaryStudy": "qx.event.type.Data" }, members: { @@ -44,15 +48,52 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { }); break; } - case "sweeper-btn": { - control = new qx.ui.form.Button(this.tr("Sweeper"), "@FontAwesome5Solid/paw/14").set({ - toolTipText: this.tr("Sweeper"), - icon: "@FontAwesome5Solid/paw/14", + case "take-snapshot-btn": { + control = new osparc.ui.form.FetchButton(this.tr("Take Snapshot")).set({ + icon: "@FontAwesome5Solid/camera/14", ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, allowGrowX: false }); - control.addListener("execute", e => { - this.fireDataEvent("showSweeper"); + control.addListener("execute", () => { + this.fireDataEvent("takeSnapshot"); + }, this); + this._add(control); + break; + } + case "convert-to-study-btn": { + control = new osparc.ui.form.FetchButton(this.tr("Convert To Study")).set({ + ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, + allowGrowX: false + }); + control.addListener("execute", () => { + this.fireDataEvent("convertToStudy"); + }, this); + this._add(control); + break; + } + case "snapshots-btn": { + control = new qx.ui.form.Button(this.tr("Snapshots")).set({ + icon: "@FontAwesome5Solid/copy/14", + ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, + allowGrowX: false + }); + control.addListener("execute", () => { + this.fireDataEvent("showSnapshots"); + }, this); + this._add(control); + break; + } + case "primary-study-btn": { + control = new qx.ui.form.Button(this.tr("Open Main Study")).set({ + icon: "@FontAwesome5Solid/external-link-alt/14", + ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, + allowGrowX: false + }); + control.addListener("execute", () => { + const primaryStudyId = this.getStudy().getSweeper().getPrimaryStudyId(); + if (primaryStudyId) { + this.fireDataEvent("openPrimaryStudy", primaryStudyId); + } }, this); this._add(control); break; @@ -67,16 +108,20 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { this._add(new qx.ui.core.Spacer(20)); - const sweeperBtn = this.getChildControl("sweeper-btn"); - sweeperBtn.exclude(); - osparc.data.model.Sweeper.isSweeperEnabled() - .then(isSweeperEnabled => { - if (isSweeperEnabled) { - sweeperBtn.show(); - } - }); + const takeSnapshotBtn = this.getChildControl("take-snapshot-btn"); + takeSnapshotBtn.exclude(); - this._startStopBtns = this.getChildControl("start-stop-btns"); + const convertToStudy = this.getChildControl("convert-to-study-btn"); + convertToStudy.exclude(); + + const primaryBtn = this.getChildControl("primary-study-btn"); + primaryBtn.exclude(); + + const snapshotsBtn = this.getChildControl("snapshots-btn"); + snapshotsBtn.exclude(); + + const startStopBtns = this._startStopBtns = this.getChildControl("start-stop-btns"); + startStopBtns.exclude(); }, // overriden @@ -85,6 +130,35 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { if (study) { const nodeIds = study.getWorkbench().getPathIds(study.getUi().getCurrentNodeId()); this._navNodes.populateButtons(nodeIds, "slash"); + + const takeSnapshotBtn = this.getChildControl("take-snapshot-btn"); + const convertToStudyBtn = this.getChildControl("convert-to-study-btn"); + const primaryBtn = this.getChildControl("primary-study-btn"); + if (study.isSnapshot()) { + takeSnapshotBtn.exclude(); + convertToStudyBtn.show(); + primaryBtn.show(); + } else { + takeSnapshotBtn.setVisibility(osparc.data.Permissions.getInstance().canDo("study.snapshot.create") ? "visible" : "excluded"); + convertToStudyBtn.exclude(); + primaryBtn.exclude(); + } + + study.getWorkbench().addListener("nNodesChanged", this.evalSnapshotsBtn, this); + this.evalSnapshotsBtn(); + + study.isSnapshot() ? this._startStopBtns.exclude() : this._startStopBtns.show(); + } + }, + + evalSnapshotsBtn: function() { + const study = this.getStudy(); + if (study) { + const allNodes = study.getWorkbench().getNodes(true); + const hasIterators = Object.values(allNodes).some(node => node.isIterator()); + const isSnapshot = study.isSnapshot(); + const snapshotsBtn = this.getChildControl("snapshots-btn"); + (hasIterators && !isSnapshot) ? snapshotsBtn.show() : snapshotsBtn.exclude(); } }, From 630b5def56c5fbe57b26018f8c6fd86a7fbdd941 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:26:09 +0200 Subject: [PATCH 015/137] navigate through snapshot/primary --- .../class/osparc/desktop/WorkbenchView.js | 100 +++++++++++++++--- 1 file changed, 86 insertions(+), 14 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 0ed2343c11d..7efbf30aa4b 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -185,30 +185,87 @@ qx.Class.define("osparc.desktop.WorkbenchView", { }); }, - __showSweeper: function() { + __showParameters: function() { const study = this.getStudy(); const sweeper = new osparc.component.sweeper.Sweeper(study); const title = this.tr("Sweeper"); - const win = osparc.ui.window.Window.popUpInWindow(sweeper, title, 400, 700); - sweeper.addListener("iterationSelected", e => { + const win = osparc.ui.window.Window.popUpInWindow(sweeper, title, 400, 500); + sweeper.addListener("openPrimaryStudy", e => { win.close(); - const iterationStudyId = e.getData(); + const primaryStudyId = e.getData(); + this.__switchStudy(primaryStudyId); + }); + }, + + __takeSnapshot: function() { + const study = this.getStudy(); + const takeSnapshotView = new osparc.component.snapshots.TakeSnapshotView(study); + const title = this.tr("Take Snapshot"); + const win = osparc.ui.window.Window.popUpInWindow(takeSnapshotView, title, 400, 120); + takeSnapshotView.addListener("takeSnapshot", () => { + const label = takeSnapshotView.getLabel(); + const saveData = takeSnapshotView.getSaveData(); + const workbenchToolbar = this.__mainPanel.getToolbar(); + const takeSnapshotBtn = workbenchToolbar.getChildControl("take-snapshot-btn"); + takeSnapshotBtn.setFetching(true); const params = { url: { - "studyId": iterationStudyId + "studyId": study.getUuid() + }, + data: { + "label": label, + "save_data": saveData } }; - osparc.data.Resources.getOne("studies", params) - .then(studyData => { - study.removeIFrames(); - const data = { - studyId: studyData.uuid - }; - this.fireDataEvent("startStudy", data); - }); + osparc.data.Resources.fetch("snapshots", "takeSnapshot", params) + .then(data => { + console.log(data); + workbenchToolbar.evalSnapshotsBtn(); + }) + .catch(err => osparc.component.message.FlashMessenger.getInstance().logAs(err.message, "ERROR")) + .finally(takeSnapshotBtn.setFetching(false)); + + win.close(); + }, this); + takeSnapshotView.addListener("cancel", () => { + win.close(); + }, this); + }, + + __showSnapshots: function() { + const study = this.getStudy(); + const sweeper = new osparc.component.snapshots.SnapshotsView(study); + const title = this.tr("Snapshots"); + const win = osparc.ui.window.Window.popUpInWindow(sweeper, title, 600, 500); + [ + "openPrimaryStudy", + "openSnapshot" + ].forEach(signalName => { + sweeper.addListener(signalName, e => { + win.close(); + const studyId = e.getData(); + this.__switchStudy(studyId); + }); }); }, + __switchStudy: function(studyId) { + const params = { + url: { + "studyId": studyId + } + }; + osparc.data.Resources.getOne("studies", params) + .then(studyData => { + const study = this.getStudy(); + study.removeIFrames(); + const data = { + studyId: studyData.uuid + }; + this.fireDataEvent("startStudy", data); + }); + }, + __showWorkbenchUI: function() { const workbench = this.getStudy().getWorkbench(); const currentNode = workbench.getNode(this.__currentNodeId); @@ -494,7 +551,22 @@ qx.Class.define("osparc.desktop.WorkbenchView", { this.nodeSelected(nodeId); }, this); }); - workbenchToolbar.addListener("showSweeper", this.__showSweeper, this); + workbenchToolbar.addListener("showSweeper", this.__showParameters, this); + if (!workbenchToolbar.hasListener("showParameters")) { + workbenchToolbar.addListener("showParameters", this.__showParameters, this); + } + if (!workbenchToolbar.hasListener("takeSnapshot")) { + workbenchToolbar.addListener("takeSnapshot", this.__takeSnapshot, this); + } + if (!workbenchToolbar.hasListener("showSnapshots")) { + workbenchToolbar.addListener("showSnapshots", this.__showSnapshots, this); + } + if (!workbenchToolbar.hasListener("openPrimaryStudy")) { + workbenchToolbar.addListener("openPrimaryStudy", e => { + const primaryStudyId = e.getData(); + this.__switchStudy(primaryStudyId); + }, this); + } nodesTree.addListener("changeSelectedNode", e => { const node = workbenchUI.getNodeUI(e.getData()); From 8787d850966bd27d0fd81e7469264c1af051bdec Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:50:02 +0200 Subject: [PATCH 016/137] isIterator --- .../web/client/source/class/osparc/data/model/Node.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/web/client/source/class/osparc/data/model/Node.js b/services/web/client/source/class/osparc/data/model/Node.js index 93c25f4c0ae..58afbb2f4d6 100644 --- a/services/web/client/source/class/osparc/data/model/Node.js +++ b/services/web/client/source/class/osparc/data/model/Node.js @@ -214,6 +214,10 @@ qx.Class.define("osparc.data.model.Node", { return (metaData && metaData.key && metaData.key.includes("nodes-group")); }, + isIterator: function(metaData) { + return (metaData && metaData.key && metaData.key.includes("data-iterator")); + }, + isDynamic: function(metaData) { return (metaData && metaData.type && metaData.type === "dynamic"); }, @@ -263,6 +267,10 @@ qx.Class.define("osparc.data.model.Node", { return osparc.data.model.Node.isContainer(this.getMetaData()); }, + isIterator: function() { + return osparc.data.model.Node.isIterator(this.getMetaData()); + }, + isDynamic: function() { return osparc.data.model.Node.isDynamic(this.getMetaData()); }, From 17adfe552e25dd105d18a5239be1e49959ed0d83 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 14:52:29 +0200 Subject: [PATCH 017/137] ask for using cache eonly for snapshots --- .../class/osparc/desktop/StudyEditor.js | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/StudyEditor.js b/services/web/client/source/class/osparc/desktop/StudyEditor.js index 6b309e28be8..485f514635e 100644 --- a/services/web/client/source/class/osparc/desktop/StudyEditor.js +++ b/services/web/client/source/class/osparc/desktop/StudyEditor.js @@ -235,17 +235,21 @@ qx.Class.define("osparc.desktop.StudyEditor", { }, __requestStartPipeline: function(studyId, partialPipeline = [], forceRestart = false) { - const startPipelineView = new osparc.desktop.StartPipelineView(partialPipeline, forceRestart); - const win = osparc.ui.window.Window.popUpInWindow(startPipelineView, "Start Pipeline", 250, 290); - startPipelineView.addListener("startPipeline", e => { - const data = e.getData(); - const useCache = data["useCache"]; - this.__reallyRequestStartPipeline(studyId, partialPipeline, forceRestart, useCache); - win.close(); - }, this); - startPipelineView.addListener("cancel", () => { - win.close(); - }, this); + if (this.getStudy().isSnapshot()) { + const startPipelineView = new osparc.desktop.StartPipelineView(partialPipeline, forceRestart); + const win = osparc.ui.window.Window.popUpInWindow(startPipelineView, "Start Pipeline", 250, 290); + startPipelineView.addListener("startPipeline", e => { + const data = e.getData(); + const useCache = data["useCache"]; + this.__reallyRequestStartPipeline(studyId, partialPipeline, forceRestart, useCache); + win.close(); + }, this); + startPipelineView.addListener("cancel", () => { + win.close(); + }, this); + } else { + this.__reallyRequestStartPipeline(studyId, partialPipeline, forceRestart); + } }, __reallyRequestStartPipeline: function(studyId, partialPipeline = [], forceRestart = false, useCache = false) { From 6c737a96b90e96f5dd2f286850e5e4ef35da0c01 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 9 Aug 2021 15:57:52 +0200 Subject: [PATCH 018/137] aesthetics --- .../web/client/source/class/osparc/desktop/WorkbenchToolbar.js | 2 +- .../web/client/source/class/osparc/desktop/WorkbenchView.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js index 13e76605f10..57c30dc2395 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js @@ -50,7 +50,7 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { } case "take-snapshot-btn": { control = new osparc.ui.form.FetchButton(this.tr("Take Snapshot")).set({ - icon: "@FontAwesome5Solid/camera/14", + icon: "@FontAwesome5Solid/code-branch/14", ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, allowGrowX: false }); diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 7efbf30aa4b..2db55ba49f5 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -201,7 +201,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { const study = this.getStudy(); const takeSnapshotView = new osparc.component.snapshots.TakeSnapshotView(study); const title = this.tr("Take Snapshot"); - const win = osparc.ui.window.Window.popUpInWindow(takeSnapshotView, title, 400, 120); + const win = osparc.ui.window.Window.popUpInWindow(takeSnapshotView, title, 400, 140); takeSnapshotView.addListener("takeSnapshot", () => { const label = takeSnapshotView.getLabel(); const saveData = takeSnapshotView.getSaveData(); From 076240fd2777c73febac5573ff2aa079ef7bb29e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 1 Jul 2021 17:37:22 +0200 Subject: [PATCH 019/137] Deprecates displayOrder --- packages/models-library/src/models_library/services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 84bd5a2b811..205f2ecd64d 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -283,12 +283,14 @@ class ServiceKeyVersion(BaseModel): "simcore/services/comp/itis/sleeper", "simcore/services/dynamic/3dviewer", ], + regex=KEY_RE, ) version: str = Field( ..., description="service version number", regex=VERSION_RE, examples=["1.0.0", "0.0.1"], + regex=VERSION_RE, ) From 94d9180f85f5ae16e3d93a78cddba997e6eb4637 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 01:11:30 +0200 Subject: [PATCH 020/137] Adding dev-feature flag and cleanup --- .../src/simcore_service_webserver/projects/module_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/module_setup.py b/services/web/server/src/simcore_service_webserver/projects/module_setup.py index d5d6e38f3c4..c723f4e8abe 100644 --- a/services/web/server/src/simcore_service_webserver/projects/module_setup.py +++ b/services/web/server/src/simcore_service_webserver/projects/module_setup.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -def _create_routes(tag, specs, *handlers_module, disable_login=False): +def _create_routes(tag, specs, *handlers_module, disable_login: bool = False): """ :param disable_login: Disables login_required decorator for testing purposes defaults to False :type disable_login: bool, optional From bf393093ee97bee3a520a49b18cdc27df4630bfc Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 01:15:25 +0200 Subject: [PATCH 021/137] Adds module with setup and handlers --- .../parametrization.py | 32 ++++ .../parametrization_api_handlers.py | 141 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 services/web/server/src/simcore_service_webserver/parametrization.py create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py new file mode 100644 index 00000000000..96614a3451e --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -0,0 +1,32 @@ +""" parametrization app module setup + + - Project parametrization + - Project snapshots + +""" +import logging + +from aiohttp import web +from servicelib.application_setup import ModuleCategory, app_module_setup + +from . import parametrization_api_handlers +from .constants import APP_SETTINGS_KEY +from .settings import ApplicationSettings + +log = logging.getLogger(__name__) + + +@app_module_setup( + __name__, + ModuleCategory.ADDON, + depends=["simcore_service_webserver.projects"], + logger=log, +) +def setup(app: web.Application): + + settings: ApplicationSettings = app[APP_SETTINGS_KEY] + if not settings.WEBSERVER_DEV_FEATURES_ENABLED: + log.warning("App module %s disabled: Marked as dev feature", __name__) + return False + + app.add_routes(parametrization_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py new file mode 100644 index 00000000000..eabf39feda0 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -0,0 +1,141 @@ +import json +from functools import wraps +from typing import Any, Callable, Dict, List +from uuid import UUID + +from aiohttp import web +from aiohttp.web_routedef import RouteDef +from pydantic.decorator import validate_arguments +from pydantic.error_wrappers import ValidationError + +from ._meta import api_version_prefix +from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY + +json_dumps = json.dumps + + +def handle_request_errors(handler: Callable): + """ + - required and type validation of path and query parameters + """ + + @wraps(handler) + async def wrapped(request: web.Request): + try: + resp = await handler(request) + return resp + + except KeyError as err: + # NOTE: handles required request.match_info[*] or request.query[*] + raise web.HTTPBadRequest(reason="Expected parameter {err}") from err + + except ValidationError as err: + # NOTE: pydantic.validate_arguments parses and validates -> ValidationError + raise web.HTTPUnprocessableEntity( + text=json_dumps({"error": err.errors()}), + content_type="application/json", + ) from err + + return wrapped + + +# API ROUTES HANDLERS --------------------------------------------------------- +routes = web.RouteTableDef() + + +@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots") +@handle_request_errors +async def _list_project_snapshots_handler(request: web.Request): + """ + Lists references on project snapshots + """ + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + snapshots = await list_project_snapshots( + project_id=request.match_info["project_id"], # type: ignore + ) + + # Append url links + url_for_snapshot = request.app.router["_get_project_snapshot_handler"].url_for + url_for_parameters = request.app.router[ + "_get_project_snapshot_parameters_handler" + ].url_for + + for snp in snapshots: + snp["url"] = url_for_snapshot( + project_id=snp["parent_id"], snapshot_id=snp["id"] + ) + snp["url_parameters"] = url_for_parameters( + project_id=snp["parent_id"], snapshot_id=snp["id"] + ) + + return snapshots + + +@validate_arguments +async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: + # project_id is param-project? + + snapshot_info_0 = { + "id": "0", + "display_name": "snapshot 0", + "parent_id": project_id, + "parameters": get_project_snapshot_parameters(project_id, snapshot_id="0"), + } + return [ + snapshot_info_0, + ] + + +@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}") +@handle_request_errors +async def _get_project_snapshot_handler(request: web.Request): + """ + Returns full project. Equivalent to /projects/{snapshot_project_id} + """ + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + await get_project_snapshot( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + +@validate_arguments +async def get_project_snapshot(project_id: UUID, snapshot_id: str): + pass + + +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters" +) +@handle_request_errors +async def _get_project_snapshot_parameters_handler( + request: web.Request, +): + # GET /projects/{id}/snapshots/{id}/parametrization -> {x:3, y:0, ...} + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + params = await get_project_snapshot_parameters( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + return params + + +@validate_arguments +async def get_project_snapshot_parameters( + project_id: UUID, snapshot_id: str +) -> Dict[str, Any]: + return {"x": 4, "y": "yes"} + + +# ------------------------------------- +assert routes # nosec + +# NOTE: names all routes with handler's +# TODO: override routes functions ? +for route_def in routes: + assert isinstance(route_def, RouteDef) # nosec + route_def.kwargs["name"] = route_def.handler.__name__ From 530e9ebcf7abc209471f00ad52b733e171ebd623 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 02:12:44 +0200 Subject: [PATCH 022/137] WIP [skip ci] --- .../parametrization.py | 2 +- .../parametrization_api_handlers.py | 20 ++++++-- .../parametrization_core.py | 8 ++++ .../parametrization_models.py | 8 ++++ .../parametrization_settings.py | 1 + .../isolated/test_parametrization_core.py | 47 +++++++++++++++++++ 6 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_core.py create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_models.py create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_settings.py create mode 100644 services/web/server/tests/unit/isolated/test_parametrization_core.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index 96614a3451e..5d53b8968c4 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -26,7 +26,7 @@ def setup(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: - log.warning("App module %s disabled: Marked as dev feature", __name__) + log.warning("App module '%s' is disabled: Marked as dev feature", __name__) return False app.add_routes(parametrization_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index eabf39feda0..67881ea5c57 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -5,6 +5,7 @@ from aiohttp import web from aiohttp.web_routedef import RouteDef +from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError @@ -75,13 +76,21 @@ async def _list_project_snapshots_handler(request: web.Request): @validate_arguments async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: # project_id is param-project? - + # TODO: add pagination + # TODO: optimizaiton will grow snapshots of a project with time! + # + + # snapshots: + # - ordered (iterations!) + # - have a parent project with all the parametrization + # snapshot_info_0 = { - "id": "0", + "id": 0, "display_name": "snapshot 0", "parent_id": project_id, - "parameters": get_project_snapshot_parameters(project_id, snapshot_id="0"), + "parameters": get_project_snapshot_parameters(project_id, snapshot_id=str(id)), } + return [ snapshot_info_0, ] @@ -103,7 +112,10 @@ async def _get_project_snapshot_handler(request: web.Request): @validate_arguments async def get_project_snapshot(project_id: UUID, snapshot_id: str): - pass + # TODO: create a fake project + # - generate project_id + # - define what changes etc... + project = Project() @routes.get( diff --git a/services/web/server/src/simcore_service_webserver/parametrization_core.py b/services/web/server/src/simcore_service_webserver/parametrization_core.py new file mode 100644 index 00000000000..c1493f33326 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_core.py @@ -0,0 +1,8 @@ +# +# +# Parameter node needs to be evaluated before the workflow is submitted +# Every evaluation creates a snapshot +# +# Constant Param has 1 input adn one output +# Optimizer can produce semi-cyclic feedback loop by connecting to output of target to +# diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/parametrization_models.py new file mode 100644 index 00000000000..7167e18fccf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_models.py @@ -0,0 +1,8 @@ +from typing import Union + +from pydantic import BaseModel + + +class Parameter(BaseModel): + name: str + value: Union[bool, float, str] diff --git a/services/web/server/src/simcore_service_webserver/parametrization_settings.py b/services/web/server/src/simcore_service_webserver/parametrization_settings.py new file mode 100644 index 00000000000..d3b6b48b51c --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_settings.py @@ -0,0 +1 @@ +# TODO: do not enable diff --git a/services/web/server/tests/unit/isolated/test_parametrization_core.py b/services/web/server/tests/unit/isolated/test_parametrization_core.py new file mode 100644 index 00000000000..4d34dde0e0e --- /dev/null +++ b/services/web/server/tests/unit/isolated/test_parametrization_core.py @@ -0,0 +1,47 @@ +from typing import Iterator, Tuple +from uuid import uuid4 + +import pytest +from models_library.projects import Project, ProjectAtDB +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID +from simcore_service_webserver.projects.projects_api import get_project_for_user + + +# is parametrized project? +def is_param(node: Node): + return node.key.split("/")[-2] == "param" + + +def is_parametrized_project(project: Project): + return any(is_param(node) for node in project.workbench.values()) + + +def iter_params(project: Project) -> Iterator[Tuple[NodeID, Node]]: + for node_id, node in project.workbench.items(): + if is_param(node): + yield NodeID(node_id), node + + +def is_const_param(node: Node) -> bool: + return is_param(node) and "const" in node.key and not node.inputs + + +async def test_it(app): + + project_id = str(uuid4()) + user_id = 0 + + prj_dict = await get_project_for_user( + app, project_id, user_id, include_templates=False, include_state=True + ) + + parent_project = Project.parse_obj(prj_dict) + + # three types of "parametrization" nodes + + # const -> replace in connecting nodes + + # generators -> iterable data sources (no inputs connected!) + + # feedback -> parametrizations with connected inputs! From d1619bfe895be245a6243ad27627b0aedbfa57af Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 03:00:09 +0200 Subject: [PATCH 023/137] adds skipsetup exception in servicelib --- .../src/servicelib/application_setup.py | 42 ++++++++++++------- .../tests/test_application_setup.py | 27 ++++++++++-- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 2b75d859222..4b01f3b50b4 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -18,6 +18,12 @@ class ModuleCategory(Enum): ADDON = 1 +class SkipModuleSetupException(Exception): + def __init__(self, *, reason) -> None: + self.reason = reason + super().__init__(reason) + + class ApplicationSetupError(Exception): pass @@ -33,9 +39,9 @@ def app_module_setup( depends: Optional[List[str]] = None, config_section: str = None, config_enabled: str = None, - logger: Optional[logging.Logger] = None, + logger: logging.Logger = log, ) -> Callable: - """ Decorator that marks a function as 'a setup function' for a given module in an application + """Decorator that marks a function as 'a setup function' for a given module in an application - Marks a function as 'setup' of a given module in an application - Ensures setup executed ONLY ONCE per app @@ -76,8 +82,6 @@ def setup(app: web.Application): # if passes config_enabled, invalidates info on section section = None - logger = logger or log - def decorate(setup_func): if "setup" not in setup_func.__name__: @@ -147,17 +151,26 @@ def _get(cfg_, parts): raise ApplicationSetupError(msg) # execution of setup - ok = setup_func(app, *args, **kargs) + try: + completed = setup_func(app, *args, **kargs) - # post-setup - if ok is None: - ok = True + # post-setup + if completed is None: + completed = True - if ok: - app[APP_SETUP_KEY].append(module_name) + if completed: + app[APP_SETUP_KEY].append(module_name) + else: + raise SkipModuleSetupException(reason="Undefined") - logger.debug("'%s' setup completed [%s]", module_name, ok) - return ok + except SkipModuleSetupException as exc: + logger.warning("Skipping '%s' setup: %s", module_name, exc.reason) + completed = False + + logger.debug( + "'%s' setup %s", module_name, "completed" if completed else "skipped" + ) + return completed setup_wrapper.metadata = setup_metadata setup_wrapper.MARK = "setup" @@ -170,10 +183,9 @@ def _get(cfg_, parts): def is_setup_function(fun): return ( inspect.isfunction(fun) - and hasattr(fun, "MARK") - and fun.MARK == "setup" + and getattr(fun, "MARK", None) == "setup" and any( param.annotation == web.Application - for name, param in inspect.signature(fun).parameters.items() + for _, param in inspect.signature(fun).parameters.items() ) ) diff --git a/packages/service-library/tests/test_application_setup.py b/packages/service-library/tests/test_application_setup.py index a003074862d..7c4c9febdf9 100644 --- a/packages/service-library/tests/test_application_setup.py +++ b/packages/service-library/tests/test_application_setup.py @@ -4,6 +4,7 @@ import logging from typing import Dict +from unittest.mock import Mock import pytest from aiohttp import web @@ -12,19 +13,22 @@ APP_SETUP_KEY, DependencyError, ModuleCategory, + SkipModuleSetupException, app_module_setup, ) -log = logging.getLogger(__name__) +log = Mock() @app_module_setup("package.bar", ModuleCategory.ADDON, logger=log) -def setup_bar(app: web.Application, arg1, kargs=55): +def setup_bar(app: web.Application, arg1, *, raise_skip: bool = False): return True @app_module_setup("package.foo", ModuleCategory.ADDON, logger=log) -def setup_foo(app: web.Application, arg1, kargs=33): +def setup_foo(app: web.Application, arg1, kargs=33, *, raise_skip: bool = False): + if raise_skip: + raise SkipModuleSetupException(reason="explicit skip") return True @@ -92,3 +96,20 @@ def test_marked_setup(app_config, app): app_config["foo"]["enabled"] = False assert not setup_foo(app, 2) + + +def test_skip_setup(app_config, app): + try: + log.reset_mock() + + assert not setup_foo(app, 1, raise_skip=True) + + # FIXME: mock logger + # assert log.warning.called + # warn_msg = log.warning.call_args()[0] + # assert "package.foo" in warn_msg + # assert "explicit skip" in warn_msg + + assert setup_foo(app, 1) + finally: + log.reset_mock() From 0a6b352eb84fc09ef88eb3cc7e52789f7e42160d Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 03:01:18 +0200 Subject: [PATCH 024/137] Uses skip setup and skips tests --- .../src/simcore_service_webserver/parametrization.py | 9 ++++++--- .../tests/unit/isolated/test_parametrization_core.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index 5d53b8968c4..a1022a28c68 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -7,7 +7,11 @@ import logging from aiohttp import web -from servicelib.application_setup import ModuleCategory, app_module_setup +from servicelib.application_setup import ( + ModuleCategory, + SkipModuleSetupException, + app_module_setup, +) from . import parametrization_api_handlers from .constants import APP_SETTINGS_KEY @@ -26,7 +30,6 @@ def setup(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: - log.warning("App module '%s' is disabled: Marked as dev feature", __name__) - return False + raise SkipModuleSetupException(reason="Development feature") app.add_routes(parametrization_api_handlers.routes) diff --git a/services/web/server/tests/unit/isolated/test_parametrization_core.py b/services/web/server/tests/unit/isolated/test_parametrization_core.py index 4d34dde0e0e..ca759ba987e 100644 --- a/services/web/server/tests/unit/isolated/test_parametrization_core.py +++ b/services/web/server/tests/unit/isolated/test_parametrization_core.py @@ -27,6 +27,7 @@ def is_const_param(node: Node) -> bool: return is_param(node) and "const" in node.key and not node.inputs +@pytest.mark.skip(reason="UNDER DEV") async def test_it(app): project_id = str(uuid4()) From cc91606130e1f4b9bda2a269e35241415c071c15 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 03:05:39 +0200 Subject: [PATCH 025/137] name-mania --- .../service-library/src/servicelib/application_setup.py | 6 +++--- packages/service-library/tests/test_application_setup.py | 4 ++-- .../server/src/simcore_service_webserver/parametrization.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 4b01f3b50b4..89377797728 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -18,7 +18,7 @@ class ModuleCategory(Enum): ADDON = 1 -class SkipModuleSetupException(Exception): +class SkipModuleSetup(Exception): def __init__(self, *, reason) -> None: self.reason = reason super().__init__(reason) @@ -161,9 +161,9 @@ def _get(cfg_, parts): if completed: app[APP_SETUP_KEY].append(module_name) else: - raise SkipModuleSetupException(reason="Undefined") + raise SkipModuleSetup(reason="Undefined") - except SkipModuleSetupException as exc: + except SkipModuleSetup as exc: logger.warning("Skipping '%s' setup: %s", module_name, exc.reason) completed = False diff --git a/packages/service-library/tests/test_application_setup.py b/packages/service-library/tests/test_application_setup.py index 7c4c9febdf9..a37b6268c17 100644 --- a/packages/service-library/tests/test_application_setup.py +++ b/packages/service-library/tests/test_application_setup.py @@ -13,7 +13,7 @@ APP_SETUP_KEY, DependencyError, ModuleCategory, - SkipModuleSetupException, + SkipModuleSetup, app_module_setup, ) @@ -28,7 +28,7 @@ def setup_bar(app: web.Application, arg1, *, raise_skip: bool = False): @app_module_setup("package.foo", ModuleCategory.ADDON, logger=log) def setup_foo(app: web.Application, arg1, kargs=33, *, raise_skip: bool = False): if raise_skip: - raise SkipModuleSetupException(reason="explicit skip") + raise SkipModuleSetup(reason="explicit skip") return True diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index a1022a28c68..00c18833ac6 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -9,7 +9,7 @@ from aiohttp import web from servicelib.application_setup import ( ModuleCategory, - SkipModuleSetupException, + SkipModuleSetup, app_module_setup, ) @@ -30,6 +30,6 @@ def setup(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: - raise SkipModuleSetupException(reason="Development feature") + raise SkipModuleSetup(reason="Development feature") app.add_routes(parametrization_api_handlers.routes) From 0c33c8574504114e565683f49b1842fa2954f93e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sun, 11 Jul 2021 17:08:49 +0200 Subject: [PATCH 026/137] WIP --- api/specs/webserver/openapi-projects.yaml | 47 ++++++ api/specs/webserver/openapi.yaml | 9 ++ .../parametrization_api_handlers.py | 8 +- .../catalog_services_openapi_generator.py | 4 +- .../sandbox/projects_openapi_generator.py | 136 ++++++++++++++++++ 5 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 services/web/server/tests/sandbox/projects_openapi_generator.py diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 3f8f909783a..1a66a9011cc 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -435,6 +435,53 @@ paths: default: $ref: "#/components/responses/DefaultErrorResponse" + projects_snapshots: + get: + summary: List Project Snapshots + parameters: + - in: path + name: project_id + required: true + schema: + format: uuid + title: Project Id + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + + projects_id_snapshots_id: + get: + summary: Get Project Snapshot + parameters: + - in: path + name: project_id + required: true + schema: + format: uuid + title: Project Id + type: string + - in: path + name: snapshot_id + required: true + schema: + exclusiveMinimum: 0.0 + title: Snapshot Id + type: integer + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + + projects_id_snapshots_id_parameters: + get: + + components: schemas: ClientSessionId: diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index fa8af63df8f..518bf6b64a9 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -210,6 +210,15 @@ paths: /projects/{study_uuid}/tags/{tag_id}: $ref: "./openapi-projects.yaml#/paths/~1projects~1{study_uuid}~1tags~1{tag_id}" + /projects/{project_id}/snapshots: + $ref: "./openapi-projects.yaml#/paths/projects_snapshots" + + /projects/{project_id}/snapshots/{snapshot_id}: + $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id" + + /projects/{project_id}/snapshots/{snapshot_id}/parameters: + $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id_parameters" + # ACTIVITY ------------------------------------------------------------------------- /activity/status: $ref: "./openapi-activity.yaml#/paths/~1activity~1status" diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index 67881ea5c57..4f5cbb68836 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -69,6 +69,7 @@ async def _list_project_snapshots_handler(request: web.Request): snp["url_parameters"] = url_for_parameters( project_id=snp["parent_id"], snapshot_id=snp["id"] ) + # snp['url_project'] = return snapshots @@ -104,18 +105,20 @@ async def _get_project_snapshot_handler(request: web.Request): """ user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - await get_project_snapshot( + prj_dict = await get_project_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) + return prj_dict # ??? @validate_arguments -async def get_project_snapshot(project_id: UUID, snapshot_id: str): +async def get_project_snapshot(project_id: UUID, snapshot_id: str) -> Dict[str, Any]: # TODO: create a fake project # - generate project_id # - define what changes etc... project = Project() + return project.dict() @routes.get( @@ -140,6 +143,7 @@ async def _get_project_snapshot_parameters_handler( async def get_project_snapshot_parameters( project_id: UUID, snapshot_id: str ) -> Dict[str, Any]: + # return {"x": 4, "y": "yes"} diff --git a/services/web/server/tests/sandbox/catalog_services_openapi_generator.py b/services/web/server/tests/sandbox/catalog_services_openapi_generator.py index dd42a98c42e..5d8b4c29dc8 100644 --- a/services/web/server/tests/sandbox/catalog_services_openapi_generator.py +++ b/services/web/server/tests/sandbox/catalog_services_openapi_generator.py @@ -1,9 +1,7 @@ # pylint: disable=unused-argument import json -import sys -from pathlib import Path -from typing import List, Optional +from typing import List from fastapi import FastAPI, Query from simcore_service_webserver.catalog_api_models import ( diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py new file mode 100644 index 00000000000..ad0c8704588 --- /dev/null +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -0,0 +1,136 @@ +# +# assist creating OAS for projects resource +# + +import json +import uuid +from datetime import datetime +from typing import Any, Dict, Optional, Union +from uuid import UUID + +import yaml +from fastapi import FastAPI, status +from pydantic import BaseModel, Field, PositiveInt +from pydantic.networks import AnyUrl + +app = FastAPI() + +error_responses = { + status.HTTP_400_BAD_REQUEST: {}, + status.HTTP_422_UNPROCESSABLE_ENTITY: {}, +} + +BuiltinTypes = Union[bool, int, float, str] +DataSchema = Union[ + BuiltinTypes, +] # any json schema? +DataLink = AnyUrl + +DataSchema = Union[DataSchema, DataLink] + + +class Node(BaseModel): + key: str + version: str = Field(..., regex=r"\d+\.\d+\.\d+") + label: str + + inputs: Dict[str, DataSchema] + # var inputs? + outputs: Dict[str, DataSchema] + # var outputs? + + +class Project(BaseModel): + id: UUID + pipeline: Dict[UUID, Node] + + +class ProjectSnapshot(BaseModel): + id: int + label: str + parent_project_id: UUID + parameters: Dict[str, Any] = {} + + taken_at: Optional[datetime] = Field( + None, + description="Timestamp of the time snapshot was taken", + ) + + +@app.get("/projects/{project_id}") +def get_project(project_id: UUID): + pass + + +@app.post("/projects/{project_id}") +def create_project(project: Project): + pass + + +@app.put("/projects/{project_id}") +def replace_project(project_id: UUID, project: Project): + pass + + +@app.patch("/projects/{project_id}") +def update_project(project_id: UUID, project: Project): + pass + + +@app.delete("/projects/{project_id}") +def delete_project(project_id: UUID): + pass + + +@app.post("/projects/{project_id}:open") +def open_project(project: Project): + pass + + +@app.post("/projects/{project_id}:start") +def start_project(project: Project): + pass + + +@app.post("/projects/{project_id}:stop") +def stop_project(project: Project): + pass + + +@app.post("/projects/{project_id}:close") +def close_project(project: Project): + pass + + +# ------------- + + +@app.get("/projects/{project_id}/snapshots") +async def list_project_snapshots(project_id: UUID): + pass + + +@app.post("/projects/{project_id}/snapshots") +async def create_project_snapshot( + project_id: UUID, snapshot_label: Optional[str] = None +): + pass + + +@app.get("/projects/{project_id}/snapshots/{snapshot_id}") +async def get_project_snapshot(project_id: UUID, snapshot_id: PositiveInt): + pass + + +@app.get("/projects/{project_id}/snapshots/{snapshot_id}/parameters") +async def get_project_snapshot_parameters(project_id: UUID, snapshot_id: str): + return {"x": 4, "y": "yes"} + + +# print(yaml.safe_dump(app.openapi())) +# print("-"*100) + + +print(json.dumps(app.openapi(), indent=2)) + +# uvicorn --reload projects_openapi_generator:app From c42754cacafb9c2749cf1d7e2dc776ef237d4d32 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 12 Jul 2021 19:29:42 +0200 Subject: [PATCH 027/137] WIP --- .../sandbox/projects_openapi_generator.py | 294 +++++++++++++++--- 1 file changed, 247 insertions(+), 47 deletions(-) diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py index ad0c8704588..58467dddc1f 100644 --- a/services/web/server/tests/sandbox/projects_openapi_generator.py +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -1,16 +1,27 @@ # -# assist creating OAS for projects resource +# Assists on the creation of project's OAS # import json -import uuid +from collections import defaultdict from datetime import datetime -from typing import Any, Dict, Optional, Union -from uuid import UUID - -import yaml -from fastapi import FastAPI, status -from pydantic import BaseModel, Field, PositiveInt +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from uuid import UUID, uuid3, uuid4 + +from fastapi import Depends, FastAPI +from fastapi import Path as PathParam +from fastapi import Request, status +from fastapi.exceptions import HTTPException +from models_library.services import PROPERTY_KEY_RE +from pydantic import ( + BaseModel, + Field, + PositiveInt, + StrictBool, + StrictFloat, + StrictInt, + constr, +) from pydantic.networks import AnyUrl app = FastAPI() @@ -20,7 +31,11 @@ status.HTTP_422_UNPROCESSABLE_ENTITY: {}, } -BuiltinTypes = Union[bool, int, float, str] + +InputID = OutputID = constr(regex=PROPERTY_KEY_RE) + +# WARNING: oder matters +BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] DataSchema = Union[ BuiltinTypes, ] # any json schema? @@ -34,9 +49,9 @@ class Node(BaseModel): version: str = Field(..., regex=r"\d+\.\d+\.\d+") label: str - inputs: Dict[str, DataSchema] + inputs: Dict[InputID, DataSchema] # var inputs? - outputs: Dict[str, DataSchema] + outputs: Dict[OutputID, DataSchema] # var outputs? @@ -44,87 +59,272 @@ class Project(BaseModel): id: UUID pipeline: Dict[UUID, Node] + def update_ids(self, name: str): + map_ids: Dict[UUID, UUID] = {} + map_ids[self.id] = uuid3(self.id, name) + map_ids.update({node_id: uuid3(node_id, name) for node_id in self.pipeline}) + + # replace ALL references + + +class Parameter(BaseModel): + name: str + value: BuiltinTypes + + node_id: UUID + output_id: OutputID -class ProjectSnapshot(BaseModel): - id: int - label: str - parent_project_id: UUID - parameters: Dict[str, Any] = {} - taken_at: Optional[datetime] = Field( - None, - description="Timestamp of the time snapshot was taken", +class Snapshot(BaseModel): + id: PositiveInt = Field(..., description="Unique snapshot identifier") + label: Optional[str] = Field(None, description="Unique human readable display name") + created_at: datetime = Field( + default_factory=datetime.utcnow, + description="Timestamp of the time snapshot was taken from parent. Notice that parent might change with time", ) + parent_id: UUID = Field(..., description="Parent's project uuid") + project_id: UUID = Field(..., description="Current project's uuid") + + +class ParameterApiModel(Parameter): + url: AnyUrl + # url_output: AnyUrl + + +class SnapshotApiModel(Snapshot): + url: AnyUrl + url_parent: AnyUrl + url_project: AnyUrl + url_parameters: Optional[AnyUrl] = None + + @classmethod + def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiModel": + return cls( + url=url_for( + "get_snapshot", + project_id=snapshot.project_id, + snapshot_id=snapshot.id, + ), + url_parent=url_for("get_project", project_id=snapshot.parent_id), + url_project=url_for("get_project", project_id=snapshot.project_id), + url_parameters=url_for( + "get_snapshot_parameters", + project_id=snapshot.parent_id, + snapshot_id=snapshot.id, + ), + **snapshot.dict(), + ) + + +#################################################################### -@app.get("/projects/{project_id}") -def get_project(project_id: UUID): - pass + +_PROJECTS: Dict[UUID, Project] = {} +_PROJECT2SNAPSHOT: Dict[UUID, UUID] = {} +_SNAPSHOTS: Dict[UUID, List[Snapshot]] = defaultdict(list) +_PARAMETERS: Dict[Tuple[UUID, int], List[Parameter]] = defaultdict(list) + + +#################################################################### + + +def get_reverse_url_mapper(request: Request) -> Callable: + def reverse_url_mapper(name: str, **path_params: Any) -> str: + return request.url_for(name, **path_params) + + return reverse_url_mapper + + +def get_valid_id(project_id: UUID = PathParam(...)) -> UUID: + if project_id not in _PROJECTS: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid id") + return project_id + + +#################################################################### + + +@app.get("/projects/{project_id}", response_model=Project) +def get_project(pid: UUID = Depends(get_valid_id)): + return _PROJECTS[pid] @app.post("/projects/{project_id}") def create_project(project: Project): - pass + if project.id not in _PROJECTS: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Invalid id") + _PROJECTS[project.id] = project @app.put("/projects/{project_id}") -def replace_project(project_id: UUID, project: Project): - pass +def replace_project(project: Project, pid: UUID = Depends(get_valid_id)): + _PROJECTS[pid] = project @app.patch("/projects/{project_id}") -def update_project(project_id: UUID, project: Project): - pass +def update_project(project: Project, pid: UUID = Depends(get_valid_id)): + raise NotImplementedError() @app.delete("/projects/{project_id}") -def delete_project(project_id: UUID): - pass +def delete_project(pid: UUID = Depends(get_valid_id)): + del _PROJECTS[pid] @app.post("/projects/{project_id}:open") -def open_project(project: Project): +def open_project(pid: UUID = Depends(get_valid_id)): pass @app.post("/projects/{project_id}:start") -def start_project(project: Project): +def start_project(use_cache: bool = True, pid: UUID = Depends(get_valid_id)): pass @app.post("/projects/{project_id}:stop") -def stop_project(project: Project): +def stop_project(pid: UUID = Depends(get_valid_id)): pass @app.post("/projects/{project_id}:close") -def close_project(project: Project): +def close_project(pid: UUID = Depends(get_valid_id)): pass -# ------------- +@app.get("/projects/{project_id}/snapshots", response_model=List[SnapshotApiModel]) +async def list_snapshots( + pid: UUID = Depends(get_valid_id), + url_for: Callable = Depends(get_reverse_url_mapper), +): + psid = _PROJECT2SNAPSHOT.get(pid) + if not psid: + return [] + project_snapshots: List[Snapshot] = _SNAPSHOTS.get(psid, []) -@app.get("/projects/{project_id}/snapshots") -async def list_project_snapshots(project_id: UUID): - pass + return [ + SnapshotApiModel.from_snapshot(snapshot, url_for) + for snapshot in project_snapshots + ] -@app.post("/projects/{project_id}/snapshots") -async def create_project_snapshot( - project_id: UUID, snapshot_label: Optional[str] = None +@app.post("/projects/{project_id}/snapshots", response_model=SnapshotApiModel) +async def create_snapshot( + pid: UUID = Depends(get_valid_id), + snapshot_label: Optional[str] = None, + url_for: Callable = Depends(get_reverse_url_mapper), ): - pass + # + # copies project and creates project_id + # run will use "use_cache" + # snapshots already in place -@app.get("/projects/{project_id}/snapshots/{snapshot_id}") -async def get_project_snapshot(project_id: UUID, snapshot_id: PositiveInt): - pass + project_snapshots: List[SnapshotApiModel] = await list_snapshots(pid, url_for) + index = project_snapshots[-1].id if len(project_snapshots) else 0 + + if snapshot_label: + if any(s.label == snapshot_label for s in project_snapshots): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"'{snapshot_label}' already exist", + ) + + else: + snapshot_label = f"snapshot {index}" + while any(s.label == snapshot_label for s in project_snapshots): + index += 1 + snapshot_label = f"snapshot {index}" + + # perform snapshot + parent_project = _PROJECTS[pid] + # create new project + project_id = uuid3(namespace=parent_project.id, name=snapshot_label) + project = parent_project.copy(update={"id": project_id}) # THIS IS WRONG -@app.get("/projects/{project_id}/snapshots/{snapshot_id}/parameters") -async def get_project_snapshot_parameters(project_id: UUID, snapshot_id: str): - return {"x": 4, "y": "yes"} + snapshot = Snapshot(id=index, parent_id=pid, project_id=project_id) + + _PROJECTS[project_id] = project + + psid = _PROJECT2SNAPSHOT.setdefault(pid, uuid3(pid, name="snapshots")) + _SNAPSHOTS[psid].append(snapshot) + + # if param-project, then call workflow-compiler to produce parameters + # differenciate between snapshots created automatically from those explicit! + + return SnapshotApiModel( + url=url_for( + "get_snapshot", project_id=snapshot.parent_id, snapshot_id=snapshot.id + ), + **snapshot.dict(), + ) + + +@app.get( + "/projects/{project_id}/snapshots/{snapshot_id}", + response_model=SnapshotApiModel, +) +async def get_snapshot( + snapshot_id: PositiveInt, + pid: UUID = Depends(get_valid_id), + url_for: Callable = Depends(get_reverse_url_mapper), +): + + psid = _PROJECT2SNAPSHOT[pid] + snapshot = next(s for s in _SNAPSHOTS[psid] if s.id == snapshot_id) + + return SnapshotApiModel( + url=url_for( + "get_snapshot", project_id=snapshot.parent_id, snapshot_id=snapshot.id + ), + **snapshot.dict(), + ) + + +@app.get( + "/projects/{project_id}/snapshots/{snapshot_id}/parameters", + response_model=List[ParameterApiModel], +) +async def list_snapshot_parameters( + snapshot_id: str, + pid: UUID = Depends(get_valid_id), + url_for: Callable = Depends(get_reverse_url_mapper), +): + + # get param snapshot + params = {"x": 4, "y": "yes"} + + result = [ + ParameterApiModel( + name=name, + value=value, + node_id=uuid4(), + output_id="output", + url=url_for( + "list_snapshot_parameters", + project_id=pid, + snapshot_id=snapshot_id, + ), + ) + for name, value in params.items() + ] + + return result + + +## workflow compiler ####################################### + + +def create_snapshots(project_id: UUID): + # get project + + # if parametrized + # iterate + # otherwise + # copy workbench and replace uuids + pass # print(yaml.safe_dump(app.openapi())) From eea9406bd8ea429c4df0e9198837568589bb3fe3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 13 Jul 2021 16:57:08 +0200 Subject: [PATCH 028/137] WIP --- .../models/snapshots.py | 38 ++++++ .../parametrization_api_handlers.py | 129 ++++++++++-------- .../parametrization_core.py | 46 +++++++ .../parametrization_models.py | 64 ++++++++- 4 files changed, 216 insertions(+), 61 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/snapshots.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py new file mode 100644 index 00000000000..cb551201d79 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -0,0 +1,38 @@ +import sqlalchemy as sa +from sqlalchemy.sql import func + +from .base import metadata + +snapshots = sa.Table( + "snapshots", + metadata, + sa.Column( + "id", + sa.BigInteger, + nullable=False, + primary_key=True, + doc="Global snapshot identifier index", + ), + sa.Column("name", sa.String, nullable=False, doc="Display name"), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=func.now(), + doc="Timestamp on creation", + ), + sa.Column( + "parent_uuid", + sa.String, + nullable=False, + unique=True, + doc="Parent project's UUID", + ), + sa.Column( + "project_uuid", + sa.String, + nullable=False, + unique=True, + doc="UUID of the project associated to this snapshot", + ), +) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index 4f5cbb68836..c4614d1dadd 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -1,16 +1,17 @@ import json from functools import wraps -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, List, Optional from uuid import UUID from aiohttp import web -from aiohttp.web_routedef import RouteDef from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError +from simcore_service_webserver.parametrization_models import Snapshot from ._meta import api_version_prefix from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY +from .parametrization_models import Snapshot, SnapshotApiModel json_dumps = json.dumps @@ -44,23 +45,24 @@ async def wrapped(request: web.Request): routes = web.RouteTableDef() -@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots") +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + name="_list_snapshots_handler", +) @handle_request_errors -async def _list_project_snapshots_handler(request: web.Request): +async def _list_snapshots_handler(request: web.Request): """ Lists references on project snapshots """ user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - snapshots = await list_project_snapshots( + snapshots = await list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) # Append url links - url_for_snapshot = request.app.router["_get_project_snapshot_handler"].url_for - url_for_parameters = request.app.router[ - "_get_project_snapshot_parameters_handler" - ].url_for + url_for_snapshot = request.app.router["_get_snapshot_handler"].url_for + url_for_parameters = request.app.router["_get_snapshot_parameters_handler"].url_for for snp in snapshots: snp["url"] = url_for_snapshot( @@ -74,8 +76,57 @@ async def _list_project_snapshots_handler(request: web.Request): return snapshots +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}", + name="_get_snapshot_handler", +) +@handle_request_errors +async def _get_snapshot_handler(request: web.Request): + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + snapshot = await get_snapshot( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + return snapshot.json() + + +@routes.post( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + name="_create_snapshot_handler", +) +@handle_request_errors +async def _create_snapshot_handler(request: web.Request): + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + snapshot = await create_snapshot( + project_id=request.match_info["project_id"], # type: ignore + snapshot_label=request.query.get("snapshot_label"), + ) + + return snapshot.json() + + +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", + name="_get_snapshot_parameters_handler", +) +@handle_request_errors +async def _get_snapshot_parameters_handler( + request: web.Request, +): + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + params = await get_snapshot_parameters( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + return params + + @validate_arguments -async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: +async def list_snapshots(project_id: UUID) -> List[Dict[str, Any]]: # project_id is param-project? # TODO: add pagination # TODO: optimizaiton will grow snapshots of a project with time! @@ -89,7 +140,7 @@ async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: "id": 0, "display_name": "snapshot 0", "parent_id": project_id, - "parameters": get_project_snapshot_parameters(project_id, snapshot_id=str(id)), + "parameters": get_snapshot_parameters(project_id, snapshot_id=str(id)), } return [ @@ -97,61 +148,23 @@ async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: ] -@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}") -@handle_request_errors -async def _get_project_snapshot_handler(request: web.Request): - """ - Returns full project. Equivalent to /projects/{snapshot_project_id} - """ - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - prj_dict = await get_project_snapshot( - project_id=request.match_info["project_id"], # type: ignore - snapshot_id=request.match_info["snapshot_id"], - ) - return prj_dict # ??? - - @validate_arguments -async def get_project_snapshot(project_id: UUID, snapshot_id: str) -> Dict[str, Any]: +async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: # TODO: create a fake project # - generate project_id # - define what changes etc... - project = Project() - return project.dict() - + pass -@routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters" -) -@handle_request_errors -async def _get_project_snapshot_parameters_handler( - request: web.Request, -): - # GET /projects/{id}/snapshots/{id}/parametrization -> {x:3, y:0, ...} - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - params = await get_project_snapshot_parameters( - project_id=request.match_info["project_id"], # type: ignore - snapshot_id=request.match_info["snapshot_id"], - ) - - return params +@validate_arguments +async def create_snapshot( + project_id: UUID, + snapshot_label: Optional[str] = None, +) -> Snapshot: + pass @validate_arguments -async def get_project_snapshot_parameters( - project_id: UUID, snapshot_id: str -) -> Dict[str, Any]: +async def get_snapshot_parameters(project_id: UUID, snapshot_id: str): # return {"x": 4, "y": "yes"} - - -# ------------------------------------- -assert routes # nosec - -# NOTE: names all routes with handler's -# TODO: override routes functions ? -for route_def in routes: - assert isinstance(route_def, RouteDef) # nosec - route_def.kwargs["name"] = route_def.handler.__name__ diff --git a/services/web/server/src/simcore_service_webserver/parametrization_core.py b/services/web/server/src/simcore_service_webserver/parametrization_core.py index c1493f33326..fead73ab024 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_core.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_core.py @@ -6,3 +6,49 @@ # Constant Param has 1 input adn one output # Optimizer can produce semi-cyclic feedback loop by connecting to output of target to # + + +from typing import Iterator, Tuple +from uuid import UUID, uuid3 + +from models_library.projects_nodes import Node + +from .parametrization_models import Snapshot +from .projects.projects_db import ProjectAtDB +from .projects.projects_utils import clone_project_document + + +def is_parametrized(node: Node) -> bool: + try: + return "parameter" == node.key.split("/")[-2] + except IndexError: + return False + + +def iter_param_nodes(project: ProjectAtDB) -> Iterator[Tuple[UUID, Node]]: + for node_id, node in project.workbench.items(): + if is_parametrized(node): + yield UUID(node_id), node + + +def is_parametrized_project(project: ProjectAtDB) -> bool: + return any(is_parametrized(node) for node in project.workbench.values()) + + +def snapshot_project(parent: ProjectAtDB, snapshot_label: str): + + if is_parametrized_project(parent): + raise NotImplementedError( + "Only non-parametrized projects can be snapshot right now" + ) + + project, nodes_map = clone_project_document( + parent.dict(), + forced_copy_project_id=str(uuid3(namespace=parent.uuid, name=snapshot_label)), + ) + + assert nodes_map # nosec + + snapshot = Snapshot( + id, label=snapshot_label, parent_id=parent.id, project_id=project.id + ) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/parametrization_models.py index 7167e18fccf..e9d13bf2391 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_models.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_models.py @@ -1,8 +1,66 @@ -from typing import Union +from datetime import datetime +from typing import Callable, Optional, Union +from uuid import UUID -from pydantic import BaseModel +from models_library.projects_nodes import OutputID +from pydantic import ( + AnyUrl, + BaseModel, + Field, + PositiveInt, + StrictBool, + StrictFloat, + StrictInt, +) + +BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] class Parameter(BaseModel): name: str - value: Union[bool, float, str] + value: BuiltinTypes + + node_id: UUID + output_id: OutputID + + +class Snapshot(BaseModel): + id: PositiveInt = Field(..., description="Unique snapshot identifier") + label: Optional[str] = Field(None, description="Unique human readable display name") + created_at: datetime = Field( + default_factory=datetime.utcnow, + description="Timestamp of the time snapshot was taken from parent. Notice that parent might change with time", + ) + + parent_id: UUID = Field(..., description="Parent's project uuid") + project_id: UUID = Field(..., description="Current project's uuid") + + +class ParameterApiModel(Parameter): + url: AnyUrl + # url_output: AnyUrl + + +class SnapshotApiModel(Snapshot): + url: AnyUrl + url_parent: AnyUrl + url_project: AnyUrl + url_parameters: Optional[AnyUrl] = None + + @classmethod + def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiModel": + return cls( + url=url_for( + "get_snapshot", + project_id=snapshot.project_id, + snapshot_id=snapshot.id, + ), + url_parent=url_for("get_project", project_id=snapshot.parent_id), + url_project=url_for("get_project", project_id=snapshot.project_id), + url_parameters=url_for( + "get_snapshot_parameters", + project_id=snapshot.parent_id, + snapshot_id=snapshot.id, + ), + **snapshot.dict(), + ) From d74aec3d569fca2a8ce12a319dfe09b3febdc98e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 14 Jul 2021 17:03:24 +0200 Subject: [PATCH 029/137] WIP --- .../parametrization_api_handlers.py | 3 + .../isolated/test_parametrization_core.py | 48 --------- .../with_dbs/11/test_parametrization_core.py | 100 ++++++++++++++++++ 3 files changed, 103 insertions(+), 48 deletions(-) delete mode 100644 services/web/server/tests/unit/isolated/test_parametrization_core.py create mode 100644 services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index c4614d1dadd..6d6dc3e5b29 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -125,6 +125,9 @@ async def _get_snapshot_parameters_handler( return params +# API ROUTES HANDLERS --------------------------------------------------------- + + @validate_arguments async def list_snapshots(project_id: UUID) -> List[Dict[str, Any]]: # project_id is param-project? diff --git a/services/web/server/tests/unit/isolated/test_parametrization_core.py b/services/web/server/tests/unit/isolated/test_parametrization_core.py deleted file mode 100644 index ca759ba987e..00000000000 --- a/services/web/server/tests/unit/isolated/test_parametrization_core.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Iterator, Tuple -from uuid import uuid4 - -import pytest -from models_library.projects import Project, ProjectAtDB -from models_library.projects_nodes import Node -from models_library.projects_nodes_io import NodeID -from simcore_service_webserver.projects.projects_api import get_project_for_user - - -# is parametrized project? -def is_param(node: Node): - return node.key.split("/")[-2] == "param" - - -def is_parametrized_project(project: Project): - return any(is_param(node) for node in project.workbench.values()) - - -def iter_params(project: Project) -> Iterator[Tuple[NodeID, Node]]: - for node_id, node in project.workbench.items(): - if is_param(node): - yield NodeID(node_id), node - - -def is_const_param(node: Node) -> bool: - return is_param(node) and "const" in node.key and not node.inputs - - -@pytest.mark.skip(reason="UNDER DEV") -async def test_it(app): - - project_id = str(uuid4()) - user_id = 0 - - prj_dict = await get_project_for_user( - app, project_id, user_id, include_templates=False, include_state=True - ) - - parent_project = Project.parse_obj(prj_dict) - - # three types of "parametrization" nodes - - # const -> replace in connecting nodes - - # generators -> iterable data sources (no inputs connected!) - - # feedback -> parametrizations with connected inputs! diff --git a/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py b/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py new file mode 100644 index 00000000000..a1060d1b51c --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py @@ -0,0 +1,100 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from typing import Dict, Iterator, Tuple +from uuid import UUID, uuid4 + +import pytest +from aiohttp import web +from models_library.projects import Project, ProjectAtDB +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID +from simcore_service_webserver.constants import APP_PROJECT_DBAPI +from simcore_service_webserver.parametrization_core import snapshot_project +from simcore_service_webserver.projects.projects_api import get_project_for_user +from simcore_service_webserver.projects.projects_db import APP_PROJECT_DBAPI +from simcore_service_webserver.projects.projects_utils import clone_project_document + + +# is parametrized project? +def is_param(node: Node): + return node.key.split("/")[-2] == "param" + + +def is_parametrized_project(project: Project): + return any(is_param(node) for node in project.workbench.values()) + + +def iter_params(project: Project) -> Iterator[Tuple[NodeID, Node]]: + for node_id, node in project.workbench.items(): + if is_param(node): + yield NodeID(node_id), node + + +def is_const_param(node: Node) -> bool: + return is_param(node) and "const" in node.key and not node.inputs + + +@pytest.fixture() +def app(): + pass + + +@pytest.fixture +def project_id() -> UUID: + return uuid4() + + +@pytest.fixture +def user_id() -> int: + return 1 + + +async def test_snapshot_standard_project( + app: web.Application, project_id: UUID, user_id: int +): + + prj_dict: Dict = await get_project_for_user( + app, str(project_id), user_id, include_templates=False, include_state=False + ) + + # validates API project data + project = Project.parse_obj(prj_dict) + + cloned_prj_dict = clone_project_document(prj_dict) + + cloned_project = Project.parse_obj(cloned_prj_dict) + + # project_snapshot = snapshot_project(standard_project) + # assert isinstance(project_snapshot, Project) + assert cloned_project.uuid != project.uuid + + projects_repo = app[APP_PROJECT_DBAPI] + + new_project = await projects_repo.replace_user_project( + new_project, user_id, project_uuid, include_templates=True + ) + + # have same pipeline but different node uuids + # assert project_snapshot. + + # removes states: snapshots is always un-run?? + + # three types of "parametrization" nodes + + # const -> replace in connecting nodes + + # generators -> iterable data sources (no inputs connected!) + + # feedback -> parametrizations with connected inputs! + + +# +# - I have a project +# - Take a snapshot +# - if the project is standard -> clone +# - if the project is parametrized +# - clone parametrized? +# - evaluate param-nodes? +# From 879ae75e15272cdddb0df0c45a1cac2190d95b35 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:30:06 +0200 Subject: [PATCH 030/137] minor --- .../src/simcore_postgres_database/models/snapshots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index cb551201d79..da3ab54e97c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -26,7 +26,7 @@ sa.String, nullable=False, unique=True, - doc="Parent project's UUID", + doc="UUID of the parent project", ), sa.Column( "project_uuid", From 76c0508f96954f76705513873de010194f14bc4f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 15:50:22 +0200 Subject: [PATCH 031/137] cleanup fixtures --- packages/postgres-database/tests/conftest.py | 16 ++++++++++------ .../tests/test_delete_projects_and_users.py | 15 ++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/postgres-database/tests/conftest.py b/packages/postgres-database/tests/conftest.py index 97876a06453..0c1b8571acd 100644 --- a/packages/postgres-database/tests/conftest.py +++ b/packages/postgres-database/tests/conftest.py @@ -42,7 +42,7 @@ def postgres_service(docker_services, docker_ip, docker_compose_file) -> str: def make_engine(postgres_service: str) -> Callable: dsn = postgres_service - def maker(is_async=True) -> Union[Coroutine, Callable]: + def maker(*, is_async=True) -> Union[Coroutine, Callable]: return aiopg.sa.create_engine(dsn) if is_async else sa.create_engine(dsn) return maker @@ -68,18 +68,22 @@ def db_metadata(): @pytest.fixture async def pg_engine(loop, make_engine, db_metadata) -> Engine: - engine = await make_engine() + async_engine = await make_engine(is_async=True) # TODO: upgrade/downgrade - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) + # NOTE: ALL is deleted before db_metadata.drop_all(sync_engine) db_metadata.create_all(sync_engine) - yield engine + yield async_engine - engine.terminate() - await engine.wait_closed() + # closes async-engine connections and terminates + async_engine.close() + await async_engine.wait_closed() + async_engine.terminate() + # NOTE: ALL is deleted after db_metadata.drop_all(sync_engine) sync_engine.dispose() diff --git a/packages/postgres-database/tests/test_delete_projects_and_users.py b/packages/postgres-database/tests/test_delete_projects_and_users.py index 4b69974fbfe..4cf85bbfa0b 100644 --- a/packages/postgres-database/tests/test_delete_projects_and_users.py +++ b/packages/postgres-database/tests/test_delete_projects_and_users.py @@ -7,21 +7,17 @@ import pytest import sqlalchemy as sa +from aiopg.sa.engine import Engine from aiopg.sa.result import ResultProxy, RowProxy from psycopg2.errors import ForeignKeyViolation # pylint: disable=no-name-in-module from pytest_simcore.helpers.rawdata_fakers import random_project, random_user -from simcore_postgres_database.models.base import metadata from simcore_postgres_database.webserver_models import projects, users @pytest.fixture -async def engine(make_engine, loop): - engine = await make_engine() - sync_engine = make_engine(False) - metadata.drop_all(sync_engine) - metadata.create_all(sync_engine) +async def engine(pg_engine: Engine): - async with engine.acquire() as conn: + async with pg_engine.acquire() as conn: await conn.execute(users.insert().values(**random_user(name="A"))) await conn.execute(users.insert().values(**random_user())) await conn.execute(users.insert().values(**random_user())) @@ -32,10 +28,7 @@ async def engine(make_engine, loop): with pytest.raises(ForeignKeyViolation): await conn.execute(projects.insert().values(**random_project(prj_owner=4))) - yield engine - - engine.close() - await engine.wait_closed() + yield pg_engine @pytest.mark.skip(reason="sandbox for dev purposes") From 6b01cad5a9138cece7dac73d481522b4c33fcc11 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 15:53:29 +0200 Subject: [PATCH 032/137] cleanup fixtures --- packages/postgres-database/tests/test_groups.py | 8 ++++---- .../tests/test_uniqueness_in_comp_tasks.py | 2 +- services/api-server/tests/unit/conftest.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/postgres-database/tests/test_groups.py b/packages/postgres-database/tests/test_groups.py index 68854b0d6a0..e7ac3975be7 100644 --- a/packages/postgres-database/tests/test_groups.py +++ b/packages/postgres-database/tests/test_groups.py @@ -41,7 +41,7 @@ async def _create_user(conn, name: str, group: RowProxy) -> RowProxy: async def test_user_group_uniqueness(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) @@ -68,7 +68,7 @@ async def test_user_group_uniqueness(make_engine): async def test_all_group(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) async with engine.acquire() as conn: @@ -119,7 +119,7 @@ async def test_all_group(make_engine): async def test_own_group(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) async with engine.acquire() as conn: @@ -165,7 +165,7 @@ async def test_own_group(make_engine): async def test_group(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) async with engine.acquire() as conn: diff --git a/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py b/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py index 25a7fd228a0..89e3d5c5573 100644 --- a/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py +++ b/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py @@ -19,7 +19,7 @@ async def engine(loop, make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index af7c57c404f..38933ad5c05 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -174,7 +174,7 @@ def is_postgres_responsive() -> bool: def make_engine(postgres_service: Dict) -> Callable: dsn = postgres_service["dsn"] # session scope freezes dsn - def maker(is_async=True) -> Union[aiopg_sa_engine.Engine, sa_engine.Engine]: + def maker(*, is_async=True) -> Union[aiopg_sa_engine.Engine, sa_engine.Engine]: if is_async: return aiopg.sa.create_engine(dsn) return sa.create_engine(dsn) @@ -198,7 +198,7 @@ def apply_migration(postgres_service: Dict, make_engine) -> Iterator[None]: pg_cli.downgrade.callback("base") pg_cli.clean.callback() # FIXME: deletes all because downgrade is not reliable! - engine = make_engine(False) + engine = make_engine(is_async=False) metadata.drop_all(engine) From 996836222791416b9179fd0d4046d7711fe96d54 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 18:47:15 +0200 Subject: [PATCH 033/137] cleanup and minor fix --- packages/models-library/src/models_library/projects.py | 8 ++++++-- packages/models-library/src/models_library/services.py | 2 -- .../tests/test_delete_projects_and_users.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 2d960b77af6..667382097d7 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -66,17 +66,21 @@ class ProjectCommons(BaseModel): @validator("thumbnail", always=True, pre=True) @classmethod - def convert_empty_str_to_none(v): + def convert_empty_str_to_none(cls, v): if isinstance(v, str) and v == "": return None return v class ProjectAtDB(ProjectCommons): - # specific DB fields + # Model used to READ from database + id: int = Field(..., description="The table primary index") + project_type: ProjectType = Field(..., alias="type", description="The project type") + prj_owner: Optional[int] = Field(..., description="The project owner id") + published: Optional[bool] = Field( False, description="Defines if a study is available publicly" ) diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 205f2ecd64d..84bd5a2b811 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -283,14 +283,12 @@ class ServiceKeyVersion(BaseModel): "simcore/services/comp/itis/sleeper", "simcore/services/dynamic/3dviewer", ], - regex=KEY_RE, ) version: str = Field( ..., description="service version number", regex=VERSION_RE, examples=["1.0.0", "0.0.1"], - regex=VERSION_RE, ) diff --git a/packages/postgres-database/tests/test_delete_projects_and_users.py b/packages/postgres-database/tests/test_delete_projects_and_users.py index 4cf85bbfa0b..cddcac6c0e2 100644 --- a/packages/postgres-database/tests/test_delete_projects_and_users.py +++ b/packages/postgres-database/tests/test_delete_projects_and_users.py @@ -1,7 +1,7 @@ # pylint: disable=no-value-for-parameter -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable from typing import List From 39a688b718463b010dc0b684d21711bfaed5bbcf Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 21:52:12 +0200 Subject: [PATCH 034/137] adds snapshot table with minimal test --- .../models/snapshots.py | 22 ++- .../postgres-database/tests/test_snapshots.py | 141 ++++++++++++++++++ .../pytest_simcore/helpers/rawdata_fakers.py | 1 + 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 packages/postgres-database/tests/test_snapshots.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index da3ab54e97c..8cce020e2f5 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -24,15 +24,35 @@ sa.Column( "parent_uuid", sa.String, + sa.ForeignKey( + "projects.uuid", + name="fk_snapshots_parent_uuid_projects", + ondelete="CASCADE", + ), nullable=False, - unique=True, doc="UUID of the parent project", ), + sa.Column( + "child_index", + sa.Integer, + nullable=False, + unique=True, + doc="Number of child (as 0-based index: 0 being the oldest, 1, ...)" + "from the same parent_id", + ), sa.Column( "project_uuid", sa.String, + sa.ForeignKey( + "projects.uuid", + name="fk_snapshots_project_uuid_projects", + ondelete="CASCADE", + ), nullable=False, unique=True, doc="UUID of the project associated to this snapshot", ), ) + + +# Snapshot : convert_to_pydantic(snapshot) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py new file mode 100644 index 00000000000..e03a8d889a8 --- /dev/null +++ b/packages/postgres-database/tests/test_snapshots.py @@ -0,0 +1,141 @@ +# pylint: disable=no-value-for-parameter +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from copy import deepcopy +from typing import Optional +from uuid import UUID, uuid3 + +import pytest +from aiopg.sa.engine import Engine +from aiopg.sa.result import ResultProxy, RowProxy +from pytest_simcore.helpers.rawdata_fakers import random_project, random_user +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.snapshots import snapshots +from simcore_postgres_database.models.users import users + + +@pytest.fixture +async def engine(pg_engine: Engine): + # injects + async with pg_engine.acquire() as conn: + # a 'me' user + user_id = await conn.scalar( + users.insert().values(**random_user(name="me")).returning(users.c.id) + ) + # has a project 'parent' + await conn.execute( + projects.insert().values(**random_project(prj_owner=user_id, name="parent")) + ) + yield pg_engine + + +async def test_creating_snapshots(engine: Engine): + async def _create_snapshot(child_index: int, parent_prj, conn) -> int: + # copy + # change uuid, and set to invisible + exclude = { + "id", + "uuid", + "creation_date", + "last_change_date", + "hidden", + "published", + } + prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} + + prj_dict["name"] += f" [snapshot {child_index}]" + prj_dict["uuid"] = uuid3(UUID(parent_prj.uuid), f"snapshot.{child_index}") + prj_dict[ + "creation_date" + ] = parent_prj.last_change_date # state of parent upon copy! + prj_dict["hidden"] = True + prj_dict["published"] = False + + # NOTE: a snapshot has no results but workbench stores some states, + # - input hashes + # - node ids + + # + # Define policies about changes in parent project and + # how it influence children + # + project_uuid: str = await conn.scalar( + projects.insert().values(**prj_dict).returning(projects.c.uuid) + ) + + assert UUID(project_uuid) == prj_dict["uuid"] + + # create snapshot + snapshot_id = await conn.scalar( + snapshots.insert() + .values( + name=f"Snapshot {child_index}", + parent_uuid=parent_prj.uuid, + child_index=child_index, + project_uuid=project_uuid, + ) + .returning(snapshots.c.id) + ) + return snapshot_id + + async with engine.acquire() as conn: + + # get parent + res: ResultProxy = await conn.execute( + projects.select().where(projects.c.name == "parent") + ) + parent_prj: Optional[RowProxy] = await res.first() + + assert parent_prj + + # take one snapshot + snapshot_one_id = await _create_snapshot(0, parent_prj, conn) + + # modify parent + updated_parent_prj = await ( + await conn.execute( + projects.update() + .values(description="foo") + .where(projects.c.id == parent_prj.id) + .returning(projects) + ) + ).first() + + assert updated_parent_prj + + assert updated_parent_prj.id == parent_prj.id + assert updated_parent_prj.description != parent_prj.description + + # take another snapshot + snapshot_two_id = await _create_snapshot(1, updated_parent_prj, conn) + + assert snapshot_one_id != snapshot_two_id + + # get project corresponding to snapshot 1 + selected_snapshot_project = await ( + await conn.execute( + projects.select().where(snapshots.c.id == snapshot_two_id) + ) + ).first() + + assert selected_snapshot_project + assert selected_snapshot_project.description == updated_parent_prj.description + + assert selected_snapshot_project.tuple() == updated_parent_prj.tuple() + + +def test_deleting_snapshots(): + # test delete child project -> deletes snapshot + # test delete snapshot -> deletes child project + + # test delete parent project -> deletes snapshots + # test delete snapshot does NOT delete parent + pass + + +def test_create_pydantic_models_from_sqlalchemy_tables(): + # SEE https://docs.sqlalchemy.org/en/14/core/metadata.html + # SEE https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py + pass diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index e7681d078d7..fe6e860b380 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -51,6 +51,7 @@ def random_project(**overrides) -> Dict[str, Any]: name=fake.word(), description=fake.sentence(), prj_owner=fake.pyint(), + thumbnail=fake.image_url(width=120, height=120), access_rights={}, workbench={}, published=False, From 89481e6d4ab4e604d02af8e82f9522192ef1e047 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 21:52:22 +0200 Subject: [PATCH 035/137] minor --- .../parametrization_api_handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index 6d6dc3e5b29..5b427e9cff3 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -9,7 +9,7 @@ from pydantic.error_wrappers import ValidationError from simcore_service_webserver.parametrization_models import Snapshot -from ._meta import api_version_prefix +from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from .parametrization_models import Snapshot, SnapshotApiModel @@ -46,7 +46,7 @@ async def wrapped(request: web.Request): @routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + f"/{vtag}/projects/{{project_id}}/snapshots", name="_list_snapshots_handler", ) @handle_request_errors @@ -77,7 +77,7 @@ async def _list_snapshots_handler(request: web.Request): @routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}", + f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}", name="_get_snapshot_handler", ) @handle_request_errors @@ -92,7 +92,7 @@ async def _get_snapshot_handler(request: web.Request): @routes.post( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + f"/{vtag}/projects/{{project_id}}/snapshots", name="_create_snapshot_handler", ) @handle_request_errors @@ -108,7 +108,7 @@ async def _create_snapshot_handler(request: web.Request): @routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", + f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", name="_get_snapshot_parameters_handler", ) @handle_request_errors From bbd113b18362e1b29583ddece5fa833d9dfa7606 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 16 Jul 2021 17:16:53 +0200 Subject: [PATCH 036/137] exploratory tests for snapshots --- .../postgres-database/tests/test_snapshots.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index e03a8d889a8..714f57361c0 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -32,17 +32,18 @@ async def engine(pg_engine: Engine): async def test_creating_snapshots(engine: Engine): + exclude = { + "id", + "uuid", + "creation_date", + "last_change_date", + "hidden", + "published", + } + async def _create_snapshot(child_index: int, parent_prj, conn) -> int: # copy # change uuid, and set to invisible - exclude = { - "id", - "uuid", - "creation_date", - "last_change_date", - "hidden", - "published", - } prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} prj_dict["name"] += f" [snapshot {child_index}]" @@ -114,16 +115,22 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert snapshot_one_id != snapshot_two_id # get project corresponding to snapshot 1 + j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) selected_snapshot_project = await ( await conn.execute( - projects.select().where(snapshots.c.id == snapshot_two_id) + projects.select() + .select_from(j) + .where(snapshots.c.id == snapshot_two_id) ) ).first() assert selected_snapshot_project assert selected_snapshot_project.description == updated_parent_prj.description - assert selected_snapshot_project.tuple() == updated_parent_prj.tuple() + def extract(t): + return {k: t[k] for k in t if k not in exclude.union({"name"})} + + assert extract(selected_snapshot_project) == extract(updated_parent_prj) def test_deleting_snapshots(): From f8475ad8e746d4ce7829942670114eb6db6d79be Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 16 Jul 2021 19:09:12 +0200 Subject: [PATCH 037/137] table to pydantic models --- .../utils_pydantic.py | 43 +++++++++++++++++++ .../postgres-database/tests/test_snapshots.py | 1 - .../tests/test_utils_pydantic.py | 15 +++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py create mode 100644 packages/postgres-database/tests/test_utils_pydantic.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py new file mode 100644 index 00000000000..d9d0d06685b --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py @@ -0,0 +1,43 @@ +from typing import Container, Optional, Type + +import sqlalchemy as sa +from pydantic import BaseConfig, BaseModel, Field, create_model + + +class OrmConfig(BaseConfig): + orm_mode = True + + +def sa_table_to_pydantic_model( + table: sa.Table, *, config: Type = OrmConfig, exclude: Container[str] = [] +) -> Type[BaseModel]: + # NOTE: basically copied from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py + fields = {} + + for column in table.columns: + name = str(column.name) + if name in exclude: + continue + + python_type: Optional[type] = None + if hasattr(column.type, "impl"): + if hasattr(column.type.impl, "python_type"): + python_type = column.type.impl.python_type + elif hasattr(column.type, "python_type"): + python_type = column.type.python_type + assert python_type, f"Could not infer python_type for {column}" # nosec + default = None + if column.default is None and not column.nullable: + default = ... + + if hasattr(column, "doc") and column.doc: + default = Field(default, description=column.doc) + + fields[name] = (python_type, default) + + # create domain models from db-schemas + pydantic_model = create_model( + table.name.capitalize(), __config__=config, **fields # type: ignore + ) + assert issubclass(pydantic_model, BaseModel) # nosec + return pydantic_model diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 714f57361c0..1c19d3aaf78 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -125,7 +125,6 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: ).first() assert selected_snapshot_project - assert selected_snapshot_project.description == updated_parent_prj.description def extract(t): return {k: t[k] for k in t if k not in exclude.union({"name"})} diff --git a/packages/postgres-database/tests/test_utils_pydantic.py b/packages/postgres-database/tests/test_utils_pydantic.py new file mode 100644 index 00000000000..41bb8627840 --- /dev/null +++ b/packages/postgres-database/tests/test_utils_pydantic.py @@ -0,0 +1,15 @@ +import pytest +from pydantic import BaseModel +from simcore_postgres_database.models import * +from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.utils_pydantic import sa_table_to_pydantic_model + + +@pytest.mark.parametrized("table_cls", metadata.tables) +def test_table_to_pydantic_models(table_cls): + + PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls, exclude={}) + assert issubclass(PydanticModelAtDB, BaseModel) + print(PydanticModelAtDB.schema_json(indent=2)) + + # TODO: create fakes automatically? From b0c83533c90ebce97cb08f1a3dad112150c53b97 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 11:56:27 +0200 Subject: [PATCH 038/137] cleanup pydantic factory --- ...ic.py => utils_pydantic_models_factory.py} | 7 +++- .../tests/test_utils_pydantic.py | 15 -------- .../test_utils_pydantic_models_factory.py | 34 +++++++++++++++++++ 3 files changed, 40 insertions(+), 16 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/{utils_pydantic.py => utils_pydantic_models_factory.py} (91%) delete mode 100644 packages/postgres-database/tests/test_utils_pydantic.py create mode 100644 packages/postgres-database/tests/test_utils_pydantic_models_factory.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py similarity index 91% rename from packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py rename to packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py index d9d0d06685b..3ee7dd16009 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py @@ -9,10 +9,15 @@ class OrmConfig(BaseConfig): def sa_table_to_pydantic_model( - table: sa.Table, *, config: Type = OrmConfig, exclude: Container[str] = [] + table: sa.Table, + *, + config: Type = OrmConfig, + exclude: Optional[Container[str]] = None, ) -> Type[BaseModel]: + # NOTE: basically copied from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py fields = {} + exclude = exclude or [] for column in table.columns: name = str(column.name) diff --git a/packages/postgres-database/tests/test_utils_pydantic.py b/packages/postgres-database/tests/test_utils_pydantic.py deleted file mode 100644 index 41bb8627840..00000000000 --- a/packages/postgres-database/tests/test_utils_pydantic.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest -from pydantic import BaseModel -from simcore_postgres_database.models import * -from simcore_postgres_database.models.base import metadata -from simcore_postgres_database.utils_pydantic import sa_table_to_pydantic_model - - -@pytest.mark.parametrized("table_cls", metadata.tables) -def test_table_to_pydantic_models(table_cls): - - PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls, exclude={}) - assert issubclass(PydanticModelAtDB, BaseModel) - print(PydanticModelAtDB.schema_json(indent=2)) - - # TODO: create fakes automatically? diff --git a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py new file mode 100644 index 00000000000..8eb31b7cb47 --- /dev/null +++ b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py @@ -0,0 +1,34 @@ +from datetime import datetime +from uuid import uuid4 + +import pytest +from pydantic import BaseModel +from simcore_postgres_database.models import * +from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.models.snapshots import snapshots +from simcore_postgres_database.utils_pydantic_models_factory import ( + sa_table_to_pydantic_model, +) + + +@pytest.mark.parametrized("table_cls", metadata.tables) +def test_table_to_pydantic_models(table_cls): + + PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) + assert issubclass(PydanticModelAtDB, BaseModel) + print(PydanticModelAtDB.schema_json(indent=2)) + + # TODO: create fakes automatically? + + +def test_snapshot_pydantic_model(): + Snapshot = sa_table_to_pydantic_model(snapshots) + + snapshot = Snapshot( + id=0, + created_at=datetime.now(), + parent_uuid=uuid4(), + child_index=2, + project_uuid=uuid4(), + ) + assert snapshot.id == 0 From c46ec2c0598e96ce560551db8250920a768bd342 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 12:01:43 +0200 Subject: [PATCH 039/137] adds pydantic dep --- packages/postgres-database/requirements/_pydantic.in | 1 + packages/postgres-database/requirements/_pydantic.txt | 10 ++++++++++ packages/postgres-database/requirements/_test.in | 1 + packages/postgres-database/requirements/dev.txt | 1 + packages/postgres-database/setup.py | 8 ++++++-- 5 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 packages/postgres-database/requirements/_pydantic.in create mode 100644 packages/postgres-database/requirements/_pydantic.txt diff --git a/packages/postgres-database/requirements/_pydantic.in b/packages/postgres-database/requirements/_pydantic.in new file mode 100644 index 00000000000..572b352f30e --- /dev/null +++ b/packages/postgres-database/requirements/_pydantic.in @@ -0,0 +1 @@ +pydantic diff --git a/packages/postgres-database/requirements/_pydantic.txt b/packages/postgres-database/requirements/_pydantic.txt new file mode 100644 index 00000000000..6605adc0dff --- /dev/null +++ b/packages/postgres-database/requirements/_pydantic.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile --output-file=requirements/_pydantic.txt --strip-extras requirements/_pydantic.in +# +pydantic==1.8.2 + # via -r requirements/_pydantic.in +typing-extensions==3.10.0.0 + # via pydantic diff --git a/packages/postgres-database/requirements/_test.in b/packages/postgres-database/requirements/_test.in index c76e9d978cf..67ad0d9fe0e 100644 --- a/packages/postgres-database/requirements/_test.in +++ b/packages/postgres-database/requirements/_test.in @@ -8,6 +8,7 @@ # --constraint _base.txt --constraint _migration.txt +--constraint _pydantic.txt # fixtures pyyaml diff --git a/packages/postgres-database/requirements/dev.txt b/packages/postgres-database/requirements/dev.txt index 8136f1a48b5..d7d03a8f861 100644 --- a/packages/postgres-database/requirements/dev.txt +++ b/packages/postgres-database/requirements/dev.txt @@ -9,6 +9,7 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt +--requirement _pydantic.txt --requirement _test.txt --requirement _tools.txt diff --git a/packages/postgres-database/setup.py b/packages/postgres-database/setup.py index b5bda1bdc81..4780ae5ca50 100644 --- a/packages/postgres-database/setup.py +++ b/packages/postgres-database/setup.py @@ -21,7 +21,7 @@ def read_reqs(reqs_path: Path): # Strong dependencies migration_requirements = read_reqs(current_dir / "requirements" / "_migration.in") test_requirements = read_reqs(current_dir / "requirements" / "_test.txt") - +pydantic_requirements = read_reqs(current_dir / "requirements" / "_pydantic.in") setup( name="simcore-postgres-database", @@ -43,7 +43,11 @@ def read_reqs(reqs_path: Path): test_suite="tests", install_requires=install_requirements, tests_require=test_requirements, - extras_require={"migration": migration_requirements, "test": test_requirements}, + extras_require={ + "migration": migration_requirements, + "test": test_requirements, + "pydantic": pydantic_requirements, + }, include_package_data=True, package_data={ "": [ From b89dac09ac19ec74c56e87bd48f2add80ae7fea2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:01:19 +0200 Subject: [PATCH 040/137] pydantic models factory --- .../postgres-database/requirements/ci.txt | 1 + .../utils_pydantic_models_factory.py | 19 +++++++++++++++++++ .../test_utils_pydantic_models_factory.py | 9 ++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/postgres-database/requirements/ci.txt b/packages/postgres-database/requirements/ci.txt index b12bf394c2e..8fc611bc091 100644 --- a/packages/postgres-database/requirements/ci.txt +++ b/packages/postgres-database/requirements/ci.txt @@ -9,6 +9,7 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt +--requirement _pydantic.txt --requirement _test.txt # installs this repo's packages diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py index 3ee7dd16009..2b51da22ab5 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py @@ -1,4 +1,5 @@ from typing import Container, Optional, Type +from uuid import UUID import sqlalchemy as sa from pydantic import BaseConfig, BaseModel, Field, create_model @@ -8,6 +9,12 @@ class OrmConfig(BaseConfig): orm_mode = True +RESERVED = { + "schema", +} +# e.g. Field name "schema" shadows a BaseModel attribute; use a different field name with "alias='schema'". + + def sa_table_to_pydantic_model( table: sa.Table, *, @@ -21,6 +28,10 @@ def sa_table_to_pydantic_model( for column in table.columns: name = str(column.name) + + if name in RESERVED: + name = f"{table.name.lower()}_{name}" + if name in exclude: continue @@ -30,11 +41,19 @@ def sa_table_to_pydantic_model( python_type = column.type.impl.python_type elif hasattr(column.type, "python_type"): python_type = column.type.python_type + assert python_type, f"Could not infer python_type for {column}" # nosec + default = None if column.default is None and not column.nullable: default = ... + # Policies based on naming conventions + if "uuid" in name.split("_") and python_type == str: + python_type = UUID + if isinstance(default, str): + default = UUID(default) + if hasattr(column, "doc") and column.doc: default = Field(default, description=column.doc) diff --git a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py index 8eb31b7cb47..8c984d42a90 100644 --- a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py +++ b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py @@ -3,7 +3,13 @@ import pytest from pydantic import BaseModel + +# pylint: disable=wildcard-import +# pylint: disable=unused-wildcard-import from simcore_postgres_database.models import * + +# pylint: enable=wildcard-import +# pylint: enable=unused-wildcard-import from simcore_postgres_database.models.base import metadata from simcore_postgres_database.models.snapshots import snapshots from simcore_postgres_database.utils_pydantic_models_factory import ( @@ -11,7 +17,7 @@ ) -@pytest.mark.parametrized("table_cls", metadata.tables) +@pytest.mark.parametrize("table_cls", metadata.tables.values(), ids=lambda t: t.name) def test_table_to_pydantic_models(table_cls): PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) @@ -26,6 +32,7 @@ def test_snapshot_pydantic_model(): snapshot = Snapshot( id=0, + name="foo", created_at=datetime.now(), parent_uuid=uuid4(), child_index=2, From 7290579fb4d82711a74e910362eddeb5caa7d789 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:13:36 +0200 Subject: [PATCH 041/137] WIP --- .../src/simcore_service_webserver/parametrization.py | 2 ++ .../parametrization_models.py | 10 +++++++--- .../tests/unit/isolated/test_parametrization_models.py | 7 +++++++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 services/web/server/tests/unit/isolated/test_parametrization_models.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index 00c18833ac6..725e91ef648 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -1,5 +1,7 @@ """ parametrization app module setup + Extend project's business logic by adding two new concepts, namely snapshots and parametrizations + - Project parametrization - Project snapshots diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/parametrization_models.py index e9d13bf2391..f1d3ad54885 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_models.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_models.py @@ -15,13 +15,14 @@ BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] - +## Domain models -------- class Parameter(BaseModel): name: str value: BuiltinTypes - node_id: UUID - output_id: OutputID + # TODO: same parameter in different nodes? + node_id: UUID = Field(..., description="Id of parametrized node") + output_id: OutputID = Field(..., description="Output where parameter is exposed") class Snapshot(BaseModel): @@ -36,6 +37,9 @@ class Snapshot(BaseModel): project_id: UUID = Field(..., description="Current project's uuid") +## API models ---------- + + class ParameterApiModel(Parameter): url: AnyUrl # url_output: AnyUrl diff --git a/services/web/server/tests/unit/isolated/test_parametrization_models.py b/services/web/server/tests/unit/isolated/test_parametrization_models.py new file mode 100644 index 00000000000..357899214c5 --- /dev/null +++ b/services/web/server/tests/unit/isolated/test_parametrization_models.py @@ -0,0 +1,7 @@ +from simcore_postgres_database.models.snapshots import snapshots +from simcore_service_webserver.parametrization_models import ( + Parameter, + ParameterApiModel, + Snapshot, + SnapshotApiModel, +) From 11290144130ebf2815bf399464e46a21314b75cc Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:17:10 +0200 Subject: [PATCH 042/137] cleanup pg snapshots table --- .../doc/img/postgres-database-models.svg | 1207 +++++++++-------- .../models/snapshots.py | 3 +- .../postgres-database/tests/test_snapshots.py | 2 +- 3 files changed, 677 insertions(+), 535 deletions(-) diff --git a/packages/postgres-database/doc/img/postgres-database-models.svg b/packages/postgres-database/doc/img/postgres-database-models.svg index 074985a08d2..368427dacb8 100644 --- a/packages/postgres-database/doc/img/postgres-database-models.svg +++ b/packages/postgres-database/doc/img/postgres-database-models.svg @@ -4,668 +4,809 @@ - - + + %3 - + file_meta_data - -file_meta_data - -file_uuid - [VARCHAR] - -location_id - [VARCHAR] - -location - [VARCHAR] - -bucket_name - [VARCHAR] - -object_name - [VARCHAR] - -project_id - [VARCHAR] - -project_name - [VARCHAR] - -node_id - [VARCHAR] - -node_name - [VARCHAR] - -file_name - [VARCHAR] - -user_id - [VARCHAR] - -user_name - [VARCHAR] - -file_id - [VARCHAR] - -raw_file_path - [VARCHAR] - -display_file_path - [VARCHAR] - -created_at - [VARCHAR] - -last_modified - [VARCHAR] - -file_size - [BIGINT] + +file_meta_data + +file_uuid + [VARCHAR] + +location_id + [VARCHAR] + +location + [VARCHAR] + +bucket_name + [VARCHAR] + +object_name + [VARCHAR] + +project_id + [VARCHAR] + +project_name + [VARCHAR] + +node_id + [VARCHAR] + +node_name + [VARCHAR] + +file_name + [VARCHAR] + +user_id + [VARCHAR] + +user_name + [VARCHAR] + +file_id + [VARCHAR] + +raw_file_path + [VARCHAR] + +display_file_path + [VARCHAR] + +created_at + [VARCHAR] + +last_modified + [VARCHAR] + +file_size + [BIGINT] + +entity_tag + [VARCHAR] + +is_soft_link + [BOOLEAN] groups - -groups - -gid - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -type - [VARCHAR(8)] - -thumbnail - [VARCHAR] - -inclusion_rules - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] + +groups + +gid + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +type + [VARCHAR(8)] + +thumbnail + [VARCHAR] + +inclusion_rules + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] user_to_groups - -user_to_groups - -uid - [BIGINT] - -gid - [BIGINT] - -access_rights - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] + +user_to_groups + +uid + [BIGINT] + +gid + [BIGINT] + +access_rights + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] groups--user_to_groups - -{0,1} -0..N + +{0,1} +0..N users - -users - -id - [BIGINT] - -name - [VARCHAR] - -email - [VARCHAR] - -password_hash - [VARCHAR] - -primary_gid - [BIGINT] - -status - [VARCHAR(20)] - -role - [VARCHAR(9)] - -created_at - [DATETIME] - -modified - [DATETIME] - -created_ip - [VARCHAR] + +users + +id + [BIGINT] + +name + [VARCHAR] + +email + [VARCHAR] + +password_hash + [VARCHAR] + +primary_gid + [BIGINT] + +status + [VARCHAR(20)] + +role + [VARCHAR(9)] + +created_at + [DATETIME] + +modified + [DATETIME] + +created_ip + [VARCHAR] groups--users - -{0,1} -0..N + +{0,1} +0..N group_classifiers - -group_classifiers - -id - [BIGINT] - -bundle - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] - -gid - [BIGINT] - -uses_scicrunch - [BOOLEAN] + +group_classifiers + +id + [BIGINT] + +bundle + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] + +gid + [BIGINT] + +uses_scicrunch + [BOOLEAN] groups--group_classifiers - -{0,1} -0..N + +{0,1} +0..N services_meta_data - -services_meta_data - -key - [VARCHAR] - -version - [VARCHAR] - -owner - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -thumbnail - [VARCHAR] - -classifiers - [VARCHAR[]] - -created - [DATETIME] - -modified - [DATETIME] - -metadata - [JSONB] + +services_meta_data + +key + [VARCHAR] + +version + [VARCHAR] + +owner + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +thumbnail + [VARCHAR] + +classifiers + [VARCHAR[]] + +created + [DATETIME] + +modified + [DATETIME] + +quality + [JSONB] groups--services_meta_data - -{0,1} -0..N + +{0,1} +0..N services_access_rights - -services_access_rights - -key - [VARCHAR] - -version - [VARCHAR] - -gid - [BIGINT] - -execute_access - [BOOLEAN] - -write_access - [BOOLEAN] - -product_name - [VARCHAR] - -created - [DATETIME] - -modified - [DATETIME] + +services_access_rights + +key + [VARCHAR] + +version + [VARCHAR] + +gid + [BIGINT] + +execute_access + [BOOLEAN] + +write_access + [BOOLEAN] + +product_name + [VARCHAR] + +created + [DATETIME] + +modified + [DATETIME] groups--services_access_rights - -{0,1} -0..N + +{0,1} +0..N - + users--user_to_groups - -{0,1} -0..N + +{0,1} +0..N projects - -projects - -id - [BIGINT] - -type - [VARCHAR(8)] - -uuid - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -thumbnail - [VARCHAR] - -prj_owner - [BIGINT] - -creation_date - [DATETIME] - -last_change_date - [DATETIME] - -access_rights - [JSONB] - -workbench - [Null] - -ui - [JSONB] - -classifiers - [VARCHAR[]] - -dev - [JSONB] - -published - [BOOLEAN] + +projects + +id + [BIGINT] + +type + [VARCHAR(8)] + +uuid + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +thumbnail + [VARCHAR] + +prj_owner + [BIGINT] + +creation_date + [DATETIME] + +last_change_date + [DATETIME] + +access_rights + [JSONB] + +workbench + [Null] + +ui + [JSONB] + +classifiers + [VARCHAR[]] + +dev + [JSONB] + +quality + [JSONB] + +published + [BOOLEAN] + +hidden + [BOOLEAN] - + users--projects - -{0,1} -0..N + +{0,1} +0..N + + + +comp_runs + +comp_runs + +run_id + [BIGINT] + +project_uuid + [VARCHAR] + +user_id + [BIGINT] + +iteration + [BIGINT] + +result + [VARCHAR(11)] + +created + [DATETIME] + +modified + [DATETIME] + +started + [DATETIME] + +ended + [DATETIME] + + + +users--comp_runs + +{0,1} +0..N - + user_to_projects - -user_to_projects - -id - [BIGINT] - -user_id - [BIGINT] - -project_id - [BIGINT] + +user_to_projects + +id + [BIGINT] + +user_id + [BIGINT] + +project_id + [BIGINT] - + users--user_to_projects - -{0,1} -0..N + +{0,1} +0..N - + tokens - -tokens - -token_id - [BIGINT] - -user_id - [BIGINT] - -token_service - [VARCHAR] - -token_data - [Null] + +tokens + +token_id + [BIGINT] + +user_id + [BIGINT] + +token_service + [VARCHAR] + +token_data + [Null] - + users--tokens - -{0,1} -0..N + +{0,1} +0..N - + api_keys - -api_keys - -id - [BIGINT] - -display_name - [VARCHAR] - -user_id - [BIGINT] - -api_key - [VARCHAR] - -api_secret - [VARCHAR] + +api_keys + +id + [BIGINT] + +display_name + [VARCHAR] + +user_id + [BIGINT] + +api_key + [VARCHAR] + +api_secret + [VARCHAR] - + users--api_keys - -{0,1} -0..N + +{0,1} +0..N - + confirmations - -confirmations - -code - [TEXT] - -user_id - [BIGINT] - -action - [VARCHAR(14)] - -data - [TEXT] - -created_at - [DATETIME] + +confirmations + +code + [TEXT] + +user_id + [BIGINT] + +action + [VARCHAR(14)] + +data + [TEXT] + +created_at + [DATETIME] - + users--confirmations - -{0,1} -0..N + +{0,1} +0..N - + tags - -tags - -id - [BIGINT] - -user_id - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -color - [VARCHAR] + +tags + +id + [BIGINT] + +user_id + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +color + [VARCHAR] - + users--tags - -{0,1} -0..N + +{0,1} +0..N - + services_meta_data--services_access_rights - -{0,1} -0..N + +{0,1} +0..N - + services_meta_data--services_access_rights - -{0,1} -0..N + +{0,1} +0..N + + + +services_consume_filetypes + +services_consume_filetypes + +service_key + [VARCHAR] + +service_version + [VARCHAR] + +service_display_name + [VARCHAR] + +service_input_port + [VARCHAR] + +filetype + [VARCHAR] + +preference_order + [SMALLINT] + +is_guest_allowed + [BOOLEAN] + + + +services_meta_data--services_consume_filetypes + +{0,1} +0..N + + + +services_meta_data--services_consume_filetypes + +{0,1} +0..N study_tags - -study_tags - -study_id - [BIGINT] - -tag_id - [BIGINT] + +study_tags + +study_id + [BIGINT] + +tag_id + [BIGINT] projects--study_tags - -{0,1} -0..N + +{0,1} +0..N - + + +snapshots + +snapshots + +id + [BIGINT] + +name + [VARCHAR] + +created_at + [DATETIME] + +parent_uuid + [VARCHAR] + +child_index + [INTEGER] + +project_uuid + [VARCHAR] + + +projects--snapshots + +{0,1} +0..N + + + +projects--snapshots + +{0,1} +0..N + + + +projects--comp_runs + +{0,1} +0..N + + + projects--user_to_projects - -{0,1} -0..N + +{0,1} +0..N - + tags--study_tags - -{0,1} -0..N + +{0,1} +0..N - + comp_pipeline - -comp_pipeline - -project_id - [VARCHAR] - -dag_adjacency_list - [Null] - -state - [VARCHAR(11)] + +comp_pipeline + +project_id + [VARCHAR] + +dag_adjacency_list + [Null] + +state + [VARCHAR(11)] - + comp_tasks - -comp_tasks - -task_id - [INTEGER] - -project_id - [VARCHAR] - -node_id - [VARCHAR] - -node_class - [VARCHAR(13)] - -job_id - [VARCHAR] - -internal_id - [INTEGER] - -schema - [Null] - -inputs - [Null] - -outputs - [Null] - -image - [Null] - -state - [VARCHAR(11)] - -submit - [DATETIME] - -start - [DATETIME] - -end - [DATETIME] + +comp_tasks + +task_id + [INTEGER] + +project_id + [VARCHAR] + +node_id + [VARCHAR] + +node_class + [VARCHAR(13)] + +job_id + [VARCHAR] + +internal_id + [INTEGER] + +schema + [Null] + +inputs + [Null] + +outputs + [Null] + +run_hash + [VARCHAR] + +image + [Null] + +state + [VARCHAR(11)] + +submit + [DATETIME] + +start + [DATETIME] + +end + [DATETIME] - + comp_pipeline--comp_tasks - -{0,1} -0..N + +{0,1} +0..N - + products - -products - -name - [VARCHAR] - -host_regex - [VARCHAR] - -created - [DATETIME] - -modified - [DATETIME] + +products + +name + [VARCHAR] + +host_regex + [VARCHAR] + +created + [DATETIME] + +modified + [DATETIME] - + products--services_access_rights - -{0,1} -0..N + +{0,1} +0..N - + scicrunch_resources - -scicrunch_resources - -rrid - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -creation_date - [DATETIME] - -last_change_date - [DATETIME] + +scicrunch_resources + +rrid + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +creation_date + [DATETIME] + +last_change_date + [DATETIME] - + dags - -dags - -id - [INTEGER] - -key - [VARCHAR] - -version - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -contact - [VARCHAR] - -workbench - [Null] + +dags + +id + [INTEGER] + +key + [VARCHAR] + +version + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +contact + [VARCHAR] + +workbench + [Null] diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index 8cce020e2f5..f23c80333ce 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -30,6 +30,7 @@ ondelete="CASCADE", ), nullable=False, + unique=False, doc="UUID of the parent project", ), sa.Column( @@ -37,7 +38,7 @@ sa.Integer, nullable=False, unique=True, - doc="Number of child (as 0-based index: 0 being the oldest, 1, ...)" + doc="0-based index in order of creation (i.e. 0 being the oldest and N-1 the latest)" "from the same parent_id", ), sa.Column( diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 1c19d3aaf78..2710c72d5ef 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -18,7 +18,7 @@ @pytest.fixture async def engine(pg_engine: Engine): - # injects + # injects ... async with pg_engine.acquire() as conn: # a 'me' user user_id = await conn.scalar( From 8673692786d38fd0654c918a0d79722fcca2ac24 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:18:07 +0200 Subject: [PATCH 043/137] renamed module app from parametrization to snapshots --- .../{parametrization.py => snapshots.py} | 13 ++++++------- ...on_api_handlers.py => snapshots_api_handlers.py} | 4 ++-- .../{parametrization_core.py => snapshots_core.py} | 2 +- ...arametrization_models.py => snapshots_models.py} | 0 ...etrization_settings.py => snapshots_settings.py} | 0 ...trization_models.py => test_snapshots_models.py} | 2 +- ...rametrization_core.py => test_snapshots_core.py} | 4 ++-- 7 files changed, 12 insertions(+), 13 deletions(-) rename services/web/server/src/simcore_service_webserver/{parametrization.py => snapshots.py} (74%) rename services/web/server/src/simcore_service_webserver/{parametrization_api_handlers.py => snapshots_api_handlers.py} (97%) rename services/web/server/src/simcore_service_webserver/{parametrization_core.py => snapshots_core.py} (97%) rename services/web/server/src/simcore_service_webserver/{parametrization_models.py => snapshots_models.py} (100%) rename services/web/server/src/simcore_service_webserver/{parametrization_settings.py => snapshots_settings.py} (100%) rename services/web/server/tests/unit/isolated/{test_parametrization_models.py => test_snapshots_models.py} (69%) rename services/web/server/tests/unit/with_dbs/11/{test_parametrization_core.py => test_snapshots_core.py} (95%) diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/snapshots.py similarity index 74% rename from services/web/server/src/simcore_service_webserver/parametrization.py rename to services/web/server/src/simcore_service_webserver/snapshots.py index 725e91ef648..47137d5bf05 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/snapshots.py @@ -1,9 +1,8 @@ -""" parametrization app module setup +""" snapshots (and parametrization) app module setup - Extend project's business logic by adding two new concepts, namely snapshots and parametrizations - - - Project parametrization - - Project snapshots + Extend project's business logic by adding two new concepts, namely + - project snapshots and + - parametrizations """ import logging @@ -15,7 +14,7 @@ app_module_setup, ) -from . import parametrization_api_handlers +from . import snapshots_api_handlers from .constants import APP_SETTINGS_KEY from .settings import ApplicationSettings @@ -34,4 +33,4 @@ def setup(app: web.Application): if not settings.WEBSERVER_DEV_FEATURES_ENABLED: raise SkipModuleSetup(reason="Development feature") - app.add_routes(parametrization_api_handlers.routes) + app.add_routes(snapshots_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py similarity index 97% rename from services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py rename to services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 5b427e9cff3..a0ff98a6b27 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -7,11 +7,11 @@ from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError -from simcore_service_webserver.parametrization_models import Snapshot +from simcore_service_webserver.snapshots_models import Snapshot from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY -from .parametrization_models import Snapshot, SnapshotApiModel +from .snapshots_models import Snapshot, SnapshotApiModel json_dumps = json.dumps diff --git a/services/web/server/src/simcore_service_webserver/parametrization_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py similarity index 97% rename from services/web/server/src/simcore_service_webserver/parametrization_core.py rename to services/web/server/src/simcore_service_webserver/snapshots_core.py index fead73ab024..9a77d8dd7be 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -13,9 +13,9 @@ from models_library.projects_nodes import Node -from .parametrization_models import Snapshot from .projects.projects_db import ProjectAtDB from .projects.projects_utils import clone_project_document +from .snapshots_models import Snapshot def is_parametrized(node: Node) -> bool: diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/parametrization_models.py rename to services/web/server/src/simcore_service_webserver/snapshots_models.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization_settings.py b/services/web/server/src/simcore_service_webserver/snapshots_settings.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/parametrization_settings.py rename to services/web/server/src/simcore_service_webserver/snapshots_settings.py diff --git a/services/web/server/tests/unit/isolated/test_parametrization_models.py b/services/web/server/tests/unit/isolated/test_snapshots_models.py similarity index 69% rename from services/web/server/tests/unit/isolated/test_parametrization_models.py rename to services/web/server/tests/unit/isolated/test_snapshots_models.py index 357899214c5..21933a6d667 100644 --- a/services/web/server/tests/unit/isolated/test_parametrization_models.py +++ b/services/web/server/tests/unit/isolated/test_snapshots_models.py @@ -1,5 +1,5 @@ from simcore_postgres_database.models.snapshots import snapshots -from simcore_service_webserver.parametrization_models import ( +from simcore_service_webserver.snapshots_models import ( Parameter, ParameterApiModel, Snapshot, diff --git a/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py b/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py similarity index 95% rename from services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py rename to services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py index a1060d1b51c..d4f14e54043 100644 --- a/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py +++ b/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py @@ -7,14 +7,14 @@ import pytest from aiohttp import web -from models_library.projects import Project, ProjectAtDB +from models_library.projects import Project # , ProjectAtDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID from simcore_service_webserver.constants import APP_PROJECT_DBAPI -from simcore_service_webserver.parametrization_core import snapshot_project from simcore_service_webserver.projects.projects_api import get_project_for_user from simcore_service_webserver.projects.projects_db import APP_PROJECT_DBAPI from simcore_service_webserver.projects.projects_utils import clone_project_document +from simcore_service_webserver.snapshots_core import snapshot_project # is parametrized project? From 71ddc60520cced9979f146b8b32985e257728012 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 09:43:31 +0200 Subject: [PATCH 044/137] Revert "adds pydantic dep" This reverts commit e00ec034ea47c95c56d4440f0ff704526600bc5b. --- packages/postgres-database/requirements/_pydantic.in | 1 - packages/postgres-database/requirements/_pydantic.txt | 10 ---------- packages/postgres-database/requirements/_test.in | 1 - packages/postgres-database/requirements/dev.txt | 1 - packages/postgres-database/setup.py | 8 ++------ 5 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 packages/postgres-database/requirements/_pydantic.in delete mode 100644 packages/postgres-database/requirements/_pydantic.txt diff --git a/packages/postgres-database/requirements/_pydantic.in b/packages/postgres-database/requirements/_pydantic.in deleted file mode 100644 index 572b352f30e..00000000000 --- a/packages/postgres-database/requirements/_pydantic.in +++ /dev/null @@ -1 +0,0 @@ -pydantic diff --git a/packages/postgres-database/requirements/_pydantic.txt b/packages/postgres-database/requirements/_pydantic.txt deleted file mode 100644 index 6605adc0dff..00000000000 --- a/packages/postgres-database/requirements/_pydantic.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile --output-file=requirements/_pydantic.txt --strip-extras requirements/_pydantic.in -# -pydantic==1.8.2 - # via -r requirements/_pydantic.in -typing-extensions==3.10.0.0 - # via pydantic diff --git a/packages/postgres-database/requirements/_test.in b/packages/postgres-database/requirements/_test.in index 67ad0d9fe0e..c76e9d978cf 100644 --- a/packages/postgres-database/requirements/_test.in +++ b/packages/postgres-database/requirements/_test.in @@ -8,7 +8,6 @@ # --constraint _base.txt --constraint _migration.txt ---constraint _pydantic.txt # fixtures pyyaml diff --git a/packages/postgres-database/requirements/dev.txt b/packages/postgres-database/requirements/dev.txt index d7d03a8f861..8136f1a48b5 100644 --- a/packages/postgres-database/requirements/dev.txt +++ b/packages/postgres-database/requirements/dev.txt @@ -9,7 +9,6 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt ---requirement _pydantic.txt --requirement _test.txt --requirement _tools.txt diff --git a/packages/postgres-database/setup.py b/packages/postgres-database/setup.py index 4780ae5ca50..b5bda1bdc81 100644 --- a/packages/postgres-database/setup.py +++ b/packages/postgres-database/setup.py @@ -21,7 +21,7 @@ def read_reqs(reqs_path: Path): # Strong dependencies migration_requirements = read_reqs(current_dir / "requirements" / "_migration.in") test_requirements = read_reqs(current_dir / "requirements" / "_test.txt") -pydantic_requirements = read_reqs(current_dir / "requirements" / "_pydantic.in") + setup( name="simcore-postgres-database", @@ -43,11 +43,7 @@ def read_reqs(reqs_path: Path): test_suite="tests", install_requires=install_requirements, tests_require=test_requirements, - extras_require={ - "migration": migration_requirements, - "test": test_requirements, - "pydantic": pydantic_requirements, - }, + extras_require={"migration": migration_requirements, "test": test_requirements}, include_package_data=True, package_data={ "": [ From 84dbe57f41264a3600d1449cff55bd1cbd4e5eb1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 09:59:39 +0200 Subject: [PATCH 045/137] removes factory from pg database --- .../utils_pydantic_models_factory.py | 67 ------------------- .../test_utils_pydantic_models_factory.py | 41 ------------ 2 files changed, 108 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py delete mode 100644 packages/postgres-database/tests/test_utils_pydantic_models_factory.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py deleted file mode 100644 index 2b51da22ab5..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Container, Optional, Type -from uuid import UUID - -import sqlalchemy as sa -from pydantic import BaseConfig, BaseModel, Field, create_model - - -class OrmConfig(BaseConfig): - orm_mode = True - - -RESERVED = { - "schema", -} -# e.g. Field name "schema" shadows a BaseModel attribute; use a different field name with "alias='schema'". - - -def sa_table_to_pydantic_model( - table: sa.Table, - *, - config: Type = OrmConfig, - exclude: Optional[Container[str]] = None, -) -> Type[BaseModel]: - - # NOTE: basically copied from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py - fields = {} - exclude = exclude or [] - - for column in table.columns: - name = str(column.name) - - if name in RESERVED: - name = f"{table.name.lower()}_{name}" - - if name in exclude: - continue - - python_type: Optional[type] = None - if hasattr(column.type, "impl"): - if hasattr(column.type.impl, "python_type"): - python_type = column.type.impl.python_type - elif hasattr(column.type, "python_type"): - python_type = column.type.python_type - - assert python_type, f"Could not infer python_type for {column}" # nosec - - default = None - if column.default is None and not column.nullable: - default = ... - - # Policies based on naming conventions - if "uuid" in name.split("_") and python_type == str: - python_type = UUID - if isinstance(default, str): - default = UUID(default) - - if hasattr(column, "doc") and column.doc: - default = Field(default, description=column.doc) - - fields[name] = (python_type, default) - - # create domain models from db-schemas - pydantic_model = create_model( - table.name.capitalize(), __config__=config, **fields # type: ignore - ) - assert issubclass(pydantic_model, BaseModel) # nosec - return pydantic_model diff --git a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py deleted file mode 100644 index 8c984d42a90..00000000000 --- a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py +++ /dev/null @@ -1,41 +0,0 @@ -from datetime import datetime -from uuid import uuid4 - -import pytest -from pydantic import BaseModel - -# pylint: disable=wildcard-import -# pylint: disable=unused-wildcard-import -from simcore_postgres_database.models import * - -# pylint: enable=wildcard-import -# pylint: enable=unused-wildcard-import -from simcore_postgres_database.models.base import metadata -from simcore_postgres_database.models.snapshots import snapshots -from simcore_postgres_database.utils_pydantic_models_factory import ( - sa_table_to_pydantic_model, -) - - -@pytest.mark.parametrize("table_cls", metadata.tables.values(), ids=lambda t: t.name) -def test_table_to_pydantic_models(table_cls): - - PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) - assert issubclass(PydanticModelAtDB, BaseModel) - print(PydanticModelAtDB.schema_json(indent=2)) - - # TODO: create fakes automatically? - - -def test_snapshot_pydantic_model(): - Snapshot = sa_table_to_pydantic_model(snapshots) - - snapshot = Snapshot( - id=0, - name="foo", - created_at=datetime.now(), - parent_uuid=uuid4(), - child_index=2, - project_uuid=uuid4(), - ) - assert snapshot.id == 0 From 1f4478c677dc73f62e2c4395ba53c8552e861830 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:03:54 +0200 Subject: [PATCH 046/137] adds db modesl factory in models-library --- packages/models-library/requirements/_test.in | 1 + .../models-library/requirements/_test.txt | 18 ++++- .../utils/database_models_factory.py | 77 +++++++++++++++++++ .../test_utils_database_models_factory.py | 39 ++++++++++ 4 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 packages/models-library/src/models_library/utils/database_models_factory.py create mode 100644 packages/models-library/tests/test_utils_database_models_factory.py diff --git a/packages/models-library/requirements/_test.in b/packages/models-library/requirements/_test.in index b7d9cece229..f89ccfcacc1 100644 --- a/packages/models-library/requirements/_test.in +++ b/packages/models-library/requirements/_test.in @@ -22,6 +22,7 @@ pytest-sugar # tools pylint # NOTE: The version in pylint at _text.txt is used as a reference for ci/helpers/install_pylint.bash coveralls +--requirement ../../../packages/postgres-database/requirements/_base.in # to test units tools diff --git a/packages/models-library/requirements/_test.txt b/packages/models-library/requirements/_test.txt index 8fe967785e6..bcd8c9e7547 100644 --- a/packages/models-library/requirements/_test.txt +++ b/packages/models-library/requirements/_test.txt @@ -6,6 +6,7 @@ # aiohttp==3.7.4.post0 # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # pytest-aiohttp astroid==2.6.6 @@ -36,6 +37,7 @@ icdiff==2.0.4 idna==2.10 # via # -c requirements/_base.txt + # -r requirements/../../../packages/postgres-database/requirements/_base.in # requests # yarl iniconfig==1.1.1 @@ -61,6 +63,8 @@ pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff +psycopg2-binary==2.9.1 + # via sqlalchemy py==1.10.0 # via pytest pylint==2.9.6 @@ -92,10 +96,16 @@ pytest-sugar==0.9.4 # via -r requirements/_test.in pyyaml==5.4.1 # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in requests==2.26.0 # via coveralls +sqlalchemy==1.3.24 + # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/postgres-database/requirements/_base.in termcolor==1.1.0 # via pytest-sugar toml==0.10.2 @@ -109,12 +119,12 @@ typing-extensions==3.10.0.0 # aiohttp urllib3==1.26.6 # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests wrapt==1.12.1 # via astroid yarl==1.6.3 - # via aiohttp - -# The following packages are considered to be unsafe in a requirements file: -# setuptools + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # aiohttp diff --git a/packages/models-library/src/models_library/utils/database_models_factory.py b/packages/models-library/src/models_library/utils/database_models_factory.py new file mode 100644 index 00000000000..dd2df28ecce --- /dev/null +++ b/packages/models-library/src/models_library/utils/database_models_factory.py @@ -0,0 +1,77 @@ +""" Automatic creation of pydantic model classes from a sqlalchemy table + + +SEE: Copied and adapted from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py +""" + +from typing import Any, Container, Dict, Optional, Type +from uuid import UUID + +import sqlalchemy as sa +from pydantic import BaseConfig, BaseModel, Field, create_model + + +class OrmConfig(BaseConfig): + orm_mode = True + + +_RESERVED = { + "schema", + # e.g. Field name "schema" shadows a BaseModel attribute; use a different field name with "alias='schema'". +} + + +def sa_table_to_pydantic_model( + table: sa.Table, + *, + config: Type = OrmConfig, + exclude: Optional[Container[str]] = None, +) -> Type[BaseModel]: + + fields = {} + exclude = exclude or [] + + for column in table.columns: + name = str(column.name) + + if name in exclude: + continue + + field_args: Dict[str, Any] = {} + + if name in _RESERVED: + field_args["alias"] = name + name = f"{table.name.lower()}_{name}" + + python_type: Optional[type] = None + if hasattr(column.type, "impl"): + if hasattr(column.type.impl, "python_type"): + python_type = column.type.impl.python_type + elif hasattr(column.type, "python_type"): + python_type = column.type.python_type + + assert python_type, f"Could not infer python_type for {column}" # nosec + + default = None + if column.default is None and not column.nullable: + default = ... + + # Policies based on naming conventions + if "uuid" in name.split("_") and python_type == str: + python_type = UUID + if isinstance(default, str): + default = UUID(default) + + field_args["default"] = default + + if hasattr(column, "doc") and column.doc: + field_args["description"] = column.doc + + fields[name] = (python_type, Field(**field_args)) + + # create domain models from db-schemas + pydantic_model = create_model( + table.name.capitalize(), __config__=config, **fields # type: ignore + ) + assert issubclass(pydantic_model, BaseModel) # nosec + return pydantic_model diff --git a/packages/models-library/tests/test_utils_database_models_factory.py b/packages/models-library/tests/test_utils_database_models_factory.py new file mode 100644 index 00000000000..f55e78a3654 --- /dev/null +++ b/packages/models-library/tests/test_utils_database_models_factory.py @@ -0,0 +1,39 @@ +from datetime import datetime +from uuid import uuid4 + +import pytest +from models_library.utils.database_models_factory import sa_table_to_pydantic_model +from pydantic import BaseModel + +# pylint: disable=wildcard-import +# pylint: disable=unused-wildcard-import +from simcore_postgres_database.models import * + +# pylint: enable=wildcard-import +# pylint: enable=unused-wildcard-import +from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.models.snapshots import snapshots + + +@pytest.mark.parametrize("table_cls", metadata.tables.values(), ids=lambda t: t.name) +def test_table_to_pydantic_models(table_cls): + + PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) + assert issubclass(PydanticModelAtDB, BaseModel) + print(PydanticModelAtDB.schema_json(indent=2)) + + # TODO: create fakes automatically? + + +def test_snapshot_pydantic_model(): + Snapshot = sa_table_to_pydantic_model(snapshots) + + snapshot = Snapshot( + id=0, + name="foo", + created_at=datetime.now(), + parent_uuid=uuid4(), + child_index=2, + project_uuid=uuid4(), + ) + assert snapshot.id == 0 From c2696fe43884d2937b23d60d29376873c1093273 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:04:14 +0200 Subject: [PATCH 047/137] wip snapshots --- .../models/snapshots.py | 4 +- .../postgres-database/tests/test_snapshots.py | 43 ++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index f23c80333ce..cf3c7dfd6e7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -19,7 +19,9 @@ sa.DateTime(), nullable=False, server_default=func.now(), - doc="Timestamp on creation", + doc="Timestamp on creation. " + "It corresponds to the last_change_date of the parent project " + "at the time the snapshot was taken.", ), sa.Column( "parent_uuid", diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 2710c72d5ef..f8442091411 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -15,6 +15,17 @@ from simcore_postgres_database.models.snapshots import snapshots from simcore_postgres_database.models.users import users +USERNAME = "me" +PARENT_PROJECT_NAME = "parent" + + +def test_it(): + import os + + cwd = os.getcwd() + + assert True + @pytest.fixture async def engine(pg_engine: Engine): @@ -22,11 +33,13 @@ async def engine(pg_engine: Engine): async with pg_engine.acquire() as conn: # a 'me' user user_id = await conn.scalar( - users.insert().values(**random_user(name="me")).returning(users.c.id) + users.insert().values(**random_user(name=USERNAME)).returning(users.c.id) ) # has a project 'parent' await conn.execute( - projects.insert().values(**random_project(prj_owner=user_id, name="parent")) + projects.insert().values( + **random_project(prj_owner=user_id, name=PARENT_PROJECT_NAME) + ) ) yield pg_engine @@ -42,15 +55,13 @@ async def test_creating_snapshots(engine: Engine): } async def _create_snapshot(child_index: int, parent_prj, conn) -> int: - # copy - # change uuid, and set to invisible + # create project-snapshot prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} prj_dict["name"] += f" [snapshot {child_index}]" prj_dict["uuid"] = uuid3(UUID(parent_prj.uuid), f"snapshot.{child_index}") - prj_dict[ - "creation_date" - ] = parent_prj.last_change_date # state of parent upon copy! + # creation_data = state of parent upon copy! WARNING: changes can be state changes and not project definition? + prj_dict["creation_date"] = parent_prj.last_change_date prj_dict["hidden"] = True prj_dict["published"] = False @@ -85,7 +96,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: # get parent res: ResultProxy = await conn.execute( - projects.select().where(projects.c.name == "parent") + projects.select().where(projects.c.name == PARENT_PROJECT_NAME) ) parent_prj: Optional[RowProxy] = await res.first() @@ -105,14 +116,21 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: ).first() assert updated_parent_prj - assert updated_parent_prj.id == parent_prj.id assert updated_parent_prj.description != parent_prj.description + assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date # take another snapshot snapshot_two_id = await _create_snapshot(1, updated_parent_prj, conn) - assert snapshot_one_id != snapshot_two_id + snapshot_two = await ( + await conn.execute( + snapshots.select().where(snapshots.c.id == snapshot_two_id) + ) + ).first() + + assert snapshot_two.id != snapshot_one_id + assert snapshot_two.created_at == updated_parent_prj.last_change_date # get project corresponding to snapshot 1 j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) @@ -125,12 +143,17 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: ).first() assert selected_snapshot_project + assert selected_snapshot_project.uuid == snapshot_two.projects_uuid + assert parent_prj.uuid == snapshot_two.parent_uuid def extract(t): return {k: t[k] for k in t if k not in exclude.union({"name"})} assert extract(selected_snapshot_project) == extract(updated_parent_prj) + # TODO: if we call to take consecutive snapshots ... of the same thing, it should + # return existing + def test_deleting_snapshots(): # test delete child project -> deletes snapshot From 332ae3e3023d4e23dd13a39f24d2947ebe0d4270 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:33:07 +0200 Subject: [PATCH 048/137] db snapshots pass --- .../models/snapshots.py | 4 +- .../postgres-database/tests/test_snapshots.py | 41 +++++++------------ 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index cf3c7dfd6e7..f2dda1ce4a4 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -1,5 +1,4 @@ import sqlalchemy as sa -from sqlalchemy.sql import func from .base import metadata @@ -18,8 +17,7 @@ "created_at", sa.DateTime(), nullable=False, - server_default=func.now(), - doc="Timestamp on creation. " + doc="Timestamp for this snapshot." "It corresponds to the last_change_date of the parent project " "at the time the snapshot was taken.", ), diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index f8442091411..12879727072 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -19,14 +19,6 @@ PARENT_PROJECT_NAME = "parent" -def test_it(): - import os - - cwd = os.getcwd() - - assert True - - @pytest.fixture async def engine(pg_engine: Engine): # injects ... @@ -55,6 +47,8 @@ async def test_creating_snapshots(engine: Engine): } async def _create_snapshot(child_index: int, parent_prj, conn) -> int: + # NOTE: used as prototype + # create project-snapshot prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} @@ -83,7 +77,8 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: snapshot_id = await conn.scalar( snapshots.insert() .values( - name=f"Snapshot {child_index}", + name=f"Snapshot {child_index} [{parent_prj.name}]", + created_at=parent_prj.last_change_date, parent_uuid=parent_prj.uuid, child_index=child_index, project_uuid=project_uuid, @@ -93,7 +88,6 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: return snapshot_id async with engine.acquire() as conn: - # get parent res: ResultProxy = await conn.execute( projects.select().where(projects.c.name == PARENT_PROJECT_NAME) @@ -103,7 +97,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert parent_prj # take one snapshot - snapshot_one_id = await _create_snapshot(0, parent_prj, conn) + first_snapshot_id = await _create_snapshot(0, parent_prj, conn) # modify parent updated_parent_prj = await ( @@ -121,30 +115,31 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date # take another snapshot - snapshot_two_id = await _create_snapshot(1, updated_parent_prj, conn) + second_snapshot_id = await _create_snapshot(1, updated_parent_prj, conn) - snapshot_two = await ( + second_snapshot = await ( await conn.execute( - snapshots.select().where(snapshots.c.id == snapshot_two_id) + snapshots.select().where(snapshots.c.id == second_snapshot_id) ) ).first() - assert snapshot_two.id != snapshot_one_id - assert snapshot_two.created_at == updated_parent_prj.last_change_date + assert second_snapshot + assert second_snapshot.id != first_snapshot_id + assert second_snapshot.created_at == updated_parent_prj.last_change_date - # get project corresponding to snapshot 1 + # get project corresponding to first snapshot j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) selected_snapshot_project = await ( await conn.execute( projects.select() .select_from(j) - .where(snapshots.c.id == snapshot_two_id) + .where(snapshots.c.id == second_snapshot_id) ) ).first() assert selected_snapshot_project - assert selected_snapshot_project.uuid == snapshot_two.projects_uuid - assert parent_prj.uuid == snapshot_two.parent_uuid + assert selected_snapshot_project.uuid == second_snapshot.project_uuid + assert parent_prj.uuid == second_snapshot.parent_uuid def extract(t): return {k: t[k] for k in t if k not in exclude.union({"name"})} @@ -162,9 +157,3 @@ def test_deleting_snapshots(): # test delete parent project -> deletes snapshots # test delete snapshot does NOT delete parent pass - - -def test_create_pydantic_models_from_sqlalchemy_tables(): - # SEE https://docs.sqlalchemy.org/en/14/core/metadata.html - # SEE https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py - pass From 628ccc2283b111b41a3e2f41450ca455b4ed7e2f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 12:58:35 +0200 Subject: [PATCH 049/137] updates web-server oas --- .../schemas/node-meta-v0.0.1-converted.yaml | 8 +- api/specs/webserver/openapi-projects.yaml | 175 +++++++++--- api/specs/webserver/openapi.yaml | 4 +- .../api/v0/openapi.yaml | 18 +- .../api/v0/schemas/node-meta-v0.0.1.json | 9 +- .../api/v0/schemas/node-meta-v0.0.1.json | 9 +- .../api/v0/openapi.yaml | 263 ++++++++++++++++++ .../api/v0/schemas/node-meta-v0.0.1.json | 9 +- .../sandbox/projects_openapi_generator.py | 43 ++- 9 files changed, 458 insertions(+), 80 deletions(-) diff --git a/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml b/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml index 912b978a09a..3a39e566cd4 100644 --- a/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml +++ b/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml @@ -119,11 +119,11 @@ properties: - type properties: displayOrder: + description: >- + DEPRECATED: new display order is taken from the item position. + This property will be removed. + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 1a66a9011cc..c310d1c7977 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -81,6 +81,7 @@ paths: $ref: "#/components/schemas/ProjectEnveloped" default: $ref: "#/components/responses/DefaultErrorResponse" + /projects/active: get: tags: @@ -437,49 +438,101 @@ paths: projects_snapshots: get: - summary: List Project Snapshots + summary: List Snapshots + operationId: list_snapshots + tags: + - project parameters: - - in: path - name: project_id - required: true - schema: - format: uuid - title: Project Id - type: string + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + type: integer + description: index to the first item to return (pagination) + default: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50 + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query responses: - '200': + "200": + description: Successful Response content: application/json: - schema: {} - description: Successful Response + schema: + title: Response List Snapshots Projects Project Id Snapshots Get + type: array + items: + $ref: "#/components/schemas/Snapshot" projects_id_snapshots_id: get: - summary: Get Project Snapshot + summary: Get Snapshot + operationId: get_snapshot + tags: + - project parameters: - - in: path - name: project_id - required: true - schema: - format: uuid - title: Project Id - type: string - - in: path - name: snapshot_id - required: true - schema: - exclusiveMinimum: 0.0 - title: Snapshot Id - type: integer + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path responses: - '200': + "200": + description: Successful Response content: application/json: - schema: {} - description: Successful Response + schema: + $ref: "#/components/schemas/Snapshot" - projects_id_snapshots_id_parameters: - get: + post: + summary: Create Snapshot + operationId: create_snapshot + parameters: + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query + responses: + "200": + description: Successful Response + content: + application/json: + schema: + $ref: "#/components/schemas/Snapshot" components: @@ -524,6 +577,66 @@ components: RunningServiceEnveloped: $ref: "../common/schemas/running_service.yaml#/components/schemas/RunningServiceEnveloped" + Snapshot: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that + parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri + responses: DefaultErrorResponse: $ref: "./openapi.yaml#/components/responses/DefaultErrorResponse" diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 518bf6b64a9..e24e7bf1320 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -216,8 +216,8 @@ paths: /projects/{project_id}/snapshots/{snapshot_id}: $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id" - /projects/{project_id}/snapshots/{snapshot_id}/parameters: - $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id_parameters" + # /projects/{project_id}/snapshots/{snapshot_id}/parameters: + # $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id_parameters" # ACTIVITY ------------------------------------------------------------------------- /activity/status: diff --git a/services/director/src/simcore_service_director/api/v0/openapi.yaml b/services/director/src/simcore_service_director/api/v0/openapi.yaml index 74d5fc9afb5..4ce769b02a8 100644 --- a/services/director/src/simcore_service_director/api/v0/openapi.yaml +++ b/services/director/src/simcore_service_director/api/v0/openapi.yaml @@ -256,11 +256,9 @@ paths: - type properties: displayOrder: + description: 'DEPRECATED: new display order is taken from the item position. This property will be removed.' + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property @@ -649,11 +647,9 @@ paths: - type properties: displayOrder: + description: 'DEPRECATED: new display order is taken from the item position. This property will be removed.' + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property @@ -2436,11 +2432,9 @@ components: - type properties: displayOrder: + description: 'DEPRECATED: new display order is taken from the item position. This property will be removed.' + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property diff --git a/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json b/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json index 660045befab..4fe1aa31465 100644 --- a/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json +++ b/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json @@ -180,12 +180,9 @@ ], "properties": { "displayOrder": { - "type": "number", - "description": "use this to numerically sort the properties for display", - "examples": [ - 1, - -0.2 - ] + "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.", + "deprecated": true, + "type": "number" }, "label": { "type": "string", diff --git a/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json b/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json index 660045befab..4fe1aa31465 100644 --- a/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json +++ b/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json @@ -180,12 +180,9 @@ ], "properties": { "displayOrder": { - "type": "number", - "description": "use this to numerically sort the properties for display", - "examples": [ - 1, - -0.2 - ] + "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.", + "deprecated": true, + "type": "number" }, "label": { "type": "string", diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index b6070e0df0e..5e07b331655 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13244,6 +13244,269 @@ paths: message: Password is not secure field: pasword status: 400 + '/projects/{project_id}/snapshots': + get: + summary: List Snapshots + operationId: list_snapshots_projects__project_id__snapshots_get + tags: + - project + parameters: + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + type: integer + description: index to the first item to return (pagination) + default: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50 + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: Response List Snapshots Projects Project Id Snapshots Get + type: array + items: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri + '/projects/{project_id}/snapshots/{snapshot_id}': + get: + summary: Get Snapshot + operationId: get_snapshot_projects__project_id__snapshots__snapshot_id__get + tags: + - project + parameters: + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri + post: + summary: Create Snapshot + operationId: create_snapshot_projects__project_id__snapshots_post + parameters: + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri /activity/status: get: operationId: get_status diff --git a/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json b/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json index 660045befab..4fe1aa31465 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json +++ b/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json @@ -180,12 +180,9 @@ ], "properties": { "displayOrder": { - "type": "number", - "description": "use this to numerically sort the properties for display", - "examples": [ - 1, - -0.2 - ] + "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.", + "deprecated": true, + "type": "number" }, "label": { "type": "string", diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py index 58467dddc1f..5c0baf38cf5 100644 --- a/services/web/server/tests/sandbox/projects_openapi_generator.py +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -10,7 +10,7 @@ from fastapi import Depends, FastAPI from fastapi import Path as PathParam -from fastapi import Request, status +from fastapi import Query, Request, status from fastapi.exceptions import HTTPException from models_library.services import PROPERTY_KEY_RE from pydantic import ( @@ -145,56 +145,66 @@ def get_valid_id(project_id: UUID = PathParam(...)) -> UUID: #################################################################### -@app.get("/projects/{project_id}", response_model=Project) +@app.get("/projects/{project_id}", response_model=Project, tags=["project"]) def get_project(pid: UUID = Depends(get_valid_id)): return _PROJECTS[pid] -@app.post("/projects/{project_id}") +@app.post("/projects/{project_id}", tags=["project"]) def create_project(project: Project): if project.id not in _PROJECTS: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Invalid id") _PROJECTS[project.id] = project -@app.put("/projects/{project_id}") +@app.put("/projects/{project_id}", tags=["project"]) def replace_project(project: Project, pid: UUID = Depends(get_valid_id)): _PROJECTS[pid] = project -@app.patch("/projects/{project_id}") +@app.patch("/projects/{project_id}", tags=["project"]) def update_project(project: Project, pid: UUID = Depends(get_valid_id)): raise NotImplementedError() -@app.delete("/projects/{project_id}") +@app.delete("/projects/{project_id}", tags=["project"]) def delete_project(pid: UUID = Depends(get_valid_id)): del _PROJECTS[pid] -@app.post("/projects/{project_id}:open") +@app.post("/projects/{project_id}:open", tags=["project"]) def open_project(pid: UUID = Depends(get_valid_id)): pass -@app.post("/projects/{project_id}:start") +@app.post("/projects/{project_id}:start", tags=["project"]) def start_project(use_cache: bool = True, pid: UUID = Depends(get_valid_id)): pass -@app.post("/projects/{project_id}:stop") +@app.post("/projects/{project_id}:stop", tags=["project"]) def stop_project(pid: UUID = Depends(get_valid_id)): pass -@app.post("/projects/{project_id}:close") +@app.post("/projects/{project_id}:close", tags=["project"]) def close_project(pid: UUID = Depends(get_valid_id)): pass -@app.get("/projects/{project_id}/snapshots", response_model=List[SnapshotApiModel]) +@app.get( + "/projects/{project_id}/snapshots", + response_model=List[SnapshotApiModel], + tags=["project"], +) async def list_snapshots( pid: UUID = Depends(get_valid_id), + offset: PositiveInt = Query( + 0, description="index to the first item to return (pagination)" + ), + limit: int = Query( + 20, description="maximum number of items to return (pagination)", ge=1, le=50 + ), url_for: Callable = Depends(get_reverse_url_mapper), ): psid = _PROJECT2SNAPSHOT.get(pid) @@ -209,7 +219,11 @@ async def list_snapshots( ] -@app.post("/projects/{project_id}/snapshots", response_model=SnapshotApiModel) +@app.post( + "/projects/{project_id}/snapshots", + response_model=SnapshotApiModel, + tags=["project"], +) async def create_snapshot( pid: UUID = Depends(get_valid_id), snapshot_label: Optional[str] = None, @@ -265,6 +279,7 @@ async def create_snapshot( @app.get( "/projects/{project_id}/snapshots/{snapshot_id}", response_model=SnapshotApiModel, + tags=["project"], ) async def get_snapshot( snapshot_id: PositiveInt, @@ -286,6 +301,7 @@ async def get_snapshot( @app.get( "/projects/{project_id}/snapshots/{snapshot_id}/parameters", response_model=List[ParameterApiModel], + tags=["project"], ) async def list_snapshot_parameters( snapshot_id: str, @@ -331,6 +347,7 @@ def create_snapshots(project_id: UUID): # print("-"*100) -print(json.dumps(app.openapi(), indent=2)) +with open("openapi-ignore.json", "wt") as f: + json.dump(app.openapi(), f, indent=2) # uvicorn --reload projects_openapi_generator:app From 78e3c2d8a90ff7ce826d0c7aac2a1e2f641e147f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 14:56:49 +0200 Subject: [PATCH 050/137] list and get snapshots --- .../simcore_service_webserver/application.py | 7 +- .../db_base_repository.py | 12 ++ .../simcore_service_webserver/snapshots.py | 3 +- .../snapshots_api_handlers.py | 162 +++++++++--------- .../simcore_service_webserver/snapshots_db.py | 57 ++++++ .../snapshots_models.py | 18 +- .../unit/isolated/test_snapshots_models.py | 2 +- 7 files changed, 176 insertions(+), 85 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/db_base_repository.py create mode 100644 services/web/server/src/simcore_service_webserver/snapshots_db.py diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index d27326004bc..4d79834503b 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -9,8 +9,6 @@ from servicelib.application import create_safe_application from servicelib.rest_pagination_utils import monkey_patch_pydantic_url_regex -monkey_patch_pydantic_url_regex() - from ._meta import WELCOME_MSG from .activity import setup_activity from .catalog import setup_catalog @@ -31,6 +29,7 @@ from .security import setup_security from .session import setup_session from .settings import setup_settings +from .snapshots import setup_snapshots from .socketio import setup_socketio from .statics import setup_statics from .storage import setup_storage @@ -40,6 +39,9 @@ from .tracing import setup_app_tracing from .users import setup_users +monkey_patch_pydantic_url_regex() + + log = logging.getLogger(__name__) @@ -75,6 +77,7 @@ def create_application(config: Dict[str, Any]) -> web.Application: setup_users(app) setup_groups(app) setup_projects(app) + setup_snapshots(app) setup_activity(app) setup_resource_manager(app) setup_tags(app) diff --git a/services/web/server/src/simcore_service_webserver/db_base_repository.py b/services/web/server/src/simcore_service_webserver/db_base_repository.py new file mode 100644 index 00000000000..28012be4fe0 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/db_base_repository.py @@ -0,0 +1,12 @@ +# a repo: retrieves engine from app +# TODO: a repo: some members acquire and retrieve connection +# TODO: a repo: any validation error in a repo is due to corrupt data in db! + +from aiohttp import web + +from .constants import APP_DB_ENGINE_KEY + + +class BaseRepository: + def __init__(self, app: web.Application): + self.engine = app[APP_DB_ENGINE_KEY] diff --git a/services/web/server/src/simcore_service_webserver/snapshots.py b/services/web/server/src/simcore_service_webserver/snapshots.py index 47137d5bf05..3c331800650 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots.py +++ b/services/web/server/src/simcore_service_webserver/snapshots.py @@ -27,10 +27,11 @@ depends=["simcore_service_webserver.projects"], logger=log, ) -def setup(app: web.Application): +def setup_snapshots(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: raise SkipModuleSetup(reason="Development feature") + # TODO: validate routes against OAS app.add_routes(snapshots_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index a0ff98a6b27..4db60275c85 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -1,19 +1,33 @@ -import json from functools import wraps -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, List, Optional from uuid import UUID +import orjson from aiohttp import web -from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError -from simcore_service_webserver.snapshots_models import Snapshot +from pydantic.main import BaseModel from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY -from .snapshots_models import Snapshot, SnapshotApiModel +from .snapshots_db import SnapshotsRepository +from .snapshots_models import Snapshot, SnapshotItem -json_dumps = json.dumps + +def _default(obj): + if isinstance(obj, BaseModel): + return obj.dict() + raise TypeError + + +def json_dumps(v) -> str: + # orjson.dumps returns bytes, to match standard json.dumps we need to decode + return orjson.dumps(v, default=_default).decode() + + +def enveloped_response(data: Any) -> web.Response: + enveloped: str = json_dumps({"data": data}) + return web.Response(text=enveloped, content_type="application/json") def handle_request_errors(handler: Callable): @@ -47,127 +61,121 @@ async def wrapped(request: web.Request): @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots", - name="_list_snapshots_handler", + name="list_project_snapshots_handler", ) @handle_request_errors -async def _list_snapshots_handler(request: web.Request): +async def list_project_snapshots_handler(request: web.Request): """ Lists references on project snapshots """ user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - snapshots = await list_snapshots( + snapshots_repo = SnapshotsRepository(request.app) + + @validate_arguments + async def list_snapshots(project_id: UUID) -> List[Snapshot]: + # project_id is param-project? + # TODO: add pagination + # TODO: optimizaiton will grow snapshots of a project with time! + # + snapshots_orm = await snapshots_repo.list(project_id) + # snapshots: + # - ordered (iterations!) + # - have a parent project with all the parametrization + + return [Snapshot.from_orm(obj) for obj in snapshots_orm] + + snapshots: List[Snapshot] = await list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) - # Append url links - url_for_snapshot = request.app.router["_get_snapshot_handler"].url_for - url_for_parameters = request.app.router["_get_snapshot_parameters_handler"].url_for + # TODO: async for snapshot in await list_snapshot is the same? - for snp in snapshots: - snp["url"] = url_for_snapshot( - project_id=snp["parent_id"], snapshot_id=snp["id"] - ) - snp["url_parameters"] = url_for_parameters( - project_id=snp["parent_id"], snapshot_id=snp["id"] - ) - # snp['url_project'] = + data = [] + for snapshot in snapshots: + # FIXME: raw dict + data.append(SnapshotItem.from_snapshot(snapshot, request.app).dict()) - return snapshots + return enveloped_response(data) @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}", - name="_get_snapshot_handler", + name="get_project_snapshot_handler", ) @handle_request_errors -async def _get_snapshot_handler(request: web.Request): +async def get_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + # FIXME: access rights ?? + + snapshots_repo = SnapshotsRepository(request.app) + + @validate_arguments + async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: + try: + snapshot_orm = await snapshots_repo.get_by_index( + project_id, int(snapshot_id) + ) + except ValueError: + snapshot_orm = await snapshots_repo.get_by_name(project_id, snapshot_id) + + if not snapshot_orm: + raise web.HTTPNotFound(reason=f"snapshot {snapshot_id} not found") + + return Snapshot.from_orm(snapshot_orm) + snapshot = await get_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) - return snapshot.json() + return enveloped_response(snapshot) @routes.post( f"/{vtag}/projects/{{project_id}}/snapshots", - name="_create_snapshot_handler", ) @handle_request_errors -async def _create_snapshot_handler(request: web.Request): +async def create_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + snapshots_repo = (SnapshotsRepository(request.app),) + + @validate_arguments + async def create_snapshot( + project_id: UUID, + snapshot_label: Optional[str] = None, + ) -> Snapshot: + raise NotImplementedError snapshot = await create_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_label=request.query.get("snapshot_label"), ) - return snapshot.json() + return enveloped_response(snapshots) @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", - name="_get_snapshot_parameters_handler", + name="get_snapshot_parameters_handler", ) @handle_request_errors -async def _get_snapshot_parameters_handler( +async def get_project_snapshot_parameters_handler( request: web.Request, ): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + @validate_arguments + async def get_snapshot_parameters( + project_id: UUID, + snapshot_id: str, + ): + # + return {"x": 4, "y": "yes"} + params = await get_snapshot_parameters( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) return params - - -# API ROUTES HANDLERS --------------------------------------------------------- - - -@validate_arguments -async def list_snapshots(project_id: UUID) -> List[Dict[str, Any]]: - # project_id is param-project? - # TODO: add pagination - # TODO: optimizaiton will grow snapshots of a project with time! - # - - # snapshots: - # - ordered (iterations!) - # - have a parent project with all the parametrization - # - snapshot_info_0 = { - "id": 0, - "display_name": "snapshot 0", - "parent_id": project_id, - "parameters": get_snapshot_parameters(project_id, snapshot_id=str(id)), - } - - return [ - snapshot_info_0, - ] - - -@validate_arguments -async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: - # TODO: create a fake project - # - generate project_id - # - define what changes etc... - pass - - -@validate_arguments -async def create_snapshot( - project_id: UUID, - snapshot_label: Optional[str] = None, -) -> Snapshot: - pass - - -@validate_arguments -async def get_snapshot_parameters(project_id: UUID, snapshot_id: str): - # - return {"x": 4, "y": "yes"} diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py new file mode 100644 index 00000000000..7fb2bc84759 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -0,0 +1,57 @@ +from typing import List, Optional +from uuid import UUID + +from aiopg.sa.result import RowProxy +from pydantic import PositiveInt +from simcore_postgres_database.models.snapshots import snapshots + +from .db_base_repository import BaseRepository + +# alias for readability +# SEE https://pydantic-docs.helpmanual.io/usage/models/#orm-mode-aka-arbitrary-class-instances +SnapshotOrm = RowProxy + + +class SnapshotsRepository(BaseRepository): + """ + Abstracts access to snapshots database table + + Gets primitive/standard parameters and returns valid orm objects + """ + + async def list(self, projec_uuid: UUID) -> List[SnapshotOrm]: + result = [] + async with self.engine.acquire() as conn: + stmt = ( + snapshots.select() + .where(snapshots.c.parent_uuid == projec_uuid) + .order_by(snapshots.c.child_index) + ) + async for row in conn.execute(stmt): + result.append(row) + return result + + async def _get(self, stmt) -> Optional[SnapshotOrm]: + async with self.engine.acquire() as conn: + return await (await conn.execute(stmt)).first() + + async def get_by_index( + self, project_uuid: UUID, snapshot_index: PositiveInt + ) -> Optional[SnapshotOrm]: + stmt = snapshots.select().where( + (snapshots.c.parent_uuid == project_uuid) + & (snapshots.c.child_index == snapshot_index) + ) + return await self._get(stmt) + + async def get_by_name( + self, project_uuid: UUID, snapshot_name: str + ) -> Optional[SnapshotOrm]: + stmt = snapshots.select().where( + (snapshots.c.parent_uuid == project_uuid) + & (snapshots.c.name == snapshot_name) + ) + return await self._get(stmt) + + async def create(self, project_uuid: UUID) -> SnapshotOrm: + pass diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index f1d3ad54885..573b6e50154 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -2,6 +2,7 @@ from typing import Callable, Optional, Union from uuid import UUID +from aiohttp import web from models_library.projects_nodes import OutputID from pydantic import ( AnyUrl, @@ -15,6 +16,7 @@ BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] + ## Domain models -------- class Parameter(BaseModel): name: str @@ -36,6 +38,9 @@ class Snapshot(BaseModel): parent_id: UUID = Field(..., description="Parent's project uuid") project_id: UUID = Field(..., description="Current project's uuid") + class Config: + orm_mode = True + ## API models ---------- @@ -45,24 +50,29 @@ class ParameterApiModel(Parameter): # url_output: AnyUrl -class SnapshotApiModel(Snapshot): +class SnapshotItem(Snapshot): + """ API model for an array item of snapshots """ + url: AnyUrl url_parent: AnyUrl url_project: AnyUrl url_parameters: Optional[AnyUrl] = None @classmethod - def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiModel": + def from_snapshot(cls, snapshot: Snapshot, app: web.Application) -> "SnapshotItem": + def url_for(router_name: str, **params): + return app.router[router_name].url_for(**params) + return cls( url=url_for( - "get_snapshot", + "get_project_snapshot_handler", project_id=snapshot.project_id, snapshot_id=snapshot.id, ), url_parent=url_for("get_project", project_id=snapshot.parent_id), url_project=url_for("get_project", project_id=snapshot.project_id), url_parameters=url_for( - "get_snapshot_parameters", + "get_project_snapshot_parameters_handler", project_id=snapshot.parent_id, snapshot_id=snapshot.id, ), diff --git a/services/web/server/tests/unit/isolated/test_snapshots_models.py b/services/web/server/tests/unit/isolated/test_snapshots_models.py index 21933a6d667..87be075f0a5 100644 --- a/services/web/server/tests/unit/isolated/test_snapshots_models.py +++ b/services/web/server/tests/unit/isolated/test_snapshots_models.py @@ -3,5 +3,5 @@ Parameter, ParameterApiModel, Snapshot, - SnapshotApiModel, + SnapshotItem, ) From c7ac5f1c0fab77d4ecd98978e4905f5bcb3bd43d Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:17:42 +0200 Subject: [PATCH 051/137] fixes OAS --- api/specs/webserver/openapi-projects.yaml | 47 ++++++++++--------- .../api/v0/openapi.yaml | 42 +++++++++-------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index c310d1c7977..8a2acab3f89 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -439,7 +439,7 @@ paths: projects_snapshots: get: summary: List Snapshots - operationId: list_snapshots + operationId: list_project_snapshots_handler tags: - project parameters: @@ -480,20 +480,12 @@ paths: type: array items: $ref: "#/components/schemas/Snapshot" - - projects_id_snapshots_id: - get: - summary: Get Snapshot - operationId: get_snapshot + post: + summary: Create Snapshot + operationId: create_project_snapshot_handler tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -501,6 +493,12 @@ paths: format: uuid name: project_id in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query responses: "200": description: Successful Response @@ -509,10 +507,19 @@ paths: schema: $ref: "#/components/schemas/Snapshot" - post: - summary: Create Snapshot - operationId: create_snapshot + projects_id_snapshots_id: + get: + summary: Get Snapshot + operationId: get_project_snapshot_handler + tags: + - project parameters: + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path - required: true schema: title: Project Id @@ -520,12 +527,6 @@ paths: format: uuid name: project_id in: path - - required: false - schema: - title: Snapshot Label - type: string - name: snapshot_label - in: query responses: "200": description: Successful Response @@ -534,7 +535,6 @@ paths: schema: $ref: "#/components/schemas/Snapshot" - components: schemas: ClientSessionId: @@ -599,7 +599,8 @@ components: created_at: title: Created At type: string - description: Timestamp of the time snapshot was taken from parent. Notice that + description: + Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time parent_id: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 5e07b331655..6a97055351d 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13247,7 +13247,7 @@ paths: '/projects/{project_id}/snapshots': get: summary: List Snapshots - operationId: list_snapshots_projects__project_id__snapshots_get + operationId: list_project_snapshots_handler tags: - project parameters: @@ -13344,19 +13344,12 @@ paths: minLength: 1 type: string format: uri - '/projects/{project_id}/snapshots/{snapshot_id}': - get: - summary: Get Snapshot - operationId: get_snapshot_projects__project_id__snapshots__snapshot_id__get + post: + summary: Create Snapshot + operationId: create_project_snapshot_handler tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -13364,6 +13357,12 @@ paths: format: uuid name: project_id in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query responses: '200': description: Successful Response @@ -13427,10 +13426,19 @@ paths: minLength: 1 type: string format: uri - post: - summary: Create Snapshot - operationId: create_snapshot_projects__project_id__snapshots_post + '/projects/{project_id}/snapshots/{snapshot_id}': + get: + summary: Get Snapshot + operationId: get_project_snapshot_handler + tags: + - project parameters: + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path - required: true schema: title: Project Id @@ -13438,12 +13446,6 @@ paths: format: uuid name: project_id in: path - - required: false - schema: - title: Snapshot Label - type: string - name: snapshot_label - in: query responses: '200': description: Successful Response From 620851182c6dc2d4f075d46288247032d42d4b8a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:18:38 +0200 Subject: [PATCH 052/137] setup snapshot app module and connects handlers to routes --- .../application_config.py | 1 + .../projects/module_setup.py | 5 +++- .../snapshots_api_handlers.py | 24 +++++++++++++------ .../simcore_service_webserver/snapshots_db.py | 6 ++--- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/application_config.py b/services/web/server/src/simcore_service_webserver/application_config.py index d35d7974fd0..8187ffdd069 100644 --- a/services/web/server/src/simcore_service_webserver/application_config.py +++ b/services/web/server/src/simcore_service_webserver/application_config.py @@ -94,6 +94,7 @@ def create_schema() -> T.Dict: addon_section("studies_access", optional=True): minimal_addon_schema(), addon_section("studies_dispatcher", optional=True): minimal_addon_schema(), addon_section("exporter", optional=True): minimal_addon_schema(), + addon_section("snapshots", optional=True): minimal_addon_schema(), } ) diff --git a/services/web/server/src/simcore_service_webserver/projects/module_setup.py b/services/web/server/src/simcore_service_webserver/projects/module_setup.py index c723f4e8abe..188d660a164 100644 --- a/services/web/server/src/simcore_service_webserver/projects/module_setup.py +++ b/services/web/server/src/simcore_service_webserver/projects/module_setup.py @@ -42,7 +42,10 @@ def _create_routes(tag, specs, *handlers_module, disable_login: bool = False): routes = map_handlers_with_operations( handlers, - filter(lambda o: tag in o[3], iter_path_operations(specs)), + filter( + lambda o: tag in o[3] and "snapshot" not in o[2], + iter_path_operations(specs), + ), strict=True, ) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 4db60275c85..74cb099b9eb 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -10,6 +10,8 @@ from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY +from .login.decorators import login_required +from .security_decorators import permission_required from .snapshots_db import SnapshotsRepository from .snapshots_models import Snapshot, SnapshotItem @@ -25,8 +27,8 @@ def json_dumps(v) -> str: return orjson.dumps(v, default=_default).decode() -def enveloped_response(data: Any) -> web.Response: - enveloped: str = json_dumps({"data": data}) +def enveloped_response(data: Any, **extra) -> web.Response: + enveloped: str = json_dumps({"data": data, **extra}) return web.Response(text=enveloped, content_type="application/json") @@ -43,7 +45,7 @@ async def wrapped(request: web.Request): except KeyError as err: # NOTE: handles required request.match_info[*] or request.query[*] - raise web.HTTPBadRequest(reason="Expected parameter {err}") from err + raise web.HTTPBadRequest(reason=f"Expected parameter {err}") from err except ValidationError as err: # NOTE: pydantic.validate_arguments parses and validates -> ValidationError @@ -63,6 +65,8 @@ async def wrapped(request: web.Request): f"/{vtag}/projects/{{project_id}}/snapshots", name="list_project_snapshots_handler", ) +@login_required +@permission_required("project.read") @handle_request_errors async def list_project_snapshots_handler(request: web.Request): """ @@ -103,6 +107,8 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}", name="get_project_snapshot_handler", ) +@login_required +@permission_required("project.read") @handle_request_errors async def get_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] @@ -133,32 +139,36 @@ async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: @routes.post( - f"/{vtag}/projects/{{project_id}}/snapshots", + f"/{vtag}/projects/{{project_id}}/snapshots", name="create_project_snapshot_handler" ) +@login_required +@permission_required("project.create") @handle_request_errors async def create_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] snapshots_repo = (SnapshotsRepository(request.app),) @validate_arguments - async def create_snapshot( + async def _create_snapshot( project_id: UUID, snapshot_label: Optional[str] = None, ) -> Snapshot: raise NotImplementedError - snapshot = await create_snapshot( + snapshot = await _create_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_label=request.query.get("snapshot_label"), ) - return enveloped_response(snapshots) + return enveloped_response(snapshot) @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", name="get_snapshot_parameters_handler", ) +@login_required +@permission_required("project.read") @handle_request_errors async def get_project_snapshot_parameters_handler( request: web.Request, diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 7fb2bc84759..3903dd37b60 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -24,7 +24,7 @@ async def list(self, projec_uuid: UUID) -> List[SnapshotOrm]: async with self.engine.acquire() as conn: stmt = ( snapshots.select() - .where(snapshots.c.parent_uuid == projec_uuid) + .where(snapshots.c.parent_uuid == str(projec_uuid)) .order_by(snapshots.c.child_index) ) async for row in conn.execute(stmt): @@ -39,7 +39,7 @@ async def get_by_index( self, project_uuid: UUID, snapshot_index: PositiveInt ) -> Optional[SnapshotOrm]: stmt = snapshots.select().where( - (snapshots.c.parent_uuid == project_uuid) + (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.child_index == snapshot_index) ) return await self._get(stmt) @@ -48,7 +48,7 @@ async def get_by_name( self, project_uuid: UUID, snapshot_name: str ) -> Optional[SnapshotOrm]: stmt = snapshots.select().where( - (snapshots.c.parent_uuid == project_uuid) + (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.name == snapshot_name) ) return await self._get(stmt) From 503e8003d4edbe9d613018180bfb3be0c79a4885 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:39:23 +0200 Subject: [PATCH 053/137] cleanup --- .../db_base_repository.py | 25 +++++++++++++++-- .../snapshots_api_handlers.py | 28 ++++++++----------- .../snapshots_models.py | 2 +- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/db_base_repository.py b/services/web/server/src/simcore_service_webserver/db_base_repository.py index 28012be4fe0..95c960c29fb 100644 --- a/services/web/server/src/simcore_service_webserver/db_base_repository.py +++ b/services/web/server/src/simcore_service_webserver/db_base_repository.py @@ -2,11 +2,30 @@ # TODO: a repo: some members acquire and retrieve connection # TODO: a repo: any validation error in a repo is due to corrupt data in db! +from typing import Optional + from aiohttp import web +from aiopg.sa.engine import Engine -from .constants import APP_DB_ENGINE_KEY +from .constants import APP_DB_ENGINE_KEY, RQT_USERID_KEY class BaseRepository: - def __init__(self, app: web.Application): - self.engine = app[APP_DB_ENGINE_KEY] + """ + Shall be created on every request + + """ + + def __init__(self, request: web.Request): + # user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + self._engine: Engine = request.app[APP_DB_ENGINE_KEY] + self._user_id: Optional[int] = request.get(RQT_USERID_KEY) + + @property + def engine(self) -> Engine: + return self._engine + + @property + def user_id(self) -> Optional[int]: + return self._user_id diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 74cb099b9eb..f22759a84d6 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -57,6 +57,10 @@ async def wrapped(request: web.Request): return wrapped +# FIXME: access rights using same approach as in access_layer.py in storage. +# A user can only check snapshots (subresource) of its project (parent resource) + + # API ROUTES HANDLERS --------------------------------------------------------- routes = web.RouteTableDef() @@ -72,12 +76,10 @@ async def list_project_snapshots_handler(request: web.Request): """ Lists references on project snapshots """ - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - snapshots_repo = SnapshotsRepository(request.app) + snapshots_repo = SnapshotsRepository(request) @validate_arguments - async def list_snapshots(project_id: UUID) -> List[Snapshot]: + async def _list_snapshots(project_id: UUID) -> List[Snapshot]: # project_id is param-project? # TODO: add pagination # TODO: optimizaiton will grow snapshots of a project with time! @@ -89,7 +91,7 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: return [Snapshot.from_orm(obj) for obj in snapshots_orm] - snapshots: List[Snapshot] = await list_snapshots( + snapshots: List[Snapshot] = await _list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) @@ -97,8 +99,7 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: data = [] for snapshot in snapshots: - # FIXME: raw dict - data.append(SnapshotItem.from_snapshot(snapshot, request.app).dict()) + data.append(SnapshotItem.from_snapshot(snapshot, request.app)) return enveloped_response(data) @@ -111,14 +112,10 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: @permission_required("project.read") @handle_request_errors async def get_project_snapshot_handler(request: web.Request): - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - # FIXME: access rights ?? - - snapshots_repo = SnapshotsRepository(request.app) + snapshots_repo = SnapshotsRepository(request) @validate_arguments - async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: + async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: try: snapshot_orm = await snapshots_repo.get_by_index( project_id, int(snapshot_id) @@ -131,7 +128,7 @@ async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: return Snapshot.from_orm(snapshot_orm) - snapshot = await get_snapshot( + snapshot = await _get_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) @@ -145,8 +142,7 @@ async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: @permission_required("project.create") @handle_request_errors async def create_project_snapshot_handler(request: web.Request): - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - snapshots_repo = (SnapshotsRepository(request.app),) + snapshots_repo = SnapshotsRepository(request) @validate_arguments async def _create_snapshot( diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index 573b6e50154..80a3b4d7464 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Callable, Optional, Union +from typing import Optional, Union from uuid import UUID from aiohttp import web From 13fd2fca35fc2461f562da083703285e2496ce5f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:43:23 +0200 Subject: [PATCH 054/137] format oas --- api/specs/webserver/openapi-projects.yaml | 12 ++++++------ .../simcore_service_webserver/api/v0/openapi.yaml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 8a2acab3f89..260d3965ebf 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -514,12 +514,6 @@ paths: tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -527,6 +521,12 @@ paths: format: uuid name: project_id in: path + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path responses: "200": description: Successful Response diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 6a97055351d..9aec6d0e910 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13433,12 +13433,6 @@ paths: tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -13446,6 +13440,12 @@ paths: format: uuid name: project_id in: path + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path responses: '200': description: Successful Response From 3a1e080d5d2fb136adfc9ee1be3577e0ecedeba5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:45:15 +0200 Subject: [PATCH 055/137] db migration: adds snapshots table --- .../5860ac6ad178_adds_snapshots_table.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py new file mode 100644 index 00000000000..8b93a8ec5d0 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py @@ -0,0 +1,50 @@ +"""adds snapshots table + +Revision ID: 5860ac6ad178 +Revises: c2d3acc313e1 +Create Date: 2021-08-11 13:21:55.415592+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "5860ac6ad178" +down_revision = "c2d3acc313e1" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "snapshots", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("parent_uuid", sa.String(), nullable=False), + sa.Column("child_index", sa.Integer(), nullable=False), + sa.Column("project_uuid", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["parent_uuid"], + ["projects.uuid"], + name="fk_snapshots_parent_uuid_projects", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_snapshots_project_uuid_projects", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("child_index"), + sa.UniqueConstraint("project_uuid"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("snapshots") + # ### end Alembic commands ### From f390cc2d85690a16e786055df258bd148008836f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 19:00:50 +0200 Subject: [PATCH 056/137] Drops child index from snapshot --- .../src/simcore_postgres_database/errors.py | 47 +++++++++++++++++++ .../5860ac6ad178_adds_snapshots_table.py | 5 +- .../models/snapshots.py | 11 ++--- .../postgres-database/tests/test_snapshots.py | 44 ++++++++++++++--- 4 files changed, 90 insertions(+), 17 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/errors.py diff --git a/packages/postgres-database/src/simcore_postgres_database/errors.py b/packages/postgres-database/src/simcore_postgres_database/errors.py new file mode 100644 index 00000000000..4d8c8e87a53 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/errors.py @@ -0,0 +1,47 @@ +# pylint: disable=unused-import + +# +# StandardError +# |__ Warning +# |__ Error +# |__ InterfaceError +# |__ DatabaseError +# |__ DataError +# |__ OperationalError +# |__ IntegrityError +# |__ InternalError +# |__ ProgrammingError +# |__ NotSupportedError +# +# aiopg reuses DBAPI exceptions +# SEE https://aiopg.readthedocs.io/en/stable/core.html?highlight=Exception#exceptions +# SEE http://initd.org/psycopg/docs/module.html#dbapi-exceptions + + +from psycopg2 import DatabaseError, DataError +from psycopg2 import Error as DBAPIError +from psycopg2 import ( + IntegrityError, + InterfaceError, + InternalError, + NotSupportedError, + OperationalError, + ProgrammingError, +) + +# pylint: disable=no-name-in-module +from psycopg2.errors import UniqueViolation + +# pylint: enable=no-name-in-module + +assert issubclass(UniqueViolation, IntegrityError) # nosec + +# TODO: see https://stackoverflow.com/questions/58740043/how-do-i-catch-a-psycopg2-errors-uniqueviolation-error-in-a-python-flask-app +# from sqlalchemy.exc import IntegrityError +# +# from psycopg2.errors import UniqueViolation +# +# try: +# s.commit() +# except IntegrityError as e: +# assert isinstance(e.orig, UniqueViolation) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py index 8b93a8ec5d0..23699028a2a 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py @@ -23,7 +23,6 @@ def upgrade(): sa.Column("name", sa.String(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("parent_uuid", sa.String(), nullable=False), - sa.Column("child_index", sa.Integer(), nullable=False), sa.Column("project_uuid", sa.String(), nullable=False), sa.ForeignKeyConstraint( ["parent_uuid"], @@ -38,8 +37,10 @@ def upgrade(): ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("child_index"), sa.UniqueConstraint("project_uuid"), + sa.UniqueConstraint( + "parent_uuid", "created_at", name="snapshot_from_project_uniqueness" + ), ) # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index f2dda1ce4a4..db5ac215031 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -33,14 +33,6 @@ unique=False, doc="UUID of the parent project", ), - sa.Column( - "child_index", - sa.Integer, - nullable=False, - unique=True, - doc="0-based index in order of creation (i.e. 0 being the oldest and N-1 the latest)" - "from the same parent_id", - ), sa.Column( "project_uuid", sa.String, @@ -53,6 +45,9 @@ unique=True, doc="UUID of the project associated to this snapshot", ), + sa.UniqueConstraint( + "parent_uuid", "created_at", name="snapshot_from_project_uniqueness" + ), ) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 12879727072..95d943c3557 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -4,13 +4,14 @@ # pylint: disable=unused-variable from copy import deepcopy -from typing import Optional +from typing import Callable, Optional, Set from uuid import UUID, uuid3 import pytest from aiopg.sa.engine import Engine from aiopg.sa.result import ResultProxy, RowProxy from pytest_simcore.helpers.rawdata_fakers import random_project, random_user +from simcore_postgres_database.errors import UniqueViolation from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.snapshots import snapshots from simcore_postgres_database.models.users import users @@ -36,8 +37,9 @@ async def engine(pg_engine: Engine): yield pg_engine -async def test_creating_snapshots(engine: Engine): - exclude = { +@pytest.fixture +def exclude(): + return { "id", "uuid", "creation_date", @@ -46,8 +48,11 @@ async def test_creating_snapshots(engine: Engine): "published", } + +@pytest.fixture +def create_snapshot(exclude) -> Callable: async def _create_snapshot(child_index: int, parent_prj, conn) -> int: - # NOTE: used as prototype + # NOTE: used as FAKE prototype # create project-snapshot prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} @@ -80,13 +85,19 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: name=f"Snapshot {child_index} [{parent_prj.name}]", created_at=parent_prj.last_change_date, parent_uuid=parent_prj.uuid, - child_index=child_index, project_uuid=project_uuid, ) .returning(snapshots.c.id) ) return snapshot_id + return _create_snapshot + + +async def test_creating_snapshots( + engine: Engine, create_snapshot: Callable, exclude: Set +): + async with engine.acquire() as conn: # get parent res: ResultProxy = await conn.execute( @@ -97,7 +108,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert parent_prj # take one snapshot - first_snapshot_id = await _create_snapshot(0, parent_prj, conn) + first_snapshot_id = await create_snapshot(0, parent_prj, conn) # modify parent updated_parent_prj = await ( @@ -115,7 +126,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date # take another snapshot - second_snapshot_id = await _create_snapshot(1, updated_parent_prj, conn) + second_snapshot_id = await create_snapshot(1, updated_parent_prj, conn) second_snapshot = await ( await conn.execute( @@ -150,6 +161,25 @@ def extract(t): # return existing +async def test_multiple_snapshots_of_same_project( + engine: Engine, create_snapshot: Callable +): + async with engine.acquire() as conn: + # get parent + res: ResultProxy = await conn.execute( + projects.select().where(projects.c.name == PARENT_PROJECT_NAME) + ) + parent_prj: Optional[RowProxy] = await res.first() + assert parent_prj + + # take first snapshot + await create_snapshot(0, parent_prj, conn) + + # no changes in the parent! + with pytest.raises(UniqueViolation): + await create_snapshot(1, parent_prj, conn) + + def test_deleting_snapshots(): # test delete child project -> deletes snapshot # test delete snapshot -> deletes child project From 7eca2144342c0918236fecba3245f9fa97fe9776 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 19:51:36 +0200 Subject: [PATCH 057/137] implements create snapshot --- .../snapshots_api_handlers.py | 46 +++++++++++--- .../snapshots_core.py | 49 +++++++++++---- .../simcore_service_webserver/snapshots_db.py | 63 ++++++++++++++----- 3 files changed, 123 insertions(+), 35 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index f22759a84d6..c4ce9cdc3bd 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -11,8 +11,11 @@ from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from .login.decorators import login_required +from .projects import projects_api +from .projects.projects_exceptions import ProjectNotFoundError from .security_decorators import permission_required -from .snapshots_db import SnapshotsRepository +from .snapshots_core import ProjectDict, take_snapshot +from .snapshots_db import ProjectsRepository, SnapshotsRepository from .snapshots_models import Snapshot, SnapshotItem @@ -54,6 +57,11 @@ async def wrapped(request: web.Request): content_type="application/json", ) from err + except ProjectNotFoundError as err: + raise web.HTTPNotFound( + reason=f"Project not found {err.project_uuid} or not accessible. Skipping snapshot" + ) from err + return wrapped @@ -94,13 +102,9 @@ async def _list_snapshots(project_id: UUID) -> List[Snapshot]: snapshots: List[Snapshot] = await _list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) - # TODO: async for snapshot in await list_snapshot is the same? - data = [] - for snapshot in snapshots: - data.append(SnapshotItem.from_snapshot(snapshot, request.app)) - + data = [SnapshotItem.from_snapshot(snp, request.app) for snp in snapshots] return enveloped_response(data) @@ -143,13 +147,41 @@ async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: @handle_request_errors async def create_project_snapshot_handler(request: web.Request): snapshots_repo = SnapshotsRepository(request) + projects_repo = ProjectsRepository(request) + user_id = request[RQT_USERID_KEY] @validate_arguments async def _create_snapshot( project_id: UUID, snapshot_label: Optional[str] = None, ) -> Snapshot: - raise NotImplementedError + + snapshot_orm = None + if snapshot_label: + snapshot_orm = snapshots_repo.get_by_name(project_id, snapshot_label) + + if not snapshot_orm: + parent: ProjectDict = await projects_api.get_project_for_user( + request.app, + str(project_id), + user_id, + include_templates=False, + include_state=False, + ) + + # pylint: disable=unused-variable + project: ProjectDict + snapshot: Snapshot + project, snapshot = await take_snapshot( + parent, + snapshot_label=snapshot_label, + ) + + # FIXME: Atomic?? project and snapshot shall be created in the same transaction!! + await projects_repo.create(project) + snapshot_orm = await snapshots_repo.create(snapshot.dict()) + + return Snapshot.from_orm(snapshot_orm) snapshot = await _create_snapshot( project_id=request.match_info["project_id"], # type: ignore diff --git a/services/web/server/src/simcore_service_webserver/snapshots_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py index 9a77d8dd7be..5bd450cb6ef 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -8,15 +8,17 @@ # -from typing import Iterator, Tuple +from typing import Any, Dict, Iterator, Optional, Tuple from uuid import UUID, uuid3 from models_library.projects_nodes import Node +from .projects import projects_utils from .projects.projects_db import ProjectAtDB -from .projects.projects_utils import clone_project_document from .snapshots_models import Snapshot +ProjectDict = Dict[str, Any] + def is_parametrized(node: Node) -> bool: try: @@ -35,20 +37,45 @@ def is_parametrized_project(project: ProjectAtDB) -> bool: return any(is_parametrized(node) for node in project.workbench.values()) -def snapshot_project(parent: ProjectAtDB, snapshot_label: str): +async def take_snapshot( + parent: ProjectDict, + snapshot_label: Optional[str] = None, +) -> Tuple[ProjectDict, Snapshot]: + + assert ProjectAtDB.parse_obj(parent) # nosec + + # FIXME: + # if is_parametrized_project(parent): + # raise NotImplementedError( + # "Only non-parametrized projects can be snapshot right now" + # ) - if is_parametrized_project(parent): - raise NotImplementedError( - "Only non-parametrized projects can be snapshot right now" - ) + # Clones parent's project document + snapshot_timestamp = parent["last_change_date"] - project, nodes_map = clone_project_document( - parent.dict(), - forced_copy_project_id=str(uuid3(namespace=parent.uuid, name=snapshot_label)), + child: ProjectDict + child, nodes_map = projects_utils.clone_project_document( + project=parent, + forced_copy_project_id=uuid3( + UUID(parent["uuid"]), f"snapshot.{snapshot_timestamp}" + ), ) + assert child # nosec assert nodes_map # nosec + assert ProjectAtDB.parse_obj(child) # nosec + + child["name"] += snapshot_label or f" [snapshot {snapshot_timestamp}]" + # creation_data = state of parent upon copy! WARNING: changes can be state changes and not project definition? + child["creation_date"] = parent["last_change_date"] + child["hidden"] = True + child["published"] = False snapshot = Snapshot( - id, label=snapshot_label, parent_id=parent.id, project_id=project.id + name=f"Snapshot {snapshot_timestamp} [{parent['name']}]", + created_at=snapshot_timestamp, + parent_uuid=parent["uuid"], + project_uuid=child["uuid"], ) + + return (child, snapshot) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 3903dd37b60..ccb6f2c7f42 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -1,15 +1,26 @@ -from typing import List, Optional +from typing import Dict, List, Optional from uuid import UUID +from aiohttp import web from aiopg.sa.result import RowProxy from pydantic import PositiveInt from simcore_postgres_database.models.snapshots import snapshots +from simcore_service_catalog.services.access_rights import OLD_SERVICES_DATE +from simcore_service_director_v2.modules.dynamic_sidecar.docker_compose_specs import ( + BASE_SERVICE_SPEC, +) +from simcore_service_webserver.snapshots_models import Snapshot from .db_base_repository import BaseRepository +from .projects.projects_db import APP_PROJECT_DBAPI # alias for readability # SEE https://pydantic-docs.helpmanual.io/usage/models/#orm-mode-aka-arbitrary-class-instances SnapshotOrm = RowProxy +SnapshotDict = Dict + +ProjectOrm = RowProxy +ProjectDict = Dict class SnapshotsRepository(BaseRepository): @@ -19,39 +30,57 @@ class SnapshotsRepository(BaseRepository): Gets primitive/standard parameters and returns valid orm objects """ - async def list(self, projec_uuid: UUID) -> List[SnapshotOrm]: - result = [] + async def list( + self, project_uuid: UUID, limit: Optional[int] = None + ) -> List[SnapshotOrm]: + """ Returns sorted list of snapshots in project""" + # TODO: add pagination + async with self.engine.acquire() as conn: - stmt = ( + query = ( snapshots.select() - .where(snapshots.c.parent_uuid == str(projec_uuid)) - .order_by(snapshots.c.child_index) + .where(snapshots.c.parent_uuid == str(project_uuid)) + .order_by(snapshots.c.id) ) - async for row in conn.execute(stmt): - result.append(row) - return result + if limit and limit > 0: + query = query.limit(limit) + + return await (await conn.execute(query)).fetchall() - async def _get(self, stmt) -> Optional[SnapshotOrm]: + async def _first(self, query) -> Optional[SnapshotOrm]: async with self.engine.acquire() as conn: - return await (await conn.execute(stmt)).first() + return await (await conn.execute(query)).first() async def get_by_index( self, project_uuid: UUID, snapshot_index: PositiveInt ) -> Optional[SnapshotOrm]: - stmt = snapshots.select().where( + query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.child_index == snapshot_index) ) - return await self._get(stmt) + return await self._first(query) async def get_by_name( self, project_uuid: UUID, snapshot_name: str ) -> Optional[SnapshotOrm]: - stmt = snapshots.select().where( + query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.name == snapshot_name) ) - return await self._get(stmt) + return await self._first(query) + + async def create(self, snapshot: SnapshotDict) -> SnapshotOrm: + # pylint: disable=no-value-for-parameter + query = snapshots.insert().values(**snapshot).returning(snapshots) + row = await self._first(query) + assert row # nosec + return row + + +class ProjectsRepository(BaseRepository): + def __init__(self, request: web.Request): + super().__init__(request) + self._dbapi = request.config_dict[APP_PROJECT_DBAPI] - async def create(self, project_uuid: UUID) -> SnapshotOrm: - pass + async def create(self, project: ProjectDict): + await self._dbapi.add_project(project, self.user_id, force_project_uuid=True) From 113ec3714c781bb21293f7684870639f4ce90ad2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 21:40:30 +0200 Subject: [PATCH 058/137] auto-doc --- .../doc/img/postgres-database-models.svg | 169 +++++++++--------- 1 file changed, 83 insertions(+), 86 deletions(-) diff --git a/packages/postgres-database/doc/img/postgres-database-models.svg b/packages/postgres-database/doc/img/postgres-database-models.svg index 368427dacb8..38603b2e51a 100644 --- a/packages/postgres-database/doc/img/postgres-database-models.svg +++ b/packages/postgres-database/doc/img/postgres-database-models.svg @@ -361,64 +361,64 @@ comp_runs - -comp_runs - -run_id - [BIGINT] - -project_uuid - [VARCHAR] - -user_id - [BIGINT] - -iteration - [BIGINT] - -result - [VARCHAR(11)] - -created - [DATETIME] - -modified - [DATETIME] - -started - [DATETIME] - -ended - [DATETIME] + +comp_runs + +run_id + [BIGINT] + +project_uuid + [VARCHAR] + +user_id + [BIGINT] + +iteration + [BIGINT] + +result + [VARCHAR(11)] + +created + [DATETIME] + +modified + [DATETIME] + +started + [DATETIME] + +ended + [DATETIME] users--comp_runs - -{0,1} -0..N + +{0,1} +0..N user_to_projects - -user_to_projects - -id - [BIGINT] - -user_id - [BIGINT] - -project_id - [BIGINT] + +user_to_projects + +id + [BIGINT] + +user_id + [BIGINT] + +project_id + [BIGINT] users--user_to_projects - -{0,1} -0..N + +{0,1} +0..N @@ -587,45 +587,42 @@ study_tags - -study_tags - -study_id - [BIGINT] - -tag_id - [BIGINT] + +study_tags + +study_id + [BIGINT] + +tag_id + [BIGINT] projects--study_tags - -{0,1} -0..N + +{0,1} +0..N snapshots - -snapshots - -id - [BIGINT] - -name - [VARCHAR] - -created_at - [DATETIME] - -parent_uuid - [VARCHAR] - -child_index - [INTEGER] - -project_uuid - [VARCHAR] + +snapshots + +id + [BIGINT] + +name + [VARCHAR] + +created_at + [DATETIME] + +parent_uuid + [VARCHAR] + +project_uuid + [VARCHAR] @@ -644,23 +641,23 @@ projects--comp_runs - -{0,1} -0..N + +{0,1} +0..N projects--user_to_projects - -{0,1} -0..N + +{0,1} +0..N tags--study_tags - -{0,1} -0..N + +{0,1} +0..N From 47379692206c95900884fb4a961ff3ed9a130b13 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:14:02 +0200 Subject: [PATCH 059/137] fixes dependency --- packages/postgres-database/requirements/ci.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/postgres-database/requirements/ci.txt b/packages/postgres-database/requirements/ci.txt index 8fc611bc091..b12bf394c2e 100644 --- a/packages/postgres-database/requirements/ci.txt +++ b/packages/postgres-database/requirements/ci.txt @@ -9,7 +9,6 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt ---requirement _pydantic.txt --requirement _test.txt # installs this repo's packages From 625b796d7519fe8b2ca443af91aac48f030374d9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:31:29 +0200 Subject: [PATCH 060/137] fixes linter on too many statements --- .../src/servicelib/application_setup.py | 28 +++++++++++-------- .../tests/test_application_setup.py | 1 - 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 89377797728..09f651f0d7a 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -32,6 +32,18 @@ class DependencyError(ApplicationSetupError): pass +def _is_app_module_enabled(cfg: Dict, parts: List[str], section) -> bool: + # navigates app_config (cfg) searching for section + for part in parts: + if section and part == "enabled": + # if section exists, no need to explicitly enable it + cfg = cfg.get(part, True) + else: + cfg = cfg[part] + assert isinstance(cfg, bool) # nosec + return cfg + + def app_module_setup( module_name: str, category: ModuleCategory, @@ -57,7 +69,7 @@ def app_module_setup( :param config_enabled: option in config to enable, defaults to None which is '$(module-section).enabled' (config_section and config_enabled are mutually exclusive) :raises DependencyError :raises ApplicationSetupError - :return: False if setup was skipped + :return: True if setup was completed or False if setup was skipped :rtype: bool :Example: @@ -112,18 +124,10 @@ def setup_wrapper(app: web.Application, *args, **kargs) -> bool: # TODO: sometimes section is optional, check in config schema cfg = app[APP_CONFIG_KEY] - def _get(cfg_, parts): - for part in parts: - if ( - section and part == "enabled" - ): # if section exists, no need to explicitly enable it - cfg_ = cfg_.get(part, True) - else: - cfg_ = cfg_[part] - return cfg_ - try: - is_enabled = _get(cfg, config_enabled.split(".")) + is_enabled = _is_app_module_enabled( + cfg, config_enabled.split("."), section + ) except KeyError as ee: raise ApplicationSetupError( f"Cannot find required option '{config_enabled}' in app config's section '{ee}'" diff --git a/packages/service-library/tests/test_application_setup.py b/packages/service-library/tests/test_application_setup.py index a37b6268c17..8ca80c37dc0 100644 --- a/packages/service-library/tests/test_application_setup.py +++ b/packages/service-library/tests/test_application_setup.py @@ -2,7 +2,6 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import logging from typing import Dict from unittest.mock import Mock From 02433aba19c0e72cec2b9b457747f889f99f8c45 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:38:32 +0200 Subject: [PATCH 061/137] fixes wrong imports --- .../web/server/src/simcore_service_webserver/snapshots_db.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index ccb6f2c7f42..c8d206990f2 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -5,11 +5,6 @@ from aiopg.sa.result import RowProxy from pydantic import PositiveInt from simcore_postgres_database.models.snapshots import snapshots -from simcore_service_catalog.services.access_rights import OLD_SERVICES_DATE -from simcore_service_director_v2.modules.dynamic_sidecar.docker_compose_specs import ( - BASE_SERVICE_SPEC, -) -from simcore_service_webserver.snapshots_models import Snapshot from .db_base_repository import BaseRepository from .projects.projects_db import APP_PROJECT_DBAPI From b15889e56d76ad56e4f7d8a5dd648e4640a624a2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 1 Jul 2021 17:37:22 +0200 Subject: [PATCH 062/137] Deprecates displayOrder --- packages/models-library/src/models_library/services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 84bd5a2b811..205f2ecd64d 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -283,12 +283,14 @@ class ServiceKeyVersion(BaseModel): "simcore/services/comp/itis/sleeper", "simcore/services/dynamic/3dviewer", ], + regex=KEY_RE, ) version: str = Field( ..., description="service version number", regex=VERSION_RE, examples=["1.0.0", "0.0.1"], + regex=VERSION_RE, ) From f5bdb485f1fb1635b582cad34d6cc05b933b3fc4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 01:11:30 +0200 Subject: [PATCH 063/137] Adding dev-feature flag and cleanup --- .../src/simcore_service_webserver/projects/module_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/module_setup.py b/services/web/server/src/simcore_service_webserver/projects/module_setup.py index d5d6e38f3c4..c723f4e8abe 100644 --- a/services/web/server/src/simcore_service_webserver/projects/module_setup.py +++ b/services/web/server/src/simcore_service_webserver/projects/module_setup.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -def _create_routes(tag, specs, *handlers_module, disable_login=False): +def _create_routes(tag, specs, *handlers_module, disable_login: bool = False): """ :param disable_login: Disables login_required decorator for testing purposes defaults to False :type disable_login: bool, optional From c2236a2d130df6639c6cedb2f15374611c108203 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 01:15:25 +0200 Subject: [PATCH 064/137] Adds module with setup and handlers --- .../parametrization.py | 32 ++++ .../parametrization_api_handlers.py | 141 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 services/web/server/src/simcore_service_webserver/parametrization.py create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py new file mode 100644 index 00000000000..96614a3451e --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -0,0 +1,32 @@ +""" parametrization app module setup + + - Project parametrization + - Project snapshots + +""" +import logging + +from aiohttp import web +from servicelib.application_setup import ModuleCategory, app_module_setup + +from . import parametrization_api_handlers +from .constants import APP_SETTINGS_KEY +from .settings import ApplicationSettings + +log = logging.getLogger(__name__) + + +@app_module_setup( + __name__, + ModuleCategory.ADDON, + depends=["simcore_service_webserver.projects"], + logger=log, +) +def setup(app: web.Application): + + settings: ApplicationSettings = app[APP_SETTINGS_KEY] + if not settings.WEBSERVER_DEV_FEATURES_ENABLED: + log.warning("App module %s disabled: Marked as dev feature", __name__) + return False + + app.add_routes(parametrization_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py new file mode 100644 index 00000000000..eabf39feda0 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -0,0 +1,141 @@ +import json +from functools import wraps +from typing import Any, Callable, Dict, List +from uuid import UUID + +from aiohttp import web +from aiohttp.web_routedef import RouteDef +from pydantic.decorator import validate_arguments +from pydantic.error_wrappers import ValidationError + +from ._meta import api_version_prefix +from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY + +json_dumps = json.dumps + + +def handle_request_errors(handler: Callable): + """ + - required and type validation of path and query parameters + """ + + @wraps(handler) + async def wrapped(request: web.Request): + try: + resp = await handler(request) + return resp + + except KeyError as err: + # NOTE: handles required request.match_info[*] or request.query[*] + raise web.HTTPBadRequest(reason="Expected parameter {err}") from err + + except ValidationError as err: + # NOTE: pydantic.validate_arguments parses and validates -> ValidationError + raise web.HTTPUnprocessableEntity( + text=json_dumps({"error": err.errors()}), + content_type="application/json", + ) from err + + return wrapped + + +# API ROUTES HANDLERS --------------------------------------------------------- +routes = web.RouteTableDef() + + +@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots") +@handle_request_errors +async def _list_project_snapshots_handler(request: web.Request): + """ + Lists references on project snapshots + """ + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + snapshots = await list_project_snapshots( + project_id=request.match_info["project_id"], # type: ignore + ) + + # Append url links + url_for_snapshot = request.app.router["_get_project_snapshot_handler"].url_for + url_for_parameters = request.app.router[ + "_get_project_snapshot_parameters_handler" + ].url_for + + for snp in snapshots: + snp["url"] = url_for_snapshot( + project_id=snp["parent_id"], snapshot_id=snp["id"] + ) + snp["url_parameters"] = url_for_parameters( + project_id=snp["parent_id"], snapshot_id=snp["id"] + ) + + return snapshots + + +@validate_arguments +async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: + # project_id is param-project? + + snapshot_info_0 = { + "id": "0", + "display_name": "snapshot 0", + "parent_id": project_id, + "parameters": get_project_snapshot_parameters(project_id, snapshot_id="0"), + } + return [ + snapshot_info_0, + ] + + +@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}") +@handle_request_errors +async def _get_project_snapshot_handler(request: web.Request): + """ + Returns full project. Equivalent to /projects/{snapshot_project_id} + """ + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + await get_project_snapshot( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + +@validate_arguments +async def get_project_snapshot(project_id: UUID, snapshot_id: str): + pass + + +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters" +) +@handle_request_errors +async def _get_project_snapshot_parameters_handler( + request: web.Request, +): + # GET /projects/{id}/snapshots/{id}/parametrization -> {x:3, y:0, ...} + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + params = await get_project_snapshot_parameters( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + return params + + +@validate_arguments +async def get_project_snapshot_parameters( + project_id: UUID, snapshot_id: str +) -> Dict[str, Any]: + return {"x": 4, "y": "yes"} + + +# ------------------------------------- +assert routes # nosec + +# NOTE: names all routes with handler's +# TODO: override routes functions ? +for route_def in routes: + assert isinstance(route_def, RouteDef) # nosec + route_def.kwargs["name"] = route_def.handler.__name__ From dd820363be506592f53199b9636bf40889bcc8ca Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 02:12:44 +0200 Subject: [PATCH 065/137] WIP [skip ci] --- .../parametrization.py | 2 +- .../parametrization_api_handlers.py | 20 ++++++-- .../parametrization_core.py | 8 ++++ .../parametrization_models.py | 8 ++++ .../parametrization_settings.py | 1 + .../isolated/test_parametrization_core.py | 47 +++++++++++++++++++ 6 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_core.py create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_models.py create mode 100644 services/web/server/src/simcore_service_webserver/parametrization_settings.py create mode 100644 services/web/server/tests/unit/isolated/test_parametrization_core.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index 96614a3451e..5d53b8968c4 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -26,7 +26,7 @@ def setup(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: - log.warning("App module %s disabled: Marked as dev feature", __name__) + log.warning("App module '%s' is disabled: Marked as dev feature", __name__) return False app.add_routes(parametrization_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index eabf39feda0..67881ea5c57 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -5,6 +5,7 @@ from aiohttp import web from aiohttp.web_routedef import RouteDef +from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError @@ -75,13 +76,21 @@ async def _list_project_snapshots_handler(request: web.Request): @validate_arguments async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: # project_id is param-project? - + # TODO: add pagination + # TODO: optimizaiton will grow snapshots of a project with time! + # + + # snapshots: + # - ordered (iterations!) + # - have a parent project with all the parametrization + # snapshot_info_0 = { - "id": "0", + "id": 0, "display_name": "snapshot 0", "parent_id": project_id, - "parameters": get_project_snapshot_parameters(project_id, snapshot_id="0"), + "parameters": get_project_snapshot_parameters(project_id, snapshot_id=str(id)), } + return [ snapshot_info_0, ] @@ -103,7 +112,10 @@ async def _get_project_snapshot_handler(request: web.Request): @validate_arguments async def get_project_snapshot(project_id: UUID, snapshot_id: str): - pass + # TODO: create a fake project + # - generate project_id + # - define what changes etc... + project = Project() @routes.get( diff --git a/services/web/server/src/simcore_service_webserver/parametrization_core.py b/services/web/server/src/simcore_service_webserver/parametrization_core.py new file mode 100644 index 00000000000..c1493f33326 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_core.py @@ -0,0 +1,8 @@ +# +# +# Parameter node needs to be evaluated before the workflow is submitted +# Every evaluation creates a snapshot +# +# Constant Param has 1 input adn one output +# Optimizer can produce semi-cyclic feedback loop by connecting to output of target to +# diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/parametrization_models.py new file mode 100644 index 00000000000..7167e18fccf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_models.py @@ -0,0 +1,8 @@ +from typing import Union + +from pydantic import BaseModel + + +class Parameter(BaseModel): + name: str + value: Union[bool, float, str] diff --git a/services/web/server/src/simcore_service_webserver/parametrization_settings.py b/services/web/server/src/simcore_service_webserver/parametrization_settings.py new file mode 100644 index 00000000000..d3b6b48b51c --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/parametrization_settings.py @@ -0,0 +1 @@ +# TODO: do not enable diff --git a/services/web/server/tests/unit/isolated/test_parametrization_core.py b/services/web/server/tests/unit/isolated/test_parametrization_core.py new file mode 100644 index 00000000000..4d34dde0e0e --- /dev/null +++ b/services/web/server/tests/unit/isolated/test_parametrization_core.py @@ -0,0 +1,47 @@ +from typing import Iterator, Tuple +from uuid import uuid4 + +import pytest +from models_library.projects import Project, ProjectAtDB +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID +from simcore_service_webserver.projects.projects_api import get_project_for_user + + +# is parametrized project? +def is_param(node: Node): + return node.key.split("/")[-2] == "param" + + +def is_parametrized_project(project: Project): + return any(is_param(node) for node in project.workbench.values()) + + +def iter_params(project: Project) -> Iterator[Tuple[NodeID, Node]]: + for node_id, node in project.workbench.items(): + if is_param(node): + yield NodeID(node_id), node + + +def is_const_param(node: Node) -> bool: + return is_param(node) and "const" in node.key and not node.inputs + + +async def test_it(app): + + project_id = str(uuid4()) + user_id = 0 + + prj_dict = await get_project_for_user( + app, project_id, user_id, include_templates=False, include_state=True + ) + + parent_project = Project.parse_obj(prj_dict) + + # three types of "parametrization" nodes + + # const -> replace in connecting nodes + + # generators -> iterable data sources (no inputs connected!) + + # feedback -> parametrizations with connected inputs! From b63b77d258b2e0466ad8c40c36e58963657d5d23 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 03:00:09 +0200 Subject: [PATCH 066/137] adds skipsetup exception in servicelib --- .../src/servicelib/application_setup.py | 42 ++++++++++++------- .../tests/test_application_setup.py | 27 ++++++++++-- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 2b75d859222..4b01f3b50b4 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -18,6 +18,12 @@ class ModuleCategory(Enum): ADDON = 1 +class SkipModuleSetupException(Exception): + def __init__(self, *, reason) -> None: + self.reason = reason + super().__init__(reason) + + class ApplicationSetupError(Exception): pass @@ -33,9 +39,9 @@ def app_module_setup( depends: Optional[List[str]] = None, config_section: str = None, config_enabled: str = None, - logger: Optional[logging.Logger] = None, + logger: logging.Logger = log, ) -> Callable: - """ Decorator that marks a function as 'a setup function' for a given module in an application + """Decorator that marks a function as 'a setup function' for a given module in an application - Marks a function as 'setup' of a given module in an application - Ensures setup executed ONLY ONCE per app @@ -76,8 +82,6 @@ def setup(app: web.Application): # if passes config_enabled, invalidates info on section section = None - logger = logger or log - def decorate(setup_func): if "setup" not in setup_func.__name__: @@ -147,17 +151,26 @@ def _get(cfg_, parts): raise ApplicationSetupError(msg) # execution of setup - ok = setup_func(app, *args, **kargs) + try: + completed = setup_func(app, *args, **kargs) - # post-setup - if ok is None: - ok = True + # post-setup + if completed is None: + completed = True - if ok: - app[APP_SETUP_KEY].append(module_name) + if completed: + app[APP_SETUP_KEY].append(module_name) + else: + raise SkipModuleSetupException(reason="Undefined") - logger.debug("'%s' setup completed [%s]", module_name, ok) - return ok + except SkipModuleSetupException as exc: + logger.warning("Skipping '%s' setup: %s", module_name, exc.reason) + completed = False + + logger.debug( + "'%s' setup %s", module_name, "completed" if completed else "skipped" + ) + return completed setup_wrapper.metadata = setup_metadata setup_wrapper.MARK = "setup" @@ -170,10 +183,9 @@ def _get(cfg_, parts): def is_setup_function(fun): return ( inspect.isfunction(fun) - and hasattr(fun, "MARK") - and fun.MARK == "setup" + and getattr(fun, "MARK", None) == "setup" and any( param.annotation == web.Application - for name, param in inspect.signature(fun).parameters.items() + for _, param in inspect.signature(fun).parameters.items() ) ) diff --git a/packages/service-library/tests/test_application_setup.py b/packages/service-library/tests/test_application_setup.py index a003074862d..7c4c9febdf9 100644 --- a/packages/service-library/tests/test_application_setup.py +++ b/packages/service-library/tests/test_application_setup.py @@ -4,6 +4,7 @@ import logging from typing import Dict +from unittest.mock import Mock import pytest from aiohttp import web @@ -12,19 +13,22 @@ APP_SETUP_KEY, DependencyError, ModuleCategory, + SkipModuleSetupException, app_module_setup, ) -log = logging.getLogger(__name__) +log = Mock() @app_module_setup("package.bar", ModuleCategory.ADDON, logger=log) -def setup_bar(app: web.Application, arg1, kargs=55): +def setup_bar(app: web.Application, arg1, *, raise_skip: bool = False): return True @app_module_setup("package.foo", ModuleCategory.ADDON, logger=log) -def setup_foo(app: web.Application, arg1, kargs=33): +def setup_foo(app: web.Application, arg1, kargs=33, *, raise_skip: bool = False): + if raise_skip: + raise SkipModuleSetupException(reason="explicit skip") return True @@ -92,3 +96,20 @@ def test_marked_setup(app_config, app): app_config["foo"]["enabled"] = False assert not setup_foo(app, 2) + + +def test_skip_setup(app_config, app): + try: + log.reset_mock() + + assert not setup_foo(app, 1, raise_skip=True) + + # FIXME: mock logger + # assert log.warning.called + # warn_msg = log.warning.call_args()[0] + # assert "package.foo" in warn_msg + # assert "explicit skip" in warn_msg + + assert setup_foo(app, 1) + finally: + log.reset_mock() From c70d372d02c8e1f01897fac640212f0cee9baa4f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 03:01:18 +0200 Subject: [PATCH 067/137] Uses skip setup and skips tests --- .../src/simcore_service_webserver/parametrization.py | 9 ++++++--- .../tests/unit/isolated/test_parametrization_core.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index 5d53b8968c4..a1022a28c68 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -7,7 +7,11 @@ import logging from aiohttp import web -from servicelib.application_setup import ModuleCategory, app_module_setup +from servicelib.application_setup import ( + ModuleCategory, + SkipModuleSetupException, + app_module_setup, +) from . import parametrization_api_handlers from .constants import APP_SETTINGS_KEY @@ -26,7 +30,6 @@ def setup(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: - log.warning("App module '%s' is disabled: Marked as dev feature", __name__) - return False + raise SkipModuleSetupException(reason="Development feature") app.add_routes(parametrization_api_handlers.routes) diff --git a/services/web/server/tests/unit/isolated/test_parametrization_core.py b/services/web/server/tests/unit/isolated/test_parametrization_core.py index 4d34dde0e0e..ca759ba987e 100644 --- a/services/web/server/tests/unit/isolated/test_parametrization_core.py +++ b/services/web/server/tests/unit/isolated/test_parametrization_core.py @@ -27,6 +27,7 @@ def is_const_param(node: Node) -> bool: return is_param(node) and "const" in node.key and not node.inputs +@pytest.mark.skip(reason="UNDER DEV") async def test_it(app): project_id = str(uuid4()) From 035018f5507b3687c4b6190d3e09d7211f1f95e1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 9 Jul 2021 03:05:39 +0200 Subject: [PATCH 068/137] name-mania --- .../service-library/src/servicelib/application_setup.py | 6 +++--- packages/service-library/tests/test_application_setup.py | 4 ++-- .../server/src/simcore_service_webserver/parametrization.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 4b01f3b50b4..89377797728 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -18,7 +18,7 @@ class ModuleCategory(Enum): ADDON = 1 -class SkipModuleSetupException(Exception): +class SkipModuleSetup(Exception): def __init__(self, *, reason) -> None: self.reason = reason super().__init__(reason) @@ -161,9 +161,9 @@ def _get(cfg_, parts): if completed: app[APP_SETUP_KEY].append(module_name) else: - raise SkipModuleSetupException(reason="Undefined") + raise SkipModuleSetup(reason="Undefined") - except SkipModuleSetupException as exc: + except SkipModuleSetup as exc: logger.warning("Skipping '%s' setup: %s", module_name, exc.reason) completed = False diff --git a/packages/service-library/tests/test_application_setup.py b/packages/service-library/tests/test_application_setup.py index 7c4c9febdf9..a37b6268c17 100644 --- a/packages/service-library/tests/test_application_setup.py +++ b/packages/service-library/tests/test_application_setup.py @@ -13,7 +13,7 @@ APP_SETUP_KEY, DependencyError, ModuleCategory, - SkipModuleSetupException, + SkipModuleSetup, app_module_setup, ) @@ -28,7 +28,7 @@ def setup_bar(app: web.Application, arg1, *, raise_skip: bool = False): @app_module_setup("package.foo", ModuleCategory.ADDON, logger=log) def setup_foo(app: web.Application, arg1, kargs=33, *, raise_skip: bool = False): if raise_skip: - raise SkipModuleSetupException(reason="explicit skip") + raise SkipModuleSetup(reason="explicit skip") return True diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index a1022a28c68..00c18833ac6 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -9,7 +9,7 @@ from aiohttp import web from servicelib.application_setup import ( ModuleCategory, - SkipModuleSetupException, + SkipModuleSetup, app_module_setup, ) @@ -30,6 +30,6 @@ def setup(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: - raise SkipModuleSetupException(reason="Development feature") + raise SkipModuleSetup(reason="Development feature") app.add_routes(parametrization_api_handlers.routes) From e3159e264947c4ccb8f7941db8b1701c3d55306b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sun, 11 Jul 2021 17:08:49 +0200 Subject: [PATCH 069/137] WIP --- api/specs/webserver/openapi-projects.yaml | 47 ++++++ api/specs/webserver/openapi.yaml | 9 ++ .../parametrization_api_handlers.py | 8 +- .../catalog_services_openapi_generator.py | 4 +- .../sandbox/projects_openapi_generator.py | 136 ++++++++++++++++++ 5 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 services/web/server/tests/sandbox/projects_openapi_generator.py diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 3f8f909783a..1a66a9011cc 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -435,6 +435,53 @@ paths: default: $ref: "#/components/responses/DefaultErrorResponse" + projects_snapshots: + get: + summary: List Project Snapshots + parameters: + - in: path + name: project_id + required: true + schema: + format: uuid + title: Project Id + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + + projects_id_snapshots_id: + get: + summary: Get Project Snapshot + parameters: + - in: path + name: project_id + required: true + schema: + format: uuid + title: Project Id + type: string + - in: path + name: snapshot_id + required: true + schema: + exclusiveMinimum: 0.0 + title: Snapshot Id + type: integer + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + + projects_id_snapshots_id_parameters: + get: + + components: schemas: ClientSessionId: diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index fa8af63df8f..518bf6b64a9 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -210,6 +210,15 @@ paths: /projects/{study_uuid}/tags/{tag_id}: $ref: "./openapi-projects.yaml#/paths/~1projects~1{study_uuid}~1tags~1{tag_id}" + /projects/{project_id}/snapshots: + $ref: "./openapi-projects.yaml#/paths/projects_snapshots" + + /projects/{project_id}/snapshots/{snapshot_id}: + $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id" + + /projects/{project_id}/snapshots/{snapshot_id}/parameters: + $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id_parameters" + # ACTIVITY ------------------------------------------------------------------------- /activity/status: $ref: "./openapi-activity.yaml#/paths/~1activity~1status" diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index 67881ea5c57..4f5cbb68836 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -69,6 +69,7 @@ async def _list_project_snapshots_handler(request: web.Request): snp["url_parameters"] = url_for_parameters( project_id=snp["parent_id"], snapshot_id=snp["id"] ) + # snp['url_project'] = return snapshots @@ -104,18 +105,20 @@ async def _get_project_snapshot_handler(request: web.Request): """ user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - await get_project_snapshot( + prj_dict = await get_project_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) + return prj_dict # ??? @validate_arguments -async def get_project_snapshot(project_id: UUID, snapshot_id: str): +async def get_project_snapshot(project_id: UUID, snapshot_id: str) -> Dict[str, Any]: # TODO: create a fake project # - generate project_id # - define what changes etc... project = Project() + return project.dict() @routes.get( @@ -140,6 +143,7 @@ async def _get_project_snapshot_parameters_handler( async def get_project_snapshot_parameters( project_id: UUID, snapshot_id: str ) -> Dict[str, Any]: + # return {"x": 4, "y": "yes"} diff --git a/services/web/server/tests/sandbox/catalog_services_openapi_generator.py b/services/web/server/tests/sandbox/catalog_services_openapi_generator.py index dd42a98c42e..5d8b4c29dc8 100644 --- a/services/web/server/tests/sandbox/catalog_services_openapi_generator.py +++ b/services/web/server/tests/sandbox/catalog_services_openapi_generator.py @@ -1,9 +1,7 @@ # pylint: disable=unused-argument import json -import sys -from pathlib import Path -from typing import List, Optional +from typing import List from fastapi import FastAPI, Query from simcore_service_webserver.catalog_api_models import ( diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py new file mode 100644 index 00000000000..ad0c8704588 --- /dev/null +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -0,0 +1,136 @@ +# +# assist creating OAS for projects resource +# + +import json +import uuid +from datetime import datetime +from typing import Any, Dict, Optional, Union +from uuid import UUID + +import yaml +from fastapi import FastAPI, status +from pydantic import BaseModel, Field, PositiveInt +from pydantic.networks import AnyUrl + +app = FastAPI() + +error_responses = { + status.HTTP_400_BAD_REQUEST: {}, + status.HTTP_422_UNPROCESSABLE_ENTITY: {}, +} + +BuiltinTypes = Union[bool, int, float, str] +DataSchema = Union[ + BuiltinTypes, +] # any json schema? +DataLink = AnyUrl + +DataSchema = Union[DataSchema, DataLink] + + +class Node(BaseModel): + key: str + version: str = Field(..., regex=r"\d+\.\d+\.\d+") + label: str + + inputs: Dict[str, DataSchema] + # var inputs? + outputs: Dict[str, DataSchema] + # var outputs? + + +class Project(BaseModel): + id: UUID + pipeline: Dict[UUID, Node] + + +class ProjectSnapshot(BaseModel): + id: int + label: str + parent_project_id: UUID + parameters: Dict[str, Any] = {} + + taken_at: Optional[datetime] = Field( + None, + description="Timestamp of the time snapshot was taken", + ) + + +@app.get("/projects/{project_id}") +def get_project(project_id: UUID): + pass + + +@app.post("/projects/{project_id}") +def create_project(project: Project): + pass + + +@app.put("/projects/{project_id}") +def replace_project(project_id: UUID, project: Project): + pass + + +@app.patch("/projects/{project_id}") +def update_project(project_id: UUID, project: Project): + pass + + +@app.delete("/projects/{project_id}") +def delete_project(project_id: UUID): + pass + + +@app.post("/projects/{project_id}:open") +def open_project(project: Project): + pass + + +@app.post("/projects/{project_id}:start") +def start_project(project: Project): + pass + + +@app.post("/projects/{project_id}:stop") +def stop_project(project: Project): + pass + + +@app.post("/projects/{project_id}:close") +def close_project(project: Project): + pass + + +# ------------- + + +@app.get("/projects/{project_id}/snapshots") +async def list_project_snapshots(project_id: UUID): + pass + + +@app.post("/projects/{project_id}/snapshots") +async def create_project_snapshot( + project_id: UUID, snapshot_label: Optional[str] = None +): + pass + + +@app.get("/projects/{project_id}/snapshots/{snapshot_id}") +async def get_project_snapshot(project_id: UUID, snapshot_id: PositiveInt): + pass + + +@app.get("/projects/{project_id}/snapshots/{snapshot_id}/parameters") +async def get_project_snapshot_parameters(project_id: UUID, snapshot_id: str): + return {"x": 4, "y": "yes"} + + +# print(yaml.safe_dump(app.openapi())) +# print("-"*100) + + +print(json.dumps(app.openapi(), indent=2)) + +# uvicorn --reload projects_openapi_generator:app From d96598f58bafe7c07088219a547d63a66b26e55b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 12 Jul 2021 19:29:42 +0200 Subject: [PATCH 070/137] WIP --- .../sandbox/projects_openapi_generator.py | 294 +++++++++++++++--- 1 file changed, 247 insertions(+), 47 deletions(-) diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py index ad0c8704588..58467dddc1f 100644 --- a/services/web/server/tests/sandbox/projects_openapi_generator.py +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -1,16 +1,27 @@ # -# assist creating OAS for projects resource +# Assists on the creation of project's OAS # import json -import uuid +from collections import defaultdict from datetime import datetime -from typing import Any, Dict, Optional, Union -from uuid import UUID - -import yaml -from fastapi import FastAPI, status -from pydantic import BaseModel, Field, PositiveInt +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from uuid import UUID, uuid3, uuid4 + +from fastapi import Depends, FastAPI +from fastapi import Path as PathParam +from fastapi import Request, status +from fastapi.exceptions import HTTPException +from models_library.services import PROPERTY_KEY_RE +from pydantic import ( + BaseModel, + Field, + PositiveInt, + StrictBool, + StrictFloat, + StrictInt, + constr, +) from pydantic.networks import AnyUrl app = FastAPI() @@ -20,7 +31,11 @@ status.HTTP_422_UNPROCESSABLE_ENTITY: {}, } -BuiltinTypes = Union[bool, int, float, str] + +InputID = OutputID = constr(regex=PROPERTY_KEY_RE) + +# WARNING: oder matters +BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] DataSchema = Union[ BuiltinTypes, ] # any json schema? @@ -34,9 +49,9 @@ class Node(BaseModel): version: str = Field(..., regex=r"\d+\.\d+\.\d+") label: str - inputs: Dict[str, DataSchema] + inputs: Dict[InputID, DataSchema] # var inputs? - outputs: Dict[str, DataSchema] + outputs: Dict[OutputID, DataSchema] # var outputs? @@ -44,87 +59,272 @@ class Project(BaseModel): id: UUID pipeline: Dict[UUID, Node] + def update_ids(self, name: str): + map_ids: Dict[UUID, UUID] = {} + map_ids[self.id] = uuid3(self.id, name) + map_ids.update({node_id: uuid3(node_id, name) for node_id in self.pipeline}) + + # replace ALL references + + +class Parameter(BaseModel): + name: str + value: BuiltinTypes + + node_id: UUID + output_id: OutputID -class ProjectSnapshot(BaseModel): - id: int - label: str - parent_project_id: UUID - parameters: Dict[str, Any] = {} - taken_at: Optional[datetime] = Field( - None, - description="Timestamp of the time snapshot was taken", +class Snapshot(BaseModel): + id: PositiveInt = Field(..., description="Unique snapshot identifier") + label: Optional[str] = Field(None, description="Unique human readable display name") + created_at: datetime = Field( + default_factory=datetime.utcnow, + description="Timestamp of the time snapshot was taken from parent. Notice that parent might change with time", ) + parent_id: UUID = Field(..., description="Parent's project uuid") + project_id: UUID = Field(..., description="Current project's uuid") + + +class ParameterApiModel(Parameter): + url: AnyUrl + # url_output: AnyUrl + + +class SnapshotApiModel(Snapshot): + url: AnyUrl + url_parent: AnyUrl + url_project: AnyUrl + url_parameters: Optional[AnyUrl] = None + + @classmethod + def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiModel": + return cls( + url=url_for( + "get_snapshot", + project_id=snapshot.project_id, + snapshot_id=snapshot.id, + ), + url_parent=url_for("get_project", project_id=snapshot.parent_id), + url_project=url_for("get_project", project_id=snapshot.project_id), + url_parameters=url_for( + "get_snapshot_parameters", + project_id=snapshot.parent_id, + snapshot_id=snapshot.id, + ), + **snapshot.dict(), + ) + + +#################################################################### -@app.get("/projects/{project_id}") -def get_project(project_id: UUID): - pass + +_PROJECTS: Dict[UUID, Project] = {} +_PROJECT2SNAPSHOT: Dict[UUID, UUID] = {} +_SNAPSHOTS: Dict[UUID, List[Snapshot]] = defaultdict(list) +_PARAMETERS: Dict[Tuple[UUID, int], List[Parameter]] = defaultdict(list) + + +#################################################################### + + +def get_reverse_url_mapper(request: Request) -> Callable: + def reverse_url_mapper(name: str, **path_params: Any) -> str: + return request.url_for(name, **path_params) + + return reverse_url_mapper + + +def get_valid_id(project_id: UUID = PathParam(...)) -> UUID: + if project_id not in _PROJECTS: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid id") + return project_id + + +#################################################################### + + +@app.get("/projects/{project_id}", response_model=Project) +def get_project(pid: UUID = Depends(get_valid_id)): + return _PROJECTS[pid] @app.post("/projects/{project_id}") def create_project(project: Project): - pass + if project.id not in _PROJECTS: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Invalid id") + _PROJECTS[project.id] = project @app.put("/projects/{project_id}") -def replace_project(project_id: UUID, project: Project): - pass +def replace_project(project: Project, pid: UUID = Depends(get_valid_id)): + _PROJECTS[pid] = project @app.patch("/projects/{project_id}") -def update_project(project_id: UUID, project: Project): - pass +def update_project(project: Project, pid: UUID = Depends(get_valid_id)): + raise NotImplementedError() @app.delete("/projects/{project_id}") -def delete_project(project_id: UUID): - pass +def delete_project(pid: UUID = Depends(get_valid_id)): + del _PROJECTS[pid] @app.post("/projects/{project_id}:open") -def open_project(project: Project): +def open_project(pid: UUID = Depends(get_valid_id)): pass @app.post("/projects/{project_id}:start") -def start_project(project: Project): +def start_project(use_cache: bool = True, pid: UUID = Depends(get_valid_id)): pass @app.post("/projects/{project_id}:stop") -def stop_project(project: Project): +def stop_project(pid: UUID = Depends(get_valid_id)): pass @app.post("/projects/{project_id}:close") -def close_project(project: Project): +def close_project(pid: UUID = Depends(get_valid_id)): pass -# ------------- +@app.get("/projects/{project_id}/snapshots", response_model=List[SnapshotApiModel]) +async def list_snapshots( + pid: UUID = Depends(get_valid_id), + url_for: Callable = Depends(get_reverse_url_mapper), +): + psid = _PROJECT2SNAPSHOT.get(pid) + if not psid: + return [] + project_snapshots: List[Snapshot] = _SNAPSHOTS.get(psid, []) -@app.get("/projects/{project_id}/snapshots") -async def list_project_snapshots(project_id: UUID): - pass + return [ + SnapshotApiModel.from_snapshot(snapshot, url_for) + for snapshot in project_snapshots + ] -@app.post("/projects/{project_id}/snapshots") -async def create_project_snapshot( - project_id: UUID, snapshot_label: Optional[str] = None +@app.post("/projects/{project_id}/snapshots", response_model=SnapshotApiModel) +async def create_snapshot( + pid: UUID = Depends(get_valid_id), + snapshot_label: Optional[str] = None, + url_for: Callable = Depends(get_reverse_url_mapper), ): - pass + # + # copies project and creates project_id + # run will use "use_cache" + # snapshots already in place -@app.get("/projects/{project_id}/snapshots/{snapshot_id}") -async def get_project_snapshot(project_id: UUID, snapshot_id: PositiveInt): - pass + project_snapshots: List[SnapshotApiModel] = await list_snapshots(pid, url_for) + index = project_snapshots[-1].id if len(project_snapshots) else 0 + + if snapshot_label: + if any(s.label == snapshot_label for s in project_snapshots): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"'{snapshot_label}' already exist", + ) + + else: + snapshot_label = f"snapshot {index}" + while any(s.label == snapshot_label for s in project_snapshots): + index += 1 + snapshot_label = f"snapshot {index}" + + # perform snapshot + parent_project = _PROJECTS[pid] + # create new project + project_id = uuid3(namespace=parent_project.id, name=snapshot_label) + project = parent_project.copy(update={"id": project_id}) # THIS IS WRONG -@app.get("/projects/{project_id}/snapshots/{snapshot_id}/parameters") -async def get_project_snapshot_parameters(project_id: UUID, snapshot_id: str): - return {"x": 4, "y": "yes"} + snapshot = Snapshot(id=index, parent_id=pid, project_id=project_id) + + _PROJECTS[project_id] = project + + psid = _PROJECT2SNAPSHOT.setdefault(pid, uuid3(pid, name="snapshots")) + _SNAPSHOTS[psid].append(snapshot) + + # if param-project, then call workflow-compiler to produce parameters + # differenciate between snapshots created automatically from those explicit! + + return SnapshotApiModel( + url=url_for( + "get_snapshot", project_id=snapshot.parent_id, snapshot_id=snapshot.id + ), + **snapshot.dict(), + ) + + +@app.get( + "/projects/{project_id}/snapshots/{snapshot_id}", + response_model=SnapshotApiModel, +) +async def get_snapshot( + snapshot_id: PositiveInt, + pid: UUID = Depends(get_valid_id), + url_for: Callable = Depends(get_reverse_url_mapper), +): + + psid = _PROJECT2SNAPSHOT[pid] + snapshot = next(s for s in _SNAPSHOTS[psid] if s.id == snapshot_id) + + return SnapshotApiModel( + url=url_for( + "get_snapshot", project_id=snapshot.parent_id, snapshot_id=snapshot.id + ), + **snapshot.dict(), + ) + + +@app.get( + "/projects/{project_id}/snapshots/{snapshot_id}/parameters", + response_model=List[ParameterApiModel], +) +async def list_snapshot_parameters( + snapshot_id: str, + pid: UUID = Depends(get_valid_id), + url_for: Callable = Depends(get_reverse_url_mapper), +): + + # get param snapshot + params = {"x": 4, "y": "yes"} + + result = [ + ParameterApiModel( + name=name, + value=value, + node_id=uuid4(), + output_id="output", + url=url_for( + "list_snapshot_parameters", + project_id=pid, + snapshot_id=snapshot_id, + ), + ) + for name, value in params.items() + ] + + return result + + +## workflow compiler ####################################### + + +def create_snapshots(project_id: UUID): + # get project + + # if parametrized + # iterate + # otherwise + # copy workbench and replace uuids + pass # print(yaml.safe_dump(app.openapi())) From 8de856de58389369fb0427f88d3d607e7a0577f5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 13 Jul 2021 16:57:08 +0200 Subject: [PATCH 071/137] WIP --- .../models/snapshots.py | 38 ++++++ .../parametrization_api_handlers.py | 129 ++++++++++-------- .../parametrization_core.py | 46 +++++++ .../parametrization_models.py | 64 ++++++++- 4 files changed, 216 insertions(+), 61 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/snapshots.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py new file mode 100644 index 00000000000..cb551201d79 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -0,0 +1,38 @@ +import sqlalchemy as sa +from sqlalchemy.sql import func + +from .base import metadata + +snapshots = sa.Table( + "snapshots", + metadata, + sa.Column( + "id", + sa.BigInteger, + nullable=False, + primary_key=True, + doc="Global snapshot identifier index", + ), + sa.Column("name", sa.String, nullable=False, doc="Display name"), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=func.now(), + doc="Timestamp on creation", + ), + sa.Column( + "parent_uuid", + sa.String, + nullable=False, + unique=True, + doc="Parent project's UUID", + ), + sa.Column( + "project_uuid", + sa.String, + nullable=False, + unique=True, + doc="UUID of the project associated to this snapshot", + ), +) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index 4f5cbb68836..c4614d1dadd 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -1,16 +1,17 @@ import json from functools import wraps -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, List, Optional from uuid import UUID from aiohttp import web -from aiohttp.web_routedef import RouteDef from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError +from simcore_service_webserver.parametrization_models import Snapshot from ._meta import api_version_prefix from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY +from .parametrization_models import Snapshot, SnapshotApiModel json_dumps = json.dumps @@ -44,23 +45,24 @@ async def wrapped(request: web.Request): routes = web.RouteTableDef() -@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots") +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + name="_list_snapshots_handler", +) @handle_request_errors -async def _list_project_snapshots_handler(request: web.Request): +async def _list_snapshots_handler(request: web.Request): """ Lists references on project snapshots """ user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - snapshots = await list_project_snapshots( + snapshots = await list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) # Append url links - url_for_snapshot = request.app.router["_get_project_snapshot_handler"].url_for - url_for_parameters = request.app.router[ - "_get_project_snapshot_parameters_handler" - ].url_for + url_for_snapshot = request.app.router["_get_snapshot_handler"].url_for + url_for_parameters = request.app.router["_get_snapshot_parameters_handler"].url_for for snp in snapshots: snp["url"] = url_for_snapshot( @@ -74,8 +76,57 @@ async def _list_project_snapshots_handler(request: web.Request): return snapshots +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}", + name="_get_snapshot_handler", +) +@handle_request_errors +async def _get_snapshot_handler(request: web.Request): + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + snapshot = await get_snapshot( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + return snapshot.json() + + +@routes.post( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + name="_create_snapshot_handler", +) +@handle_request_errors +async def _create_snapshot_handler(request: web.Request): + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + snapshot = await create_snapshot( + project_id=request.match_info["project_id"], # type: ignore + snapshot_label=request.query.get("snapshot_label"), + ) + + return snapshot.json() + + +@routes.get( + f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", + name="_get_snapshot_parameters_handler", +) +@handle_request_errors +async def _get_snapshot_parameters_handler( + request: web.Request, +): + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + params = await get_snapshot_parameters( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + return params + + @validate_arguments -async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: +async def list_snapshots(project_id: UUID) -> List[Dict[str, Any]]: # project_id is param-project? # TODO: add pagination # TODO: optimizaiton will grow snapshots of a project with time! @@ -89,7 +140,7 @@ async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: "id": 0, "display_name": "snapshot 0", "parent_id": project_id, - "parameters": get_project_snapshot_parameters(project_id, snapshot_id=str(id)), + "parameters": get_snapshot_parameters(project_id, snapshot_id=str(id)), } return [ @@ -97,61 +148,23 @@ async def list_project_snapshots(project_id: UUID) -> List[Dict[str, Any]]: ] -@routes.get(f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}") -@handle_request_errors -async def _get_project_snapshot_handler(request: web.Request): - """ - Returns full project. Equivalent to /projects/{snapshot_project_id} - """ - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - prj_dict = await get_project_snapshot( - project_id=request.match_info["project_id"], # type: ignore - snapshot_id=request.match_info["snapshot_id"], - ) - return prj_dict # ??? - - @validate_arguments -async def get_project_snapshot(project_id: UUID, snapshot_id: str) -> Dict[str, Any]: +async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: # TODO: create a fake project # - generate project_id # - define what changes etc... - project = Project() - return project.dict() - + pass -@routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters" -) -@handle_request_errors -async def _get_project_snapshot_parameters_handler( - request: web.Request, -): - # GET /projects/{id}/snapshots/{id}/parametrization -> {x:3, y:0, ...} - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - params = await get_project_snapshot_parameters( - project_id=request.match_info["project_id"], # type: ignore - snapshot_id=request.match_info["snapshot_id"], - ) - - return params +@validate_arguments +async def create_snapshot( + project_id: UUID, + snapshot_label: Optional[str] = None, +) -> Snapshot: + pass @validate_arguments -async def get_project_snapshot_parameters( - project_id: UUID, snapshot_id: str -) -> Dict[str, Any]: +async def get_snapshot_parameters(project_id: UUID, snapshot_id: str): # return {"x": 4, "y": "yes"} - - -# ------------------------------------- -assert routes # nosec - -# NOTE: names all routes with handler's -# TODO: override routes functions ? -for route_def in routes: - assert isinstance(route_def, RouteDef) # nosec - route_def.kwargs["name"] = route_def.handler.__name__ diff --git a/services/web/server/src/simcore_service_webserver/parametrization_core.py b/services/web/server/src/simcore_service_webserver/parametrization_core.py index c1493f33326..fead73ab024 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_core.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_core.py @@ -6,3 +6,49 @@ # Constant Param has 1 input adn one output # Optimizer can produce semi-cyclic feedback loop by connecting to output of target to # + + +from typing import Iterator, Tuple +from uuid import UUID, uuid3 + +from models_library.projects_nodes import Node + +from .parametrization_models import Snapshot +from .projects.projects_db import ProjectAtDB +from .projects.projects_utils import clone_project_document + + +def is_parametrized(node: Node) -> bool: + try: + return "parameter" == node.key.split("/")[-2] + except IndexError: + return False + + +def iter_param_nodes(project: ProjectAtDB) -> Iterator[Tuple[UUID, Node]]: + for node_id, node in project.workbench.items(): + if is_parametrized(node): + yield UUID(node_id), node + + +def is_parametrized_project(project: ProjectAtDB) -> bool: + return any(is_parametrized(node) for node in project.workbench.values()) + + +def snapshot_project(parent: ProjectAtDB, snapshot_label: str): + + if is_parametrized_project(parent): + raise NotImplementedError( + "Only non-parametrized projects can be snapshot right now" + ) + + project, nodes_map = clone_project_document( + parent.dict(), + forced_copy_project_id=str(uuid3(namespace=parent.uuid, name=snapshot_label)), + ) + + assert nodes_map # nosec + + snapshot = Snapshot( + id, label=snapshot_label, parent_id=parent.id, project_id=project.id + ) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/parametrization_models.py index 7167e18fccf..e9d13bf2391 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_models.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_models.py @@ -1,8 +1,66 @@ -from typing import Union +from datetime import datetime +from typing import Callable, Optional, Union +from uuid import UUID -from pydantic import BaseModel +from models_library.projects_nodes import OutputID +from pydantic import ( + AnyUrl, + BaseModel, + Field, + PositiveInt, + StrictBool, + StrictFloat, + StrictInt, +) + +BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] class Parameter(BaseModel): name: str - value: Union[bool, float, str] + value: BuiltinTypes + + node_id: UUID + output_id: OutputID + + +class Snapshot(BaseModel): + id: PositiveInt = Field(..., description="Unique snapshot identifier") + label: Optional[str] = Field(None, description="Unique human readable display name") + created_at: datetime = Field( + default_factory=datetime.utcnow, + description="Timestamp of the time snapshot was taken from parent. Notice that parent might change with time", + ) + + parent_id: UUID = Field(..., description="Parent's project uuid") + project_id: UUID = Field(..., description="Current project's uuid") + + +class ParameterApiModel(Parameter): + url: AnyUrl + # url_output: AnyUrl + + +class SnapshotApiModel(Snapshot): + url: AnyUrl + url_parent: AnyUrl + url_project: AnyUrl + url_parameters: Optional[AnyUrl] = None + + @classmethod + def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiModel": + return cls( + url=url_for( + "get_snapshot", + project_id=snapshot.project_id, + snapshot_id=snapshot.id, + ), + url_parent=url_for("get_project", project_id=snapshot.parent_id), + url_project=url_for("get_project", project_id=snapshot.project_id), + url_parameters=url_for( + "get_snapshot_parameters", + project_id=snapshot.parent_id, + snapshot_id=snapshot.id, + ), + **snapshot.dict(), + ) From 1020ee230950a6308d3f325aefa03cf9cea7b674 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 14 Jul 2021 17:03:24 +0200 Subject: [PATCH 072/137] WIP --- .../parametrization_api_handlers.py | 3 + .../isolated/test_parametrization_core.py | 48 --------- .../with_dbs/11/test_parametrization_core.py | 100 ++++++++++++++++++ 3 files changed, 103 insertions(+), 48 deletions(-) delete mode 100644 services/web/server/tests/unit/isolated/test_parametrization_core.py create mode 100644 services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index c4614d1dadd..6d6dc3e5b29 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -125,6 +125,9 @@ async def _get_snapshot_parameters_handler( return params +# API ROUTES HANDLERS --------------------------------------------------------- + + @validate_arguments async def list_snapshots(project_id: UUID) -> List[Dict[str, Any]]: # project_id is param-project? diff --git a/services/web/server/tests/unit/isolated/test_parametrization_core.py b/services/web/server/tests/unit/isolated/test_parametrization_core.py deleted file mode 100644 index ca759ba987e..00000000000 --- a/services/web/server/tests/unit/isolated/test_parametrization_core.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Iterator, Tuple -from uuid import uuid4 - -import pytest -from models_library.projects import Project, ProjectAtDB -from models_library.projects_nodes import Node -from models_library.projects_nodes_io import NodeID -from simcore_service_webserver.projects.projects_api import get_project_for_user - - -# is parametrized project? -def is_param(node: Node): - return node.key.split("/")[-2] == "param" - - -def is_parametrized_project(project: Project): - return any(is_param(node) for node in project.workbench.values()) - - -def iter_params(project: Project) -> Iterator[Tuple[NodeID, Node]]: - for node_id, node in project.workbench.items(): - if is_param(node): - yield NodeID(node_id), node - - -def is_const_param(node: Node) -> bool: - return is_param(node) and "const" in node.key and not node.inputs - - -@pytest.mark.skip(reason="UNDER DEV") -async def test_it(app): - - project_id = str(uuid4()) - user_id = 0 - - prj_dict = await get_project_for_user( - app, project_id, user_id, include_templates=False, include_state=True - ) - - parent_project = Project.parse_obj(prj_dict) - - # three types of "parametrization" nodes - - # const -> replace in connecting nodes - - # generators -> iterable data sources (no inputs connected!) - - # feedback -> parametrizations with connected inputs! diff --git a/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py b/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py new file mode 100644 index 00000000000..a1060d1b51c --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py @@ -0,0 +1,100 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from typing import Dict, Iterator, Tuple +from uuid import UUID, uuid4 + +import pytest +from aiohttp import web +from models_library.projects import Project, ProjectAtDB +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID +from simcore_service_webserver.constants import APP_PROJECT_DBAPI +from simcore_service_webserver.parametrization_core import snapshot_project +from simcore_service_webserver.projects.projects_api import get_project_for_user +from simcore_service_webserver.projects.projects_db import APP_PROJECT_DBAPI +from simcore_service_webserver.projects.projects_utils import clone_project_document + + +# is parametrized project? +def is_param(node: Node): + return node.key.split("/")[-2] == "param" + + +def is_parametrized_project(project: Project): + return any(is_param(node) for node in project.workbench.values()) + + +def iter_params(project: Project) -> Iterator[Tuple[NodeID, Node]]: + for node_id, node in project.workbench.items(): + if is_param(node): + yield NodeID(node_id), node + + +def is_const_param(node: Node) -> bool: + return is_param(node) and "const" in node.key and not node.inputs + + +@pytest.fixture() +def app(): + pass + + +@pytest.fixture +def project_id() -> UUID: + return uuid4() + + +@pytest.fixture +def user_id() -> int: + return 1 + + +async def test_snapshot_standard_project( + app: web.Application, project_id: UUID, user_id: int +): + + prj_dict: Dict = await get_project_for_user( + app, str(project_id), user_id, include_templates=False, include_state=False + ) + + # validates API project data + project = Project.parse_obj(prj_dict) + + cloned_prj_dict = clone_project_document(prj_dict) + + cloned_project = Project.parse_obj(cloned_prj_dict) + + # project_snapshot = snapshot_project(standard_project) + # assert isinstance(project_snapshot, Project) + assert cloned_project.uuid != project.uuid + + projects_repo = app[APP_PROJECT_DBAPI] + + new_project = await projects_repo.replace_user_project( + new_project, user_id, project_uuid, include_templates=True + ) + + # have same pipeline but different node uuids + # assert project_snapshot. + + # removes states: snapshots is always un-run?? + + # three types of "parametrization" nodes + + # const -> replace in connecting nodes + + # generators -> iterable data sources (no inputs connected!) + + # feedback -> parametrizations with connected inputs! + + +# +# - I have a project +# - Take a snapshot +# - if the project is standard -> clone +# - if the project is parametrized +# - clone parametrized? +# - evaluate param-nodes? +# From a433cf9af2b497fe930f8453563b56836d77ddf3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 11:30:06 +0200 Subject: [PATCH 073/137] minor --- .../src/simcore_postgres_database/models/snapshots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index cb551201d79..da3ab54e97c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -26,7 +26,7 @@ sa.String, nullable=False, unique=True, - doc="Parent project's UUID", + doc="UUID of the parent project", ), sa.Column( "project_uuid", From eb46369084cf639a54fa204142fea41d9c3c96a0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 15:50:22 +0200 Subject: [PATCH 074/137] cleanup fixtures --- packages/postgres-database/tests/conftest.py | 16 ++++++++++------ .../tests/test_delete_projects_and_users.py | 15 ++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/postgres-database/tests/conftest.py b/packages/postgres-database/tests/conftest.py index 97876a06453..0c1b8571acd 100644 --- a/packages/postgres-database/tests/conftest.py +++ b/packages/postgres-database/tests/conftest.py @@ -42,7 +42,7 @@ def postgres_service(docker_services, docker_ip, docker_compose_file) -> str: def make_engine(postgres_service: str) -> Callable: dsn = postgres_service - def maker(is_async=True) -> Union[Coroutine, Callable]: + def maker(*, is_async=True) -> Union[Coroutine, Callable]: return aiopg.sa.create_engine(dsn) if is_async else sa.create_engine(dsn) return maker @@ -68,18 +68,22 @@ def db_metadata(): @pytest.fixture async def pg_engine(loop, make_engine, db_metadata) -> Engine: - engine = await make_engine() + async_engine = await make_engine(is_async=True) # TODO: upgrade/downgrade - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) + # NOTE: ALL is deleted before db_metadata.drop_all(sync_engine) db_metadata.create_all(sync_engine) - yield engine + yield async_engine - engine.terminate() - await engine.wait_closed() + # closes async-engine connections and terminates + async_engine.close() + await async_engine.wait_closed() + async_engine.terminate() + # NOTE: ALL is deleted after db_metadata.drop_all(sync_engine) sync_engine.dispose() diff --git a/packages/postgres-database/tests/test_delete_projects_and_users.py b/packages/postgres-database/tests/test_delete_projects_and_users.py index 4b69974fbfe..4cf85bbfa0b 100644 --- a/packages/postgres-database/tests/test_delete_projects_and_users.py +++ b/packages/postgres-database/tests/test_delete_projects_and_users.py @@ -7,21 +7,17 @@ import pytest import sqlalchemy as sa +from aiopg.sa.engine import Engine from aiopg.sa.result import ResultProxy, RowProxy from psycopg2.errors import ForeignKeyViolation # pylint: disable=no-name-in-module from pytest_simcore.helpers.rawdata_fakers import random_project, random_user -from simcore_postgres_database.models.base import metadata from simcore_postgres_database.webserver_models import projects, users @pytest.fixture -async def engine(make_engine, loop): - engine = await make_engine() - sync_engine = make_engine(False) - metadata.drop_all(sync_engine) - metadata.create_all(sync_engine) +async def engine(pg_engine: Engine): - async with engine.acquire() as conn: + async with pg_engine.acquire() as conn: await conn.execute(users.insert().values(**random_user(name="A"))) await conn.execute(users.insert().values(**random_user())) await conn.execute(users.insert().values(**random_user())) @@ -32,10 +28,7 @@ async def engine(make_engine, loop): with pytest.raises(ForeignKeyViolation): await conn.execute(projects.insert().values(**random_project(prj_owner=4))) - yield engine - - engine.close() - await engine.wait_closed() + yield pg_engine @pytest.mark.skip(reason="sandbox for dev purposes") From a71761c6e91e8ef44d7c751c5cf9202da54ff019 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 15:53:29 +0200 Subject: [PATCH 075/137] cleanup fixtures --- packages/postgres-database/tests/test_groups.py | 8 ++++---- .../tests/test_uniqueness_in_comp_tasks.py | 2 +- services/api-server/tests/unit/conftest.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/postgres-database/tests/test_groups.py b/packages/postgres-database/tests/test_groups.py index 68854b0d6a0..e7ac3975be7 100644 --- a/packages/postgres-database/tests/test_groups.py +++ b/packages/postgres-database/tests/test_groups.py @@ -41,7 +41,7 @@ async def _create_user(conn, name: str, group: RowProxy) -> RowProxy: async def test_user_group_uniqueness(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) @@ -68,7 +68,7 @@ async def test_user_group_uniqueness(make_engine): async def test_all_group(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) async with engine.acquire() as conn: @@ -119,7 +119,7 @@ async def test_all_group(make_engine): async def test_own_group(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) async with engine.acquire() as conn: @@ -165,7 +165,7 @@ async def test_own_group(make_engine): async def test_group(make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) async with engine.acquire() as conn: diff --git a/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py b/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py index 25a7fd228a0..89e3d5c5573 100644 --- a/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py +++ b/packages/postgres-database/tests/test_uniqueness_in_comp_tasks.py @@ -19,7 +19,7 @@ async def engine(loop, make_engine): engine = await make_engine() - sync_engine = make_engine(False) + sync_engine = make_engine(is_async=False) metadata.drop_all(sync_engine) metadata.create_all(sync_engine) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index af7c57c404f..38933ad5c05 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -174,7 +174,7 @@ def is_postgres_responsive() -> bool: def make_engine(postgres_service: Dict) -> Callable: dsn = postgres_service["dsn"] # session scope freezes dsn - def maker(is_async=True) -> Union[aiopg_sa_engine.Engine, sa_engine.Engine]: + def maker(*, is_async=True) -> Union[aiopg_sa_engine.Engine, sa_engine.Engine]: if is_async: return aiopg.sa.create_engine(dsn) return sa.create_engine(dsn) @@ -198,7 +198,7 @@ def apply_migration(postgres_service: Dict, make_engine) -> Iterator[None]: pg_cli.downgrade.callback("base") pg_cli.clean.callback() # FIXME: deletes all because downgrade is not reliable! - engine = make_engine(False) + engine = make_engine(is_async=False) metadata.drop_all(engine) From 18ee075c72a98be63b9b0a5dac359f8ce7c53d65 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 18:47:15 +0200 Subject: [PATCH 076/137] cleanup and minor fix --- packages/models-library/src/models_library/projects.py | 8 ++++++-- packages/models-library/src/models_library/services.py | 2 -- .../tests/test_delete_projects_and_users.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 2d960b77af6..667382097d7 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -66,17 +66,21 @@ class ProjectCommons(BaseModel): @validator("thumbnail", always=True, pre=True) @classmethod - def convert_empty_str_to_none(v): + def convert_empty_str_to_none(cls, v): if isinstance(v, str) and v == "": return None return v class ProjectAtDB(ProjectCommons): - # specific DB fields + # Model used to READ from database + id: int = Field(..., description="The table primary index") + project_type: ProjectType = Field(..., alias="type", description="The project type") + prj_owner: Optional[int] = Field(..., description="The project owner id") + published: Optional[bool] = Field( False, description="Defines if a study is available publicly" ) diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 205f2ecd64d..84bd5a2b811 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -283,14 +283,12 @@ class ServiceKeyVersion(BaseModel): "simcore/services/comp/itis/sleeper", "simcore/services/dynamic/3dviewer", ], - regex=KEY_RE, ) version: str = Field( ..., description="service version number", regex=VERSION_RE, examples=["1.0.0", "0.0.1"], - regex=VERSION_RE, ) diff --git a/packages/postgres-database/tests/test_delete_projects_and_users.py b/packages/postgres-database/tests/test_delete_projects_and_users.py index 4cf85bbfa0b..cddcac6c0e2 100644 --- a/packages/postgres-database/tests/test_delete_projects_and_users.py +++ b/packages/postgres-database/tests/test_delete_projects_and_users.py @@ -1,7 +1,7 @@ # pylint: disable=no-value-for-parameter -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable from typing import List From 4f2dea857221d936d81aef24528f32a59de4d5ff Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 21:52:12 +0200 Subject: [PATCH 077/137] adds snapshot table with minimal test --- .../models/snapshots.py | 22 ++- .../postgres-database/tests/test_snapshots.py | 141 ++++++++++++++++++ .../pytest_simcore/helpers/rawdata_fakers.py | 1 + 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 packages/postgres-database/tests/test_snapshots.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index da3ab54e97c..8cce020e2f5 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -24,15 +24,35 @@ sa.Column( "parent_uuid", sa.String, + sa.ForeignKey( + "projects.uuid", + name="fk_snapshots_parent_uuid_projects", + ondelete="CASCADE", + ), nullable=False, - unique=True, doc="UUID of the parent project", ), + sa.Column( + "child_index", + sa.Integer, + nullable=False, + unique=True, + doc="Number of child (as 0-based index: 0 being the oldest, 1, ...)" + "from the same parent_id", + ), sa.Column( "project_uuid", sa.String, + sa.ForeignKey( + "projects.uuid", + name="fk_snapshots_project_uuid_projects", + ondelete="CASCADE", + ), nullable=False, unique=True, doc="UUID of the project associated to this snapshot", ), ) + + +# Snapshot : convert_to_pydantic(snapshot) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py new file mode 100644 index 00000000000..e03a8d889a8 --- /dev/null +++ b/packages/postgres-database/tests/test_snapshots.py @@ -0,0 +1,141 @@ +# pylint: disable=no-value-for-parameter +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from copy import deepcopy +from typing import Optional +from uuid import UUID, uuid3 + +import pytest +from aiopg.sa.engine import Engine +from aiopg.sa.result import ResultProxy, RowProxy +from pytest_simcore.helpers.rawdata_fakers import random_project, random_user +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.snapshots import snapshots +from simcore_postgres_database.models.users import users + + +@pytest.fixture +async def engine(pg_engine: Engine): + # injects + async with pg_engine.acquire() as conn: + # a 'me' user + user_id = await conn.scalar( + users.insert().values(**random_user(name="me")).returning(users.c.id) + ) + # has a project 'parent' + await conn.execute( + projects.insert().values(**random_project(prj_owner=user_id, name="parent")) + ) + yield pg_engine + + +async def test_creating_snapshots(engine: Engine): + async def _create_snapshot(child_index: int, parent_prj, conn) -> int: + # copy + # change uuid, and set to invisible + exclude = { + "id", + "uuid", + "creation_date", + "last_change_date", + "hidden", + "published", + } + prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} + + prj_dict["name"] += f" [snapshot {child_index}]" + prj_dict["uuid"] = uuid3(UUID(parent_prj.uuid), f"snapshot.{child_index}") + prj_dict[ + "creation_date" + ] = parent_prj.last_change_date # state of parent upon copy! + prj_dict["hidden"] = True + prj_dict["published"] = False + + # NOTE: a snapshot has no results but workbench stores some states, + # - input hashes + # - node ids + + # + # Define policies about changes in parent project and + # how it influence children + # + project_uuid: str = await conn.scalar( + projects.insert().values(**prj_dict).returning(projects.c.uuid) + ) + + assert UUID(project_uuid) == prj_dict["uuid"] + + # create snapshot + snapshot_id = await conn.scalar( + snapshots.insert() + .values( + name=f"Snapshot {child_index}", + parent_uuid=parent_prj.uuid, + child_index=child_index, + project_uuid=project_uuid, + ) + .returning(snapshots.c.id) + ) + return snapshot_id + + async with engine.acquire() as conn: + + # get parent + res: ResultProxy = await conn.execute( + projects.select().where(projects.c.name == "parent") + ) + parent_prj: Optional[RowProxy] = await res.first() + + assert parent_prj + + # take one snapshot + snapshot_one_id = await _create_snapshot(0, parent_prj, conn) + + # modify parent + updated_parent_prj = await ( + await conn.execute( + projects.update() + .values(description="foo") + .where(projects.c.id == parent_prj.id) + .returning(projects) + ) + ).first() + + assert updated_parent_prj + + assert updated_parent_prj.id == parent_prj.id + assert updated_parent_prj.description != parent_prj.description + + # take another snapshot + snapshot_two_id = await _create_snapshot(1, updated_parent_prj, conn) + + assert snapshot_one_id != snapshot_two_id + + # get project corresponding to snapshot 1 + selected_snapshot_project = await ( + await conn.execute( + projects.select().where(snapshots.c.id == snapshot_two_id) + ) + ).first() + + assert selected_snapshot_project + assert selected_snapshot_project.description == updated_parent_prj.description + + assert selected_snapshot_project.tuple() == updated_parent_prj.tuple() + + +def test_deleting_snapshots(): + # test delete child project -> deletes snapshot + # test delete snapshot -> deletes child project + + # test delete parent project -> deletes snapshots + # test delete snapshot does NOT delete parent + pass + + +def test_create_pydantic_models_from_sqlalchemy_tables(): + # SEE https://docs.sqlalchemy.org/en/14/core/metadata.html + # SEE https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py + pass diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index e7681d078d7..fe6e860b380 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -51,6 +51,7 @@ def random_project(**overrides) -> Dict[str, Any]: name=fake.word(), description=fake.sentence(), prj_owner=fake.pyint(), + thumbnail=fake.image_url(width=120, height=120), access_rights={}, workbench={}, published=False, From 208d7766ba36953ec4bcd025094ab42bff9ff02f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 15 Jul 2021 21:52:22 +0200 Subject: [PATCH 078/137] minor --- .../parametrization_api_handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py index 6d6dc3e5b29..5b427e9cff3 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py @@ -9,7 +9,7 @@ from pydantic.error_wrappers import ValidationError from simcore_service_webserver.parametrization_models import Snapshot -from ._meta import api_version_prefix +from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from .parametrization_models import Snapshot, SnapshotApiModel @@ -46,7 +46,7 @@ async def wrapped(request: web.Request): @routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + f"/{vtag}/projects/{{project_id}}/snapshots", name="_list_snapshots_handler", ) @handle_request_errors @@ -77,7 +77,7 @@ async def _list_snapshots_handler(request: web.Request): @routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}", + f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}", name="_get_snapshot_handler", ) @handle_request_errors @@ -92,7 +92,7 @@ async def _get_snapshot_handler(request: web.Request): @routes.post( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots", + f"/{vtag}/projects/{{project_id}}/snapshots", name="_create_snapshot_handler", ) @handle_request_errors @@ -108,7 +108,7 @@ async def _create_snapshot_handler(request: web.Request): @routes.get( - f"/{api_version_prefix}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", + f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", name="_get_snapshot_parameters_handler", ) @handle_request_errors From 5a67984428c98ef7bd56d472fd5e389cf98d35e0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 16 Jul 2021 17:16:53 +0200 Subject: [PATCH 079/137] exploratory tests for snapshots --- .../postgres-database/tests/test_snapshots.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index e03a8d889a8..714f57361c0 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -32,17 +32,18 @@ async def engine(pg_engine: Engine): async def test_creating_snapshots(engine: Engine): + exclude = { + "id", + "uuid", + "creation_date", + "last_change_date", + "hidden", + "published", + } + async def _create_snapshot(child_index: int, parent_prj, conn) -> int: # copy # change uuid, and set to invisible - exclude = { - "id", - "uuid", - "creation_date", - "last_change_date", - "hidden", - "published", - } prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} prj_dict["name"] += f" [snapshot {child_index}]" @@ -114,16 +115,22 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert snapshot_one_id != snapshot_two_id # get project corresponding to snapshot 1 + j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) selected_snapshot_project = await ( await conn.execute( - projects.select().where(snapshots.c.id == snapshot_two_id) + projects.select() + .select_from(j) + .where(snapshots.c.id == snapshot_two_id) ) ).first() assert selected_snapshot_project assert selected_snapshot_project.description == updated_parent_prj.description - assert selected_snapshot_project.tuple() == updated_parent_prj.tuple() + def extract(t): + return {k: t[k] for k in t if k not in exclude.union({"name"})} + + assert extract(selected_snapshot_project) == extract(updated_parent_prj) def test_deleting_snapshots(): From 16fd418316031b11d32abb2c380d29a99a2325c4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 16 Jul 2021 19:09:12 +0200 Subject: [PATCH 080/137] table to pydantic models --- .../utils_pydantic.py | 43 +++++++++++++++++++ .../postgres-database/tests/test_snapshots.py | 1 - .../tests/test_utils_pydantic.py | 15 +++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py create mode 100644 packages/postgres-database/tests/test_utils_pydantic.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py new file mode 100644 index 00000000000..d9d0d06685b --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py @@ -0,0 +1,43 @@ +from typing import Container, Optional, Type + +import sqlalchemy as sa +from pydantic import BaseConfig, BaseModel, Field, create_model + + +class OrmConfig(BaseConfig): + orm_mode = True + + +def sa_table_to_pydantic_model( + table: sa.Table, *, config: Type = OrmConfig, exclude: Container[str] = [] +) -> Type[BaseModel]: + # NOTE: basically copied from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py + fields = {} + + for column in table.columns: + name = str(column.name) + if name in exclude: + continue + + python_type: Optional[type] = None + if hasattr(column.type, "impl"): + if hasattr(column.type.impl, "python_type"): + python_type = column.type.impl.python_type + elif hasattr(column.type, "python_type"): + python_type = column.type.python_type + assert python_type, f"Could not infer python_type for {column}" # nosec + default = None + if column.default is None and not column.nullable: + default = ... + + if hasattr(column, "doc") and column.doc: + default = Field(default, description=column.doc) + + fields[name] = (python_type, default) + + # create domain models from db-schemas + pydantic_model = create_model( + table.name.capitalize(), __config__=config, **fields # type: ignore + ) + assert issubclass(pydantic_model, BaseModel) # nosec + return pydantic_model diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 714f57361c0..1c19d3aaf78 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -125,7 +125,6 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: ).first() assert selected_snapshot_project - assert selected_snapshot_project.description == updated_parent_prj.description def extract(t): return {k: t[k] for k in t if k not in exclude.union({"name"})} diff --git a/packages/postgres-database/tests/test_utils_pydantic.py b/packages/postgres-database/tests/test_utils_pydantic.py new file mode 100644 index 00000000000..41bb8627840 --- /dev/null +++ b/packages/postgres-database/tests/test_utils_pydantic.py @@ -0,0 +1,15 @@ +import pytest +from pydantic import BaseModel +from simcore_postgres_database.models import * +from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.utils_pydantic import sa_table_to_pydantic_model + + +@pytest.mark.parametrized("table_cls", metadata.tables) +def test_table_to_pydantic_models(table_cls): + + PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls, exclude={}) + assert issubclass(PydanticModelAtDB, BaseModel) + print(PydanticModelAtDB.schema_json(indent=2)) + + # TODO: create fakes automatically? From 40ebfd86ecda74555f1ffdd0928e519870c05f45 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 11:56:27 +0200 Subject: [PATCH 081/137] cleanup pydantic factory --- ...ic.py => utils_pydantic_models_factory.py} | 7 +++- .../tests/test_utils_pydantic.py | 15 -------- .../test_utils_pydantic_models_factory.py | 34 +++++++++++++++++++ 3 files changed, 40 insertions(+), 16 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/{utils_pydantic.py => utils_pydantic_models_factory.py} (91%) delete mode 100644 packages/postgres-database/tests/test_utils_pydantic.py create mode 100644 packages/postgres-database/tests/test_utils_pydantic_models_factory.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py similarity index 91% rename from packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py rename to packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py index d9d0d06685b..3ee7dd16009 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py @@ -9,10 +9,15 @@ class OrmConfig(BaseConfig): def sa_table_to_pydantic_model( - table: sa.Table, *, config: Type = OrmConfig, exclude: Container[str] = [] + table: sa.Table, + *, + config: Type = OrmConfig, + exclude: Optional[Container[str]] = None, ) -> Type[BaseModel]: + # NOTE: basically copied from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py fields = {} + exclude = exclude or [] for column in table.columns: name = str(column.name) diff --git a/packages/postgres-database/tests/test_utils_pydantic.py b/packages/postgres-database/tests/test_utils_pydantic.py deleted file mode 100644 index 41bb8627840..00000000000 --- a/packages/postgres-database/tests/test_utils_pydantic.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest -from pydantic import BaseModel -from simcore_postgres_database.models import * -from simcore_postgres_database.models.base import metadata -from simcore_postgres_database.utils_pydantic import sa_table_to_pydantic_model - - -@pytest.mark.parametrized("table_cls", metadata.tables) -def test_table_to_pydantic_models(table_cls): - - PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls, exclude={}) - assert issubclass(PydanticModelAtDB, BaseModel) - print(PydanticModelAtDB.schema_json(indent=2)) - - # TODO: create fakes automatically? diff --git a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py new file mode 100644 index 00000000000..8eb31b7cb47 --- /dev/null +++ b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py @@ -0,0 +1,34 @@ +from datetime import datetime +from uuid import uuid4 + +import pytest +from pydantic import BaseModel +from simcore_postgres_database.models import * +from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.models.snapshots import snapshots +from simcore_postgres_database.utils_pydantic_models_factory import ( + sa_table_to_pydantic_model, +) + + +@pytest.mark.parametrized("table_cls", metadata.tables) +def test_table_to_pydantic_models(table_cls): + + PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) + assert issubclass(PydanticModelAtDB, BaseModel) + print(PydanticModelAtDB.schema_json(indent=2)) + + # TODO: create fakes automatically? + + +def test_snapshot_pydantic_model(): + Snapshot = sa_table_to_pydantic_model(snapshots) + + snapshot = Snapshot( + id=0, + created_at=datetime.now(), + parent_uuid=uuid4(), + child_index=2, + project_uuid=uuid4(), + ) + assert snapshot.id == 0 From 163d2ed01d5797fc1182707703141b60c98ed8e8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 12:01:43 +0200 Subject: [PATCH 082/137] adds pydantic dep --- packages/postgres-database/requirements/_pydantic.in | 1 + packages/postgres-database/requirements/_pydantic.txt | 10 ++++++++++ packages/postgres-database/requirements/_test.in | 1 + packages/postgres-database/requirements/dev.txt | 1 + packages/postgres-database/setup.py | 8 ++++++-- 5 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 packages/postgres-database/requirements/_pydantic.in create mode 100644 packages/postgres-database/requirements/_pydantic.txt diff --git a/packages/postgres-database/requirements/_pydantic.in b/packages/postgres-database/requirements/_pydantic.in new file mode 100644 index 00000000000..572b352f30e --- /dev/null +++ b/packages/postgres-database/requirements/_pydantic.in @@ -0,0 +1 @@ +pydantic diff --git a/packages/postgres-database/requirements/_pydantic.txt b/packages/postgres-database/requirements/_pydantic.txt new file mode 100644 index 00000000000..6605adc0dff --- /dev/null +++ b/packages/postgres-database/requirements/_pydantic.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# pip-compile --output-file=requirements/_pydantic.txt --strip-extras requirements/_pydantic.in +# +pydantic==1.8.2 + # via -r requirements/_pydantic.in +typing-extensions==3.10.0.0 + # via pydantic diff --git a/packages/postgres-database/requirements/_test.in b/packages/postgres-database/requirements/_test.in index c76e9d978cf..67ad0d9fe0e 100644 --- a/packages/postgres-database/requirements/_test.in +++ b/packages/postgres-database/requirements/_test.in @@ -8,6 +8,7 @@ # --constraint _base.txt --constraint _migration.txt +--constraint _pydantic.txt # fixtures pyyaml diff --git a/packages/postgres-database/requirements/dev.txt b/packages/postgres-database/requirements/dev.txt index 8136f1a48b5..d7d03a8f861 100644 --- a/packages/postgres-database/requirements/dev.txt +++ b/packages/postgres-database/requirements/dev.txt @@ -9,6 +9,7 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt +--requirement _pydantic.txt --requirement _test.txt --requirement _tools.txt diff --git a/packages/postgres-database/setup.py b/packages/postgres-database/setup.py index b5bda1bdc81..4780ae5ca50 100644 --- a/packages/postgres-database/setup.py +++ b/packages/postgres-database/setup.py @@ -21,7 +21,7 @@ def read_reqs(reqs_path: Path): # Strong dependencies migration_requirements = read_reqs(current_dir / "requirements" / "_migration.in") test_requirements = read_reqs(current_dir / "requirements" / "_test.txt") - +pydantic_requirements = read_reqs(current_dir / "requirements" / "_pydantic.in") setup( name="simcore-postgres-database", @@ -43,7 +43,11 @@ def read_reqs(reqs_path: Path): test_suite="tests", install_requires=install_requirements, tests_require=test_requirements, - extras_require={"migration": migration_requirements, "test": test_requirements}, + extras_require={ + "migration": migration_requirements, + "test": test_requirements, + "pydantic": pydantic_requirements, + }, include_package_data=True, package_data={ "": [ From e0802e99e3b4fb64035a12bbd02c09d5b67a0bd4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:01:19 +0200 Subject: [PATCH 083/137] pydantic models factory --- .../postgres-database/requirements/ci.txt | 1 + .../utils_pydantic_models_factory.py | 19 +++++++++++++++++++ .../test_utils_pydantic_models_factory.py | 9 ++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/postgres-database/requirements/ci.txt b/packages/postgres-database/requirements/ci.txt index b12bf394c2e..8fc611bc091 100644 --- a/packages/postgres-database/requirements/ci.txt +++ b/packages/postgres-database/requirements/ci.txt @@ -9,6 +9,7 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt +--requirement _pydantic.txt --requirement _test.txt # installs this repo's packages diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py index 3ee7dd16009..2b51da22ab5 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py @@ -1,4 +1,5 @@ from typing import Container, Optional, Type +from uuid import UUID import sqlalchemy as sa from pydantic import BaseConfig, BaseModel, Field, create_model @@ -8,6 +9,12 @@ class OrmConfig(BaseConfig): orm_mode = True +RESERVED = { + "schema", +} +# e.g. Field name "schema" shadows a BaseModel attribute; use a different field name with "alias='schema'". + + def sa_table_to_pydantic_model( table: sa.Table, *, @@ -21,6 +28,10 @@ def sa_table_to_pydantic_model( for column in table.columns: name = str(column.name) + + if name in RESERVED: + name = f"{table.name.lower()}_{name}" + if name in exclude: continue @@ -30,11 +41,19 @@ def sa_table_to_pydantic_model( python_type = column.type.impl.python_type elif hasattr(column.type, "python_type"): python_type = column.type.python_type + assert python_type, f"Could not infer python_type for {column}" # nosec + default = None if column.default is None and not column.nullable: default = ... + # Policies based on naming conventions + if "uuid" in name.split("_") and python_type == str: + python_type = UUID + if isinstance(default, str): + default = UUID(default) + if hasattr(column, "doc") and column.doc: default = Field(default, description=column.doc) diff --git a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py index 8eb31b7cb47..8c984d42a90 100644 --- a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py +++ b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py @@ -3,7 +3,13 @@ import pytest from pydantic import BaseModel + +# pylint: disable=wildcard-import +# pylint: disable=unused-wildcard-import from simcore_postgres_database.models import * + +# pylint: enable=wildcard-import +# pylint: enable=unused-wildcard-import from simcore_postgres_database.models.base import metadata from simcore_postgres_database.models.snapshots import snapshots from simcore_postgres_database.utils_pydantic_models_factory import ( @@ -11,7 +17,7 @@ ) -@pytest.mark.parametrized("table_cls", metadata.tables) +@pytest.mark.parametrize("table_cls", metadata.tables.values(), ids=lambda t: t.name) def test_table_to_pydantic_models(table_cls): PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) @@ -26,6 +32,7 @@ def test_snapshot_pydantic_model(): snapshot = Snapshot( id=0, + name="foo", created_at=datetime.now(), parent_uuid=uuid4(), child_index=2, From 3ee6971350687e745f259284952c6d0002f1ae0f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:13:36 +0200 Subject: [PATCH 084/137] WIP --- .../src/simcore_service_webserver/parametrization.py | 2 ++ .../parametrization_models.py | 10 +++++++--- .../tests/unit/isolated/test_parametrization_models.py | 7 +++++++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 services/web/server/tests/unit/isolated/test_parametrization_models.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/parametrization.py index 00c18833ac6..725e91ef648 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/parametrization.py @@ -1,5 +1,7 @@ """ parametrization app module setup + Extend project's business logic by adding two new concepts, namely snapshots and parametrizations + - Project parametrization - Project snapshots diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/parametrization_models.py index e9d13bf2391..f1d3ad54885 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_models.py +++ b/services/web/server/src/simcore_service_webserver/parametrization_models.py @@ -15,13 +15,14 @@ BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] - +## Domain models -------- class Parameter(BaseModel): name: str value: BuiltinTypes - node_id: UUID - output_id: OutputID + # TODO: same parameter in different nodes? + node_id: UUID = Field(..., description="Id of parametrized node") + output_id: OutputID = Field(..., description="Output where parameter is exposed") class Snapshot(BaseModel): @@ -36,6 +37,9 @@ class Snapshot(BaseModel): project_id: UUID = Field(..., description="Current project's uuid") +## API models ---------- + + class ParameterApiModel(Parameter): url: AnyUrl # url_output: AnyUrl diff --git a/services/web/server/tests/unit/isolated/test_parametrization_models.py b/services/web/server/tests/unit/isolated/test_parametrization_models.py new file mode 100644 index 00000000000..357899214c5 --- /dev/null +++ b/services/web/server/tests/unit/isolated/test_parametrization_models.py @@ -0,0 +1,7 @@ +from simcore_postgres_database.models.snapshots import snapshots +from simcore_service_webserver.parametrization_models import ( + Parameter, + ParameterApiModel, + Snapshot, + SnapshotApiModel, +) From e7636d2869e499183471688b29f2c7d694d3b43f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:17:10 +0200 Subject: [PATCH 085/137] cleanup pg snapshots table --- .../doc/img/postgres-database-models.svg | 1207 +++++++++-------- .../models/snapshots.py | 3 +- .../postgres-database/tests/test_snapshots.py | 2 +- 3 files changed, 677 insertions(+), 535 deletions(-) diff --git a/packages/postgres-database/doc/img/postgres-database-models.svg b/packages/postgres-database/doc/img/postgres-database-models.svg index 074985a08d2..368427dacb8 100644 --- a/packages/postgres-database/doc/img/postgres-database-models.svg +++ b/packages/postgres-database/doc/img/postgres-database-models.svg @@ -4,668 +4,809 @@ - - + + %3 - + file_meta_data - -file_meta_data - -file_uuid - [VARCHAR] - -location_id - [VARCHAR] - -location - [VARCHAR] - -bucket_name - [VARCHAR] - -object_name - [VARCHAR] - -project_id - [VARCHAR] - -project_name - [VARCHAR] - -node_id - [VARCHAR] - -node_name - [VARCHAR] - -file_name - [VARCHAR] - -user_id - [VARCHAR] - -user_name - [VARCHAR] - -file_id - [VARCHAR] - -raw_file_path - [VARCHAR] - -display_file_path - [VARCHAR] - -created_at - [VARCHAR] - -last_modified - [VARCHAR] - -file_size - [BIGINT] + +file_meta_data + +file_uuid + [VARCHAR] + +location_id + [VARCHAR] + +location + [VARCHAR] + +bucket_name + [VARCHAR] + +object_name + [VARCHAR] + +project_id + [VARCHAR] + +project_name + [VARCHAR] + +node_id + [VARCHAR] + +node_name + [VARCHAR] + +file_name + [VARCHAR] + +user_id + [VARCHAR] + +user_name + [VARCHAR] + +file_id + [VARCHAR] + +raw_file_path + [VARCHAR] + +display_file_path + [VARCHAR] + +created_at + [VARCHAR] + +last_modified + [VARCHAR] + +file_size + [BIGINT] + +entity_tag + [VARCHAR] + +is_soft_link + [BOOLEAN] groups - -groups - -gid - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -type - [VARCHAR(8)] - -thumbnail - [VARCHAR] - -inclusion_rules - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] + +groups + +gid + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +type + [VARCHAR(8)] + +thumbnail + [VARCHAR] + +inclusion_rules + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] user_to_groups - -user_to_groups - -uid - [BIGINT] - -gid - [BIGINT] - -access_rights - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] + +user_to_groups + +uid + [BIGINT] + +gid + [BIGINT] + +access_rights + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] groups--user_to_groups - -{0,1} -0..N + +{0,1} +0..N users - -users - -id - [BIGINT] - -name - [VARCHAR] - -email - [VARCHAR] - -password_hash - [VARCHAR] - -primary_gid - [BIGINT] - -status - [VARCHAR(20)] - -role - [VARCHAR(9)] - -created_at - [DATETIME] - -modified - [DATETIME] - -created_ip - [VARCHAR] + +users + +id + [BIGINT] + +name + [VARCHAR] + +email + [VARCHAR] + +password_hash + [VARCHAR] + +primary_gid + [BIGINT] + +status + [VARCHAR(20)] + +role + [VARCHAR(9)] + +created_at + [DATETIME] + +modified + [DATETIME] + +created_ip + [VARCHAR] groups--users - -{0,1} -0..N + +{0,1} +0..N group_classifiers - -group_classifiers - -id - [BIGINT] - -bundle - [JSONB] - -created - [DATETIME] - -modified - [DATETIME] - -gid - [BIGINT] - -uses_scicrunch - [BOOLEAN] + +group_classifiers + +id + [BIGINT] + +bundle + [JSONB] + +created + [DATETIME] + +modified + [DATETIME] + +gid + [BIGINT] + +uses_scicrunch + [BOOLEAN] groups--group_classifiers - -{0,1} -0..N + +{0,1} +0..N services_meta_data - -services_meta_data - -key - [VARCHAR] - -version - [VARCHAR] - -owner - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -thumbnail - [VARCHAR] - -classifiers - [VARCHAR[]] - -created - [DATETIME] - -modified - [DATETIME] - -metadata - [JSONB] + +services_meta_data + +key + [VARCHAR] + +version + [VARCHAR] + +owner + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +thumbnail + [VARCHAR] + +classifiers + [VARCHAR[]] + +created + [DATETIME] + +modified + [DATETIME] + +quality + [JSONB] groups--services_meta_data - -{0,1} -0..N + +{0,1} +0..N services_access_rights - -services_access_rights - -key - [VARCHAR] - -version - [VARCHAR] - -gid - [BIGINT] - -execute_access - [BOOLEAN] - -write_access - [BOOLEAN] - -product_name - [VARCHAR] - -created - [DATETIME] - -modified - [DATETIME] + +services_access_rights + +key + [VARCHAR] + +version + [VARCHAR] + +gid + [BIGINT] + +execute_access + [BOOLEAN] + +write_access + [BOOLEAN] + +product_name + [VARCHAR] + +created + [DATETIME] + +modified + [DATETIME] groups--services_access_rights - -{0,1} -0..N + +{0,1} +0..N - + users--user_to_groups - -{0,1} -0..N + +{0,1} +0..N projects - -projects - -id - [BIGINT] - -type - [VARCHAR(8)] - -uuid - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -thumbnail - [VARCHAR] - -prj_owner - [BIGINT] - -creation_date - [DATETIME] - -last_change_date - [DATETIME] - -access_rights - [JSONB] - -workbench - [Null] - -ui - [JSONB] - -classifiers - [VARCHAR[]] - -dev - [JSONB] - -published - [BOOLEAN] + +projects + +id + [BIGINT] + +type + [VARCHAR(8)] + +uuid + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +thumbnail + [VARCHAR] + +prj_owner + [BIGINT] + +creation_date + [DATETIME] + +last_change_date + [DATETIME] + +access_rights + [JSONB] + +workbench + [Null] + +ui + [JSONB] + +classifiers + [VARCHAR[]] + +dev + [JSONB] + +quality + [JSONB] + +published + [BOOLEAN] + +hidden + [BOOLEAN] - + users--projects - -{0,1} -0..N + +{0,1} +0..N + + + +comp_runs + +comp_runs + +run_id + [BIGINT] + +project_uuid + [VARCHAR] + +user_id + [BIGINT] + +iteration + [BIGINT] + +result + [VARCHAR(11)] + +created + [DATETIME] + +modified + [DATETIME] + +started + [DATETIME] + +ended + [DATETIME] + + + +users--comp_runs + +{0,1} +0..N - + user_to_projects - -user_to_projects - -id - [BIGINT] - -user_id - [BIGINT] - -project_id - [BIGINT] + +user_to_projects + +id + [BIGINT] + +user_id + [BIGINT] + +project_id + [BIGINT] - + users--user_to_projects - -{0,1} -0..N + +{0,1} +0..N - + tokens - -tokens - -token_id - [BIGINT] - -user_id - [BIGINT] - -token_service - [VARCHAR] - -token_data - [Null] + +tokens + +token_id + [BIGINT] + +user_id + [BIGINT] + +token_service + [VARCHAR] + +token_data + [Null] - + users--tokens - -{0,1} -0..N + +{0,1} +0..N - + api_keys - -api_keys - -id - [BIGINT] - -display_name - [VARCHAR] - -user_id - [BIGINT] - -api_key - [VARCHAR] - -api_secret - [VARCHAR] + +api_keys + +id + [BIGINT] + +display_name + [VARCHAR] + +user_id + [BIGINT] + +api_key + [VARCHAR] + +api_secret + [VARCHAR] - + users--api_keys - -{0,1} -0..N + +{0,1} +0..N - + confirmations - -confirmations - -code - [TEXT] - -user_id - [BIGINT] - -action - [VARCHAR(14)] - -data - [TEXT] - -created_at - [DATETIME] + +confirmations + +code + [TEXT] + +user_id + [BIGINT] + +action + [VARCHAR(14)] + +data + [TEXT] + +created_at + [DATETIME] - + users--confirmations - -{0,1} -0..N + +{0,1} +0..N - + tags - -tags - -id - [BIGINT] - -user_id - [BIGINT] - -name - [VARCHAR] - -description - [VARCHAR] - -color - [VARCHAR] + +tags + +id + [BIGINT] + +user_id + [BIGINT] + +name + [VARCHAR] + +description + [VARCHAR] + +color + [VARCHAR] - + users--tags - -{0,1} -0..N + +{0,1} +0..N - + services_meta_data--services_access_rights - -{0,1} -0..N + +{0,1} +0..N - + services_meta_data--services_access_rights - -{0,1} -0..N + +{0,1} +0..N + + + +services_consume_filetypes + +services_consume_filetypes + +service_key + [VARCHAR] + +service_version + [VARCHAR] + +service_display_name + [VARCHAR] + +service_input_port + [VARCHAR] + +filetype + [VARCHAR] + +preference_order + [SMALLINT] + +is_guest_allowed + [BOOLEAN] + + + +services_meta_data--services_consume_filetypes + +{0,1} +0..N + + + +services_meta_data--services_consume_filetypes + +{0,1} +0..N study_tags - -study_tags - -study_id - [BIGINT] - -tag_id - [BIGINT] + +study_tags + +study_id + [BIGINT] + +tag_id + [BIGINT] projects--study_tags - -{0,1} -0..N + +{0,1} +0..N - + + +snapshots + +snapshots + +id + [BIGINT] + +name + [VARCHAR] + +created_at + [DATETIME] + +parent_uuid + [VARCHAR] + +child_index + [INTEGER] + +project_uuid + [VARCHAR] + + +projects--snapshots + +{0,1} +0..N + + + +projects--snapshots + +{0,1} +0..N + + + +projects--comp_runs + +{0,1} +0..N + + + projects--user_to_projects - -{0,1} -0..N + +{0,1} +0..N - + tags--study_tags - -{0,1} -0..N + +{0,1} +0..N - + comp_pipeline - -comp_pipeline - -project_id - [VARCHAR] - -dag_adjacency_list - [Null] - -state - [VARCHAR(11)] + +comp_pipeline + +project_id + [VARCHAR] + +dag_adjacency_list + [Null] + +state + [VARCHAR(11)] - + comp_tasks - -comp_tasks - -task_id - [INTEGER] - -project_id - [VARCHAR] - -node_id - [VARCHAR] - -node_class - [VARCHAR(13)] - -job_id - [VARCHAR] - -internal_id - [INTEGER] - -schema - [Null] - -inputs - [Null] - -outputs - [Null] - -image - [Null] - -state - [VARCHAR(11)] - -submit - [DATETIME] - -start - [DATETIME] - -end - [DATETIME] + +comp_tasks + +task_id + [INTEGER] + +project_id + [VARCHAR] + +node_id + [VARCHAR] + +node_class + [VARCHAR(13)] + +job_id + [VARCHAR] + +internal_id + [INTEGER] + +schema + [Null] + +inputs + [Null] + +outputs + [Null] + +run_hash + [VARCHAR] + +image + [Null] + +state + [VARCHAR(11)] + +submit + [DATETIME] + +start + [DATETIME] + +end + [DATETIME] - + comp_pipeline--comp_tasks - -{0,1} -0..N + +{0,1} +0..N - + products - -products - -name - [VARCHAR] - -host_regex - [VARCHAR] - -created - [DATETIME] - -modified - [DATETIME] + +products + +name + [VARCHAR] + +host_regex + [VARCHAR] + +created + [DATETIME] + +modified + [DATETIME] - + products--services_access_rights - -{0,1} -0..N + +{0,1} +0..N - + scicrunch_resources - -scicrunch_resources - -rrid - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -creation_date - [DATETIME] - -last_change_date - [DATETIME] + +scicrunch_resources + +rrid + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +creation_date + [DATETIME] + +last_change_date + [DATETIME] - + dags - -dags - -id - [INTEGER] - -key - [VARCHAR] - -version - [VARCHAR] - -name - [VARCHAR] - -description - [VARCHAR] - -contact - [VARCHAR] - -workbench - [Null] + +dags + +id + [INTEGER] + +key + [VARCHAR] + +version + [VARCHAR] + +name + [VARCHAR] + +description + [VARCHAR] + +contact + [VARCHAR] + +workbench + [Null] diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index 8cce020e2f5..f23c80333ce 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -30,6 +30,7 @@ ondelete="CASCADE", ), nullable=False, + unique=False, doc="UUID of the parent project", ), sa.Column( @@ -37,7 +38,7 @@ sa.Integer, nullable=False, unique=True, - doc="Number of child (as 0-based index: 0 being the oldest, 1, ...)" + doc="0-based index in order of creation (i.e. 0 being the oldest and N-1 the latest)" "from the same parent_id", ), sa.Column( diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 1c19d3aaf78..2710c72d5ef 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -18,7 +18,7 @@ @pytest.fixture async def engine(pg_engine: Engine): - # injects + # injects ... async with pg_engine.acquire() as conn: # a 'me' user user_id = await conn.scalar( From 1e08bbdb29a06b322cb500965a114d15eada1018 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:18:07 +0200 Subject: [PATCH 086/137] renamed module app from parametrization to snapshots --- .../{parametrization.py => snapshots.py} | 13 ++++++------- ...on_api_handlers.py => snapshots_api_handlers.py} | 4 ++-- .../{parametrization_core.py => snapshots_core.py} | 2 +- ...arametrization_models.py => snapshots_models.py} | 0 ...etrization_settings.py => snapshots_settings.py} | 0 ...trization_models.py => test_snapshots_models.py} | 2 +- ...rametrization_core.py => test_snapshots_core.py} | 4 ++-- 7 files changed, 12 insertions(+), 13 deletions(-) rename services/web/server/src/simcore_service_webserver/{parametrization.py => snapshots.py} (74%) rename services/web/server/src/simcore_service_webserver/{parametrization_api_handlers.py => snapshots_api_handlers.py} (97%) rename services/web/server/src/simcore_service_webserver/{parametrization_core.py => snapshots_core.py} (97%) rename services/web/server/src/simcore_service_webserver/{parametrization_models.py => snapshots_models.py} (100%) rename services/web/server/src/simcore_service_webserver/{parametrization_settings.py => snapshots_settings.py} (100%) rename services/web/server/tests/unit/isolated/{test_parametrization_models.py => test_snapshots_models.py} (69%) rename services/web/server/tests/unit/with_dbs/11/{test_parametrization_core.py => test_snapshots_core.py} (95%) diff --git a/services/web/server/src/simcore_service_webserver/parametrization.py b/services/web/server/src/simcore_service_webserver/snapshots.py similarity index 74% rename from services/web/server/src/simcore_service_webserver/parametrization.py rename to services/web/server/src/simcore_service_webserver/snapshots.py index 725e91ef648..47137d5bf05 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization.py +++ b/services/web/server/src/simcore_service_webserver/snapshots.py @@ -1,9 +1,8 @@ -""" parametrization app module setup +""" snapshots (and parametrization) app module setup - Extend project's business logic by adding two new concepts, namely snapshots and parametrizations - - - Project parametrization - - Project snapshots + Extend project's business logic by adding two new concepts, namely + - project snapshots and + - parametrizations """ import logging @@ -15,7 +14,7 @@ app_module_setup, ) -from . import parametrization_api_handlers +from . import snapshots_api_handlers from .constants import APP_SETTINGS_KEY from .settings import ApplicationSettings @@ -34,4 +33,4 @@ def setup(app: web.Application): if not settings.WEBSERVER_DEV_FEATURES_ENABLED: raise SkipModuleSetup(reason="Development feature") - app.add_routes(parametrization_api_handlers.routes) + app.add_routes(snapshots_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py similarity index 97% rename from services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py rename to services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 5b427e9cff3..a0ff98a6b27 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -7,11 +7,11 @@ from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError -from simcore_service_webserver.parametrization_models import Snapshot +from simcore_service_webserver.snapshots_models import Snapshot from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY -from .parametrization_models import Snapshot, SnapshotApiModel +from .snapshots_models import Snapshot, SnapshotApiModel json_dumps = json.dumps diff --git a/services/web/server/src/simcore_service_webserver/parametrization_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py similarity index 97% rename from services/web/server/src/simcore_service_webserver/parametrization_core.py rename to services/web/server/src/simcore_service_webserver/snapshots_core.py index fead73ab024..9a77d8dd7be 100644 --- a/services/web/server/src/simcore_service_webserver/parametrization_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -13,9 +13,9 @@ from models_library.projects_nodes import Node -from .parametrization_models import Snapshot from .projects.projects_db import ProjectAtDB from .projects.projects_utils import clone_project_document +from .snapshots_models import Snapshot def is_parametrized(node: Node) -> bool: diff --git a/services/web/server/src/simcore_service_webserver/parametrization_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/parametrization_models.py rename to services/web/server/src/simcore_service_webserver/snapshots_models.py diff --git a/services/web/server/src/simcore_service_webserver/parametrization_settings.py b/services/web/server/src/simcore_service_webserver/snapshots_settings.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/parametrization_settings.py rename to services/web/server/src/simcore_service_webserver/snapshots_settings.py diff --git a/services/web/server/tests/unit/isolated/test_parametrization_models.py b/services/web/server/tests/unit/isolated/test_snapshots_models.py similarity index 69% rename from services/web/server/tests/unit/isolated/test_parametrization_models.py rename to services/web/server/tests/unit/isolated/test_snapshots_models.py index 357899214c5..21933a6d667 100644 --- a/services/web/server/tests/unit/isolated/test_parametrization_models.py +++ b/services/web/server/tests/unit/isolated/test_snapshots_models.py @@ -1,5 +1,5 @@ from simcore_postgres_database.models.snapshots import snapshots -from simcore_service_webserver.parametrization_models import ( +from simcore_service_webserver.snapshots_models import ( Parameter, ParameterApiModel, Snapshot, diff --git a/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py b/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py similarity index 95% rename from services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py rename to services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py index a1060d1b51c..d4f14e54043 100644 --- a/services/web/server/tests/unit/with_dbs/11/test_parametrization_core.py +++ b/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py @@ -7,14 +7,14 @@ import pytest from aiohttp import web -from models_library.projects import Project, ProjectAtDB +from models_library.projects import Project # , ProjectAtDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID from simcore_service_webserver.constants import APP_PROJECT_DBAPI -from simcore_service_webserver.parametrization_core import snapshot_project from simcore_service_webserver.projects.projects_api import get_project_for_user from simcore_service_webserver.projects.projects_db import APP_PROJECT_DBAPI from simcore_service_webserver.projects.projects_utils import clone_project_document +from simcore_service_webserver.snapshots_core import snapshot_project # is parametrized project? From 5bcca1962dc205d01a03a2d25f7a388dcb14a808 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 09:43:31 +0200 Subject: [PATCH 087/137] Revert "adds pydantic dep" This reverts commit e00ec034ea47c95c56d4440f0ff704526600bc5b. --- packages/postgres-database/requirements/_pydantic.in | 1 - packages/postgres-database/requirements/_pydantic.txt | 10 ---------- packages/postgres-database/requirements/_test.in | 1 - packages/postgres-database/requirements/dev.txt | 1 - packages/postgres-database/setup.py | 8 ++------ 5 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 packages/postgres-database/requirements/_pydantic.in delete mode 100644 packages/postgres-database/requirements/_pydantic.txt diff --git a/packages/postgres-database/requirements/_pydantic.in b/packages/postgres-database/requirements/_pydantic.in deleted file mode 100644 index 572b352f30e..00000000000 --- a/packages/postgres-database/requirements/_pydantic.in +++ /dev/null @@ -1 +0,0 @@ -pydantic diff --git a/packages/postgres-database/requirements/_pydantic.txt b/packages/postgres-database/requirements/_pydantic.txt deleted file mode 100644 index 6605adc0dff..00000000000 --- a/packages/postgres-database/requirements/_pydantic.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile --output-file=requirements/_pydantic.txt --strip-extras requirements/_pydantic.in -# -pydantic==1.8.2 - # via -r requirements/_pydantic.in -typing-extensions==3.10.0.0 - # via pydantic diff --git a/packages/postgres-database/requirements/_test.in b/packages/postgres-database/requirements/_test.in index 67ad0d9fe0e..c76e9d978cf 100644 --- a/packages/postgres-database/requirements/_test.in +++ b/packages/postgres-database/requirements/_test.in @@ -8,7 +8,6 @@ # --constraint _base.txt --constraint _migration.txt ---constraint _pydantic.txt # fixtures pyyaml diff --git a/packages/postgres-database/requirements/dev.txt b/packages/postgres-database/requirements/dev.txt index d7d03a8f861..8136f1a48b5 100644 --- a/packages/postgres-database/requirements/dev.txt +++ b/packages/postgres-database/requirements/dev.txt @@ -9,7 +9,6 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt ---requirement _pydantic.txt --requirement _test.txt --requirement _tools.txt diff --git a/packages/postgres-database/setup.py b/packages/postgres-database/setup.py index 4780ae5ca50..b5bda1bdc81 100644 --- a/packages/postgres-database/setup.py +++ b/packages/postgres-database/setup.py @@ -21,7 +21,7 @@ def read_reqs(reqs_path: Path): # Strong dependencies migration_requirements = read_reqs(current_dir / "requirements" / "_migration.in") test_requirements = read_reqs(current_dir / "requirements" / "_test.txt") -pydantic_requirements = read_reqs(current_dir / "requirements" / "_pydantic.in") + setup( name="simcore-postgres-database", @@ -43,11 +43,7 @@ def read_reqs(reqs_path: Path): test_suite="tests", install_requires=install_requirements, tests_require=test_requirements, - extras_require={ - "migration": migration_requirements, - "test": test_requirements, - "pydantic": pydantic_requirements, - }, + extras_require={"migration": migration_requirements, "test": test_requirements}, include_package_data=True, package_data={ "": [ From 81f42694c14e22c26b8102db9544c453b0559916 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 09:59:39 +0200 Subject: [PATCH 088/137] removes factory from pg database --- .../utils_pydantic_models_factory.py | 67 ------------------- .../test_utils_pydantic_models_factory.py | 41 ------------ 2 files changed, 108 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py delete mode 100644 packages/postgres-database/tests/test_utils_pydantic_models_factory.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py b/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py deleted file mode 100644 index 2b51da22ab5..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/utils_pydantic_models_factory.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Container, Optional, Type -from uuid import UUID - -import sqlalchemy as sa -from pydantic import BaseConfig, BaseModel, Field, create_model - - -class OrmConfig(BaseConfig): - orm_mode = True - - -RESERVED = { - "schema", -} -# e.g. Field name "schema" shadows a BaseModel attribute; use a different field name with "alias='schema'". - - -def sa_table_to_pydantic_model( - table: sa.Table, - *, - config: Type = OrmConfig, - exclude: Optional[Container[str]] = None, -) -> Type[BaseModel]: - - # NOTE: basically copied from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py - fields = {} - exclude = exclude or [] - - for column in table.columns: - name = str(column.name) - - if name in RESERVED: - name = f"{table.name.lower()}_{name}" - - if name in exclude: - continue - - python_type: Optional[type] = None - if hasattr(column.type, "impl"): - if hasattr(column.type.impl, "python_type"): - python_type = column.type.impl.python_type - elif hasattr(column.type, "python_type"): - python_type = column.type.python_type - - assert python_type, f"Could not infer python_type for {column}" # nosec - - default = None - if column.default is None and not column.nullable: - default = ... - - # Policies based on naming conventions - if "uuid" in name.split("_") and python_type == str: - python_type = UUID - if isinstance(default, str): - default = UUID(default) - - if hasattr(column, "doc") and column.doc: - default = Field(default, description=column.doc) - - fields[name] = (python_type, default) - - # create domain models from db-schemas - pydantic_model = create_model( - table.name.capitalize(), __config__=config, **fields # type: ignore - ) - assert issubclass(pydantic_model, BaseModel) # nosec - return pydantic_model diff --git a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py b/packages/postgres-database/tests/test_utils_pydantic_models_factory.py deleted file mode 100644 index 8c984d42a90..00000000000 --- a/packages/postgres-database/tests/test_utils_pydantic_models_factory.py +++ /dev/null @@ -1,41 +0,0 @@ -from datetime import datetime -from uuid import uuid4 - -import pytest -from pydantic import BaseModel - -# pylint: disable=wildcard-import -# pylint: disable=unused-wildcard-import -from simcore_postgres_database.models import * - -# pylint: enable=wildcard-import -# pylint: enable=unused-wildcard-import -from simcore_postgres_database.models.base import metadata -from simcore_postgres_database.models.snapshots import snapshots -from simcore_postgres_database.utils_pydantic_models_factory import ( - sa_table_to_pydantic_model, -) - - -@pytest.mark.parametrize("table_cls", metadata.tables.values(), ids=lambda t: t.name) -def test_table_to_pydantic_models(table_cls): - - PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) - assert issubclass(PydanticModelAtDB, BaseModel) - print(PydanticModelAtDB.schema_json(indent=2)) - - # TODO: create fakes automatically? - - -def test_snapshot_pydantic_model(): - Snapshot = sa_table_to_pydantic_model(snapshots) - - snapshot = Snapshot( - id=0, - name="foo", - created_at=datetime.now(), - parent_uuid=uuid4(), - child_index=2, - project_uuid=uuid4(), - ) - assert snapshot.id == 0 From b900491e34c016a8d32fa003f094ec37d5e0121f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:03:54 +0200 Subject: [PATCH 089/137] adds db modesl factory in models-library --- packages/models-library/requirements/_test.in | 1 + .../models-library/requirements/_test.txt | 18 ++++- .../utils/database_models_factory.py | 77 +++++++++++++++++++ .../test_utils_database_models_factory.py | 39 ++++++++++ 4 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 packages/models-library/src/models_library/utils/database_models_factory.py create mode 100644 packages/models-library/tests/test_utils_database_models_factory.py diff --git a/packages/models-library/requirements/_test.in b/packages/models-library/requirements/_test.in index b7d9cece229..f89ccfcacc1 100644 --- a/packages/models-library/requirements/_test.in +++ b/packages/models-library/requirements/_test.in @@ -22,6 +22,7 @@ pytest-sugar # tools pylint # NOTE: The version in pylint at _text.txt is used as a reference for ci/helpers/install_pylint.bash coveralls +--requirement ../../../packages/postgres-database/requirements/_base.in # to test units tools diff --git a/packages/models-library/requirements/_test.txt b/packages/models-library/requirements/_test.txt index 8fe967785e6..bcd8c9e7547 100644 --- a/packages/models-library/requirements/_test.txt +++ b/packages/models-library/requirements/_test.txt @@ -6,6 +6,7 @@ # aiohttp==3.7.4.post0 # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # pytest-aiohttp astroid==2.6.6 @@ -36,6 +37,7 @@ icdiff==2.0.4 idna==2.10 # via # -c requirements/_base.txt + # -r requirements/../../../packages/postgres-database/requirements/_base.in # requests # yarl iniconfig==1.1.1 @@ -61,6 +63,8 @@ pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff +psycopg2-binary==2.9.1 + # via sqlalchemy py==1.10.0 # via pytest pylint==2.9.6 @@ -92,10 +96,16 @@ pytest-sugar==0.9.4 # via -r requirements/_test.in pyyaml==5.4.1 # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in requests==2.26.0 # via coveralls +sqlalchemy==1.3.24 + # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/postgres-database/requirements/_base.in termcolor==1.1.0 # via pytest-sugar toml==0.10.2 @@ -109,12 +119,12 @@ typing-extensions==3.10.0.0 # aiohttp urllib3==1.26.6 # via + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests wrapt==1.12.1 # via astroid yarl==1.6.3 - # via aiohttp - -# The following packages are considered to be unsafe in a requirements file: -# setuptools + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # aiohttp diff --git a/packages/models-library/src/models_library/utils/database_models_factory.py b/packages/models-library/src/models_library/utils/database_models_factory.py new file mode 100644 index 00000000000..dd2df28ecce --- /dev/null +++ b/packages/models-library/src/models_library/utils/database_models_factory.py @@ -0,0 +1,77 @@ +""" Automatic creation of pydantic model classes from a sqlalchemy table + + +SEE: Copied and adapted from https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py +""" + +from typing import Any, Container, Dict, Optional, Type +from uuid import UUID + +import sqlalchemy as sa +from pydantic import BaseConfig, BaseModel, Field, create_model + + +class OrmConfig(BaseConfig): + orm_mode = True + + +_RESERVED = { + "schema", + # e.g. Field name "schema" shadows a BaseModel attribute; use a different field name with "alias='schema'". +} + + +def sa_table_to_pydantic_model( + table: sa.Table, + *, + config: Type = OrmConfig, + exclude: Optional[Container[str]] = None, +) -> Type[BaseModel]: + + fields = {} + exclude = exclude or [] + + for column in table.columns: + name = str(column.name) + + if name in exclude: + continue + + field_args: Dict[str, Any] = {} + + if name in _RESERVED: + field_args["alias"] = name + name = f"{table.name.lower()}_{name}" + + python_type: Optional[type] = None + if hasattr(column.type, "impl"): + if hasattr(column.type.impl, "python_type"): + python_type = column.type.impl.python_type + elif hasattr(column.type, "python_type"): + python_type = column.type.python_type + + assert python_type, f"Could not infer python_type for {column}" # nosec + + default = None + if column.default is None and not column.nullable: + default = ... + + # Policies based on naming conventions + if "uuid" in name.split("_") and python_type == str: + python_type = UUID + if isinstance(default, str): + default = UUID(default) + + field_args["default"] = default + + if hasattr(column, "doc") and column.doc: + field_args["description"] = column.doc + + fields[name] = (python_type, Field(**field_args)) + + # create domain models from db-schemas + pydantic_model = create_model( + table.name.capitalize(), __config__=config, **fields # type: ignore + ) + assert issubclass(pydantic_model, BaseModel) # nosec + return pydantic_model diff --git a/packages/models-library/tests/test_utils_database_models_factory.py b/packages/models-library/tests/test_utils_database_models_factory.py new file mode 100644 index 00000000000..f55e78a3654 --- /dev/null +++ b/packages/models-library/tests/test_utils_database_models_factory.py @@ -0,0 +1,39 @@ +from datetime import datetime +from uuid import uuid4 + +import pytest +from models_library.utils.database_models_factory import sa_table_to_pydantic_model +from pydantic import BaseModel + +# pylint: disable=wildcard-import +# pylint: disable=unused-wildcard-import +from simcore_postgres_database.models import * + +# pylint: enable=wildcard-import +# pylint: enable=unused-wildcard-import +from simcore_postgres_database.models.base import metadata +from simcore_postgres_database.models.snapshots import snapshots + + +@pytest.mark.parametrize("table_cls", metadata.tables.values(), ids=lambda t: t.name) +def test_table_to_pydantic_models(table_cls): + + PydanticModelAtDB = sa_table_to_pydantic_model(table=table_cls) + assert issubclass(PydanticModelAtDB, BaseModel) + print(PydanticModelAtDB.schema_json(indent=2)) + + # TODO: create fakes automatically? + + +def test_snapshot_pydantic_model(): + Snapshot = sa_table_to_pydantic_model(snapshots) + + snapshot = Snapshot( + id=0, + name="foo", + created_at=datetime.now(), + parent_uuid=uuid4(), + child_index=2, + project_uuid=uuid4(), + ) + assert snapshot.id == 0 From 92672538b950a8cd0275a495797c08bd79fb5e41 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:04:14 +0200 Subject: [PATCH 090/137] wip snapshots --- .../models/snapshots.py | 4 +- .../postgres-database/tests/test_snapshots.py | 43 ++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index f23c80333ce..cf3c7dfd6e7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -19,7 +19,9 @@ sa.DateTime(), nullable=False, server_default=func.now(), - doc="Timestamp on creation", + doc="Timestamp on creation. " + "It corresponds to the last_change_date of the parent project " + "at the time the snapshot was taken.", ), sa.Column( "parent_uuid", diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 2710c72d5ef..f8442091411 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -15,6 +15,17 @@ from simcore_postgres_database.models.snapshots import snapshots from simcore_postgres_database.models.users import users +USERNAME = "me" +PARENT_PROJECT_NAME = "parent" + + +def test_it(): + import os + + cwd = os.getcwd() + + assert True + @pytest.fixture async def engine(pg_engine: Engine): @@ -22,11 +33,13 @@ async def engine(pg_engine: Engine): async with pg_engine.acquire() as conn: # a 'me' user user_id = await conn.scalar( - users.insert().values(**random_user(name="me")).returning(users.c.id) + users.insert().values(**random_user(name=USERNAME)).returning(users.c.id) ) # has a project 'parent' await conn.execute( - projects.insert().values(**random_project(prj_owner=user_id, name="parent")) + projects.insert().values( + **random_project(prj_owner=user_id, name=PARENT_PROJECT_NAME) + ) ) yield pg_engine @@ -42,15 +55,13 @@ async def test_creating_snapshots(engine: Engine): } async def _create_snapshot(child_index: int, parent_prj, conn) -> int: - # copy - # change uuid, and set to invisible + # create project-snapshot prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} prj_dict["name"] += f" [snapshot {child_index}]" prj_dict["uuid"] = uuid3(UUID(parent_prj.uuid), f"snapshot.{child_index}") - prj_dict[ - "creation_date" - ] = parent_prj.last_change_date # state of parent upon copy! + # creation_data = state of parent upon copy! WARNING: changes can be state changes and not project definition? + prj_dict["creation_date"] = parent_prj.last_change_date prj_dict["hidden"] = True prj_dict["published"] = False @@ -85,7 +96,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: # get parent res: ResultProxy = await conn.execute( - projects.select().where(projects.c.name == "parent") + projects.select().where(projects.c.name == PARENT_PROJECT_NAME) ) parent_prj: Optional[RowProxy] = await res.first() @@ -105,14 +116,21 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: ).first() assert updated_parent_prj - assert updated_parent_prj.id == parent_prj.id assert updated_parent_prj.description != parent_prj.description + assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date # take another snapshot snapshot_two_id = await _create_snapshot(1, updated_parent_prj, conn) - assert snapshot_one_id != snapshot_two_id + snapshot_two = await ( + await conn.execute( + snapshots.select().where(snapshots.c.id == snapshot_two_id) + ) + ).first() + + assert snapshot_two.id != snapshot_one_id + assert snapshot_two.created_at == updated_parent_prj.last_change_date # get project corresponding to snapshot 1 j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) @@ -125,12 +143,17 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: ).first() assert selected_snapshot_project + assert selected_snapshot_project.uuid == snapshot_two.projects_uuid + assert parent_prj.uuid == snapshot_two.parent_uuid def extract(t): return {k: t[k] for k in t if k not in exclude.union({"name"})} assert extract(selected_snapshot_project) == extract(updated_parent_prj) + # TODO: if we call to take consecutive snapshots ... of the same thing, it should + # return existing + def test_deleting_snapshots(): # test delete child project -> deletes snapshot From 0edd132fae0759e995f9c2000fb50b9ae9513f7c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:33:07 +0200 Subject: [PATCH 091/137] db snapshots pass --- .../models/snapshots.py | 4 +- .../postgres-database/tests/test_snapshots.py | 41 +++++++------------ 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index cf3c7dfd6e7..f2dda1ce4a4 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -1,5 +1,4 @@ import sqlalchemy as sa -from sqlalchemy.sql import func from .base import metadata @@ -18,8 +17,7 @@ "created_at", sa.DateTime(), nullable=False, - server_default=func.now(), - doc="Timestamp on creation. " + doc="Timestamp for this snapshot." "It corresponds to the last_change_date of the parent project " "at the time the snapshot was taken.", ), diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index f8442091411..12879727072 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -19,14 +19,6 @@ PARENT_PROJECT_NAME = "parent" -def test_it(): - import os - - cwd = os.getcwd() - - assert True - - @pytest.fixture async def engine(pg_engine: Engine): # injects ... @@ -55,6 +47,8 @@ async def test_creating_snapshots(engine: Engine): } async def _create_snapshot(child_index: int, parent_prj, conn) -> int: + # NOTE: used as prototype + # create project-snapshot prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} @@ -83,7 +77,8 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: snapshot_id = await conn.scalar( snapshots.insert() .values( - name=f"Snapshot {child_index}", + name=f"Snapshot {child_index} [{parent_prj.name}]", + created_at=parent_prj.last_change_date, parent_uuid=parent_prj.uuid, child_index=child_index, project_uuid=project_uuid, @@ -93,7 +88,6 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: return snapshot_id async with engine.acquire() as conn: - # get parent res: ResultProxy = await conn.execute( projects.select().where(projects.c.name == PARENT_PROJECT_NAME) @@ -103,7 +97,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert parent_prj # take one snapshot - snapshot_one_id = await _create_snapshot(0, parent_prj, conn) + first_snapshot_id = await _create_snapshot(0, parent_prj, conn) # modify parent updated_parent_prj = await ( @@ -121,30 +115,31 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date # take another snapshot - snapshot_two_id = await _create_snapshot(1, updated_parent_prj, conn) + second_snapshot_id = await _create_snapshot(1, updated_parent_prj, conn) - snapshot_two = await ( + second_snapshot = await ( await conn.execute( - snapshots.select().where(snapshots.c.id == snapshot_two_id) + snapshots.select().where(snapshots.c.id == second_snapshot_id) ) ).first() - assert snapshot_two.id != snapshot_one_id - assert snapshot_two.created_at == updated_parent_prj.last_change_date + assert second_snapshot + assert second_snapshot.id != first_snapshot_id + assert second_snapshot.created_at == updated_parent_prj.last_change_date - # get project corresponding to snapshot 1 + # get project corresponding to first snapshot j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) selected_snapshot_project = await ( await conn.execute( projects.select() .select_from(j) - .where(snapshots.c.id == snapshot_two_id) + .where(snapshots.c.id == second_snapshot_id) ) ).first() assert selected_snapshot_project - assert selected_snapshot_project.uuid == snapshot_two.projects_uuid - assert parent_prj.uuid == snapshot_two.parent_uuid + assert selected_snapshot_project.uuid == second_snapshot.project_uuid + assert parent_prj.uuid == second_snapshot.parent_uuid def extract(t): return {k: t[k] for k in t if k not in exclude.union({"name"})} @@ -162,9 +157,3 @@ def test_deleting_snapshots(): # test delete parent project -> deletes snapshots # test delete snapshot does NOT delete parent pass - - -def test_create_pydantic_models_from_sqlalchemy_tables(): - # SEE https://docs.sqlalchemy.org/en/14/core/metadata.html - # SEE https://github.com/tiangolo/pydantic-sqlalchemy/blob/master/pydantic_sqlalchemy/main.py - pass From 0c670b1f75a2b16c778d56e51c2adb3b37d3b2cb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 12:58:35 +0200 Subject: [PATCH 092/137] updates web-server oas --- .../schemas/node-meta-v0.0.1-converted.yaml | 8 +- api/specs/webserver/openapi-projects.yaml | 175 +++++++++--- api/specs/webserver/openapi.yaml | 4 +- .../api/v0/openapi.yaml | 18 +- .../api/v0/schemas/node-meta-v0.0.1.json | 9 +- .../api/v0/schemas/node-meta-v0.0.1.json | 9 +- .../api/v0/openapi.yaml | 263 ++++++++++++++++++ .../api/v0/schemas/node-meta-v0.0.1.json | 9 +- .../sandbox/projects_openapi_generator.py | 43 ++- 9 files changed, 458 insertions(+), 80 deletions(-) diff --git a/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml b/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml index 912b978a09a..3a39e566cd4 100644 --- a/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml +++ b/api/specs/common/schemas/node-meta-v0.0.1-converted.yaml @@ -119,11 +119,11 @@ properties: - type properties: displayOrder: + description: >- + DEPRECATED: new display order is taken from the item position. + This property will be removed. + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 1a66a9011cc..c310d1c7977 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -81,6 +81,7 @@ paths: $ref: "#/components/schemas/ProjectEnveloped" default: $ref: "#/components/responses/DefaultErrorResponse" + /projects/active: get: tags: @@ -437,49 +438,101 @@ paths: projects_snapshots: get: - summary: List Project Snapshots + summary: List Snapshots + operationId: list_snapshots + tags: + - project parameters: - - in: path - name: project_id - required: true - schema: - format: uuid - title: Project Id - type: string + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + type: integer + description: index to the first item to return (pagination) + default: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50 + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query responses: - '200': + "200": + description: Successful Response content: application/json: - schema: {} - description: Successful Response + schema: + title: Response List Snapshots Projects Project Id Snapshots Get + type: array + items: + $ref: "#/components/schemas/Snapshot" projects_id_snapshots_id: get: - summary: Get Project Snapshot + summary: Get Snapshot + operationId: get_snapshot + tags: + - project parameters: - - in: path - name: project_id - required: true - schema: - format: uuid - title: Project Id - type: string - - in: path - name: snapshot_id - required: true - schema: - exclusiveMinimum: 0.0 - title: Snapshot Id - type: integer + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path responses: - '200': + "200": + description: Successful Response content: application/json: - schema: {} - description: Successful Response + schema: + $ref: "#/components/schemas/Snapshot" - projects_id_snapshots_id_parameters: - get: + post: + summary: Create Snapshot + operationId: create_snapshot + parameters: + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query + responses: + "200": + description: Successful Response + content: + application/json: + schema: + $ref: "#/components/schemas/Snapshot" components: @@ -524,6 +577,66 @@ components: RunningServiceEnveloped: $ref: "../common/schemas/running_service.yaml#/components/schemas/RunningServiceEnveloped" + Snapshot: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that + parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri + responses: DefaultErrorResponse: $ref: "./openapi.yaml#/components/responses/DefaultErrorResponse" diff --git a/api/specs/webserver/openapi.yaml b/api/specs/webserver/openapi.yaml index 518bf6b64a9..e24e7bf1320 100644 --- a/api/specs/webserver/openapi.yaml +++ b/api/specs/webserver/openapi.yaml @@ -216,8 +216,8 @@ paths: /projects/{project_id}/snapshots/{snapshot_id}: $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id" - /projects/{project_id}/snapshots/{snapshot_id}/parameters: - $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id_parameters" + # /projects/{project_id}/snapshots/{snapshot_id}/parameters: + # $ref: "./openapi-projects.yaml#/paths/projects_id_snapshots_id_parameters" # ACTIVITY ------------------------------------------------------------------------- /activity/status: diff --git a/services/director/src/simcore_service_director/api/v0/openapi.yaml b/services/director/src/simcore_service_director/api/v0/openapi.yaml index 74d5fc9afb5..4ce769b02a8 100644 --- a/services/director/src/simcore_service_director/api/v0/openapi.yaml +++ b/services/director/src/simcore_service_director/api/v0/openapi.yaml @@ -256,11 +256,9 @@ paths: - type properties: displayOrder: + description: 'DEPRECATED: new display order is taken from the item position. This property will be removed.' + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property @@ -649,11 +647,9 @@ paths: - type properties: displayOrder: + description: 'DEPRECATED: new display order is taken from the item position. This property will be removed.' + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property @@ -2436,11 +2432,9 @@ components: - type properties: displayOrder: + description: 'DEPRECATED: new display order is taken from the item position. This property will be removed.' + deprecated: true type: number - description: use this to numerically sort the properties for display - example: - - 1 - - -0.2 label: type: string description: short name for the property diff --git a/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json b/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json index 660045befab..4fe1aa31465 100644 --- a/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json +++ b/services/director/src/simcore_service_director/api/v0/schemas/node-meta-v0.0.1.json @@ -180,12 +180,9 @@ ], "properties": { "displayOrder": { - "type": "number", - "description": "use this to numerically sort the properties for display", - "examples": [ - 1, - -0.2 - ] + "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.", + "deprecated": true, + "type": "number" }, "label": { "type": "string", diff --git a/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json b/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json index 660045befab..4fe1aa31465 100644 --- a/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json +++ b/services/storage/src/simcore_service_storage/api/v0/schemas/node-meta-v0.0.1.json @@ -180,12 +180,9 @@ ], "properties": { "displayOrder": { - "type": "number", - "description": "use this to numerically sort the properties for display", - "examples": [ - 1, - -0.2 - ] + "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.", + "deprecated": true, + "type": "number" }, "label": { "type": "string", diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index b6070e0df0e..5e07b331655 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13244,6 +13244,269 @@ paths: message: Password is not secure field: pasword status: 400 + '/projects/{project_id}/snapshots': + get: + summary: List Snapshots + operationId: list_snapshots_projects__project_id__snapshots_get + tags: + - project + parameters: + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - description: index to the first item to return (pagination) + required: false + schema: + title: Offset + type: integer + description: index to the first item to return (pagination) + default: 0 + name: offset + in: query + - description: maximum number of items to return (pagination) + required: false + schema: + title: Limit + maximum: 50 + minimum: 1 + type: integer + description: maximum number of items to return (pagination) + default: 20 + name: limit + in: query + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: Response List Snapshots Projects Project Id Snapshots Get + type: array + items: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri + '/projects/{project_id}/snapshots/{snapshot_id}': + get: + summary: Get Snapshot + operationId: get_snapshot_projects__project_id__snapshots__snapshot_id__get + tags: + - project + parameters: + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri + post: + summary: Create Snapshot + operationId: create_snapshot_projects__project_id__snapshots_post + parameters: + - required: true + schema: + title: Project Id + type: string + format: uuid + name: project_id + in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query + responses: + '200': + description: Successful Response + content: + application/json: + schema: + title: Snapshot + required: + - id + - parent_id + - project_id + - url + - url_parent + - url_project + type: object + properties: + id: + title: Id + type: integer + description: Unique snapshot identifier + label: + title: Label + type: string + description: Unique human readable display name + created_at: + title: Created At + type: string + description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time + format: date-time + parent_id: + title: Parent Id + type: string + description: Parent's project uuid + format: uuid + project_id: + title: Project Id + type: string + description: Current project's uuid + format: uuid + url: + title: Url + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parent: + title: Url Parent + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_project: + title: Url Project + maxLength: 65536 + minLength: 1 + type: string + format: uri + url_parameters: + title: Url Parameters + maxLength: 65536 + minLength: 1 + type: string + format: uri /activity/status: get: operationId: get_status diff --git a/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json b/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json index 660045befab..4fe1aa31465 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json +++ b/services/web/server/src/simcore_service_webserver/api/v0/schemas/node-meta-v0.0.1.json @@ -180,12 +180,9 @@ ], "properties": { "displayOrder": { - "type": "number", - "description": "use this to numerically sort the properties for display", - "examples": [ - 1, - -0.2 - ] + "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.", + "deprecated": true, + "type": "number" }, "label": { "type": "string", diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py index 58467dddc1f..5c0baf38cf5 100644 --- a/services/web/server/tests/sandbox/projects_openapi_generator.py +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -10,7 +10,7 @@ from fastapi import Depends, FastAPI from fastapi import Path as PathParam -from fastapi import Request, status +from fastapi import Query, Request, status from fastapi.exceptions import HTTPException from models_library.services import PROPERTY_KEY_RE from pydantic import ( @@ -145,56 +145,66 @@ def get_valid_id(project_id: UUID = PathParam(...)) -> UUID: #################################################################### -@app.get("/projects/{project_id}", response_model=Project) +@app.get("/projects/{project_id}", response_model=Project, tags=["project"]) def get_project(pid: UUID = Depends(get_valid_id)): return _PROJECTS[pid] -@app.post("/projects/{project_id}") +@app.post("/projects/{project_id}", tags=["project"]) def create_project(project: Project): if project.id not in _PROJECTS: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Invalid id") _PROJECTS[project.id] = project -@app.put("/projects/{project_id}") +@app.put("/projects/{project_id}", tags=["project"]) def replace_project(project: Project, pid: UUID = Depends(get_valid_id)): _PROJECTS[pid] = project -@app.patch("/projects/{project_id}") +@app.patch("/projects/{project_id}", tags=["project"]) def update_project(project: Project, pid: UUID = Depends(get_valid_id)): raise NotImplementedError() -@app.delete("/projects/{project_id}") +@app.delete("/projects/{project_id}", tags=["project"]) def delete_project(pid: UUID = Depends(get_valid_id)): del _PROJECTS[pid] -@app.post("/projects/{project_id}:open") +@app.post("/projects/{project_id}:open", tags=["project"]) def open_project(pid: UUID = Depends(get_valid_id)): pass -@app.post("/projects/{project_id}:start") +@app.post("/projects/{project_id}:start", tags=["project"]) def start_project(use_cache: bool = True, pid: UUID = Depends(get_valid_id)): pass -@app.post("/projects/{project_id}:stop") +@app.post("/projects/{project_id}:stop", tags=["project"]) def stop_project(pid: UUID = Depends(get_valid_id)): pass -@app.post("/projects/{project_id}:close") +@app.post("/projects/{project_id}:close", tags=["project"]) def close_project(pid: UUID = Depends(get_valid_id)): pass -@app.get("/projects/{project_id}/snapshots", response_model=List[SnapshotApiModel]) +@app.get( + "/projects/{project_id}/snapshots", + response_model=List[SnapshotApiModel], + tags=["project"], +) async def list_snapshots( pid: UUID = Depends(get_valid_id), + offset: PositiveInt = Query( + 0, description="index to the first item to return (pagination)" + ), + limit: int = Query( + 20, description="maximum number of items to return (pagination)", ge=1, le=50 + ), url_for: Callable = Depends(get_reverse_url_mapper), ): psid = _PROJECT2SNAPSHOT.get(pid) @@ -209,7 +219,11 @@ async def list_snapshots( ] -@app.post("/projects/{project_id}/snapshots", response_model=SnapshotApiModel) +@app.post( + "/projects/{project_id}/snapshots", + response_model=SnapshotApiModel, + tags=["project"], +) async def create_snapshot( pid: UUID = Depends(get_valid_id), snapshot_label: Optional[str] = None, @@ -265,6 +279,7 @@ async def create_snapshot( @app.get( "/projects/{project_id}/snapshots/{snapshot_id}", response_model=SnapshotApiModel, + tags=["project"], ) async def get_snapshot( snapshot_id: PositiveInt, @@ -286,6 +301,7 @@ async def get_snapshot( @app.get( "/projects/{project_id}/snapshots/{snapshot_id}/parameters", response_model=List[ParameterApiModel], + tags=["project"], ) async def list_snapshot_parameters( snapshot_id: str, @@ -331,6 +347,7 @@ def create_snapshots(project_id: UUID): # print("-"*100) -print(json.dumps(app.openapi(), indent=2)) +with open("openapi-ignore.json", "wt") as f: + json.dump(app.openapi(), f, indent=2) # uvicorn --reload projects_openapi_generator:app From a7fc1544a65fb045bdb0d91fd1ec506316f05231 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 14:56:49 +0200 Subject: [PATCH 093/137] list and get snapshots --- .../simcore_service_webserver/application.py | 7 +- .../db_base_repository.py | 12 ++ .../simcore_service_webserver/snapshots.py | 3 +- .../snapshots_api_handlers.py | 162 +++++++++--------- .../simcore_service_webserver/snapshots_db.py | 57 ++++++ .../snapshots_models.py | 18 +- .../unit/isolated/test_snapshots_models.py | 2 +- 7 files changed, 176 insertions(+), 85 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/db_base_repository.py create mode 100644 services/web/server/src/simcore_service_webserver/snapshots_db.py diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index d27326004bc..4d79834503b 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -9,8 +9,6 @@ from servicelib.application import create_safe_application from servicelib.rest_pagination_utils import monkey_patch_pydantic_url_regex -monkey_patch_pydantic_url_regex() - from ._meta import WELCOME_MSG from .activity import setup_activity from .catalog import setup_catalog @@ -31,6 +29,7 @@ from .security import setup_security from .session import setup_session from .settings import setup_settings +from .snapshots import setup_snapshots from .socketio import setup_socketio from .statics import setup_statics from .storage import setup_storage @@ -40,6 +39,9 @@ from .tracing import setup_app_tracing from .users import setup_users +monkey_patch_pydantic_url_regex() + + log = logging.getLogger(__name__) @@ -75,6 +77,7 @@ def create_application(config: Dict[str, Any]) -> web.Application: setup_users(app) setup_groups(app) setup_projects(app) + setup_snapshots(app) setup_activity(app) setup_resource_manager(app) setup_tags(app) diff --git a/services/web/server/src/simcore_service_webserver/db_base_repository.py b/services/web/server/src/simcore_service_webserver/db_base_repository.py new file mode 100644 index 00000000000..28012be4fe0 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/db_base_repository.py @@ -0,0 +1,12 @@ +# a repo: retrieves engine from app +# TODO: a repo: some members acquire and retrieve connection +# TODO: a repo: any validation error in a repo is due to corrupt data in db! + +from aiohttp import web + +from .constants import APP_DB_ENGINE_KEY + + +class BaseRepository: + def __init__(self, app: web.Application): + self.engine = app[APP_DB_ENGINE_KEY] diff --git a/services/web/server/src/simcore_service_webserver/snapshots.py b/services/web/server/src/simcore_service_webserver/snapshots.py index 47137d5bf05..3c331800650 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots.py +++ b/services/web/server/src/simcore_service_webserver/snapshots.py @@ -27,10 +27,11 @@ depends=["simcore_service_webserver.projects"], logger=log, ) -def setup(app: web.Application): +def setup_snapshots(app: web.Application): settings: ApplicationSettings = app[APP_SETTINGS_KEY] if not settings.WEBSERVER_DEV_FEATURES_ENABLED: raise SkipModuleSetup(reason="Development feature") + # TODO: validate routes against OAS app.add_routes(snapshots_api_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index a0ff98a6b27..4db60275c85 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -1,19 +1,33 @@ -import json from functools import wraps -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, List, Optional from uuid import UUID +import orjson from aiohttp import web -from models_library.projects import Project from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError -from simcore_service_webserver.snapshots_models import Snapshot +from pydantic.main import BaseModel from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY -from .snapshots_models import Snapshot, SnapshotApiModel +from .snapshots_db import SnapshotsRepository +from .snapshots_models import Snapshot, SnapshotItem -json_dumps = json.dumps + +def _default(obj): + if isinstance(obj, BaseModel): + return obj.dict() + raise TypeError + + +def json_dumps(v) -> str: + # orjson.dumps returns bytes, to match standard json.dumps we need to decode + return orjson.dumps(v, default=_default).decode() + + +def enveloped_response(data: Any) -> web.Response: + enveloped: str = json_dumps({"data": data}) + return web.Response(text=enveloped, content_type="application/json") def handle_request_errors(handler: Callable): @@ -47,127 +61,121 @@ async def wrapped(request: web.Request): @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots", - name="_list_snapshots_handler", + name="list_project_snapshots_handler", ) @handle_request_errors -async def _list_snapshots_handler(request: web.Request): +async def list_project_snapshots_handler(request: web.Request): """ Lists references on project snapshots """ user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - snapshots = await list_snapshots( + snapshots_repo = SnapshotsRepository(request.app) + + @validate_arguments + async def list_snapshots(project_id: UUID) -> List[Snapshot]: + # project_id is param-project? + # TODO: add pagination + # TODO: optimizaiton will grow snapshots of a project with time! + # + snapshots_orm = await snapshots_repo.list(project_id) + # snapshots: + # - ordered (iterations!) + # - have a parent project with all the parametrization + + return [Snapshot.from_orm(obj) for obj in snapshots_orm] + + snapshots: List[Snapshot] = await list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) - # Append url links - url_for_snapshot = request.app.router["_get_snapshot_handler"].url_for - url_for_parameters = request.app.router["_get_snapshot_parameters_handler"].url_for + # TODO: async for snapshot in await list_snapshot is the same? - for snp in snapshots: - snp["url"] = url_for_snapshot( - project_id=snp["parent_id"], snapshot_id=snp["id"] - ) - snp["url_parameters"] = url_for_parameters( - project_id=snp["parent_id"], snapshot_id=snp["id"] - ) - # snp['url_project'] = + data = [] + for snapshot in snapshots: + # FIXME: raw dict + data.append(SnapshotItem.from_snapshot(snapshot, request.app).dict()) - return snapshots + return enveloped_response(data) @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}", - name="_get_snapshot_handler", + name="get_project_snapshot_handler", ) @handle_request_errors -async def _get_snapshot_handler(request: web.Request): +async def get_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + # FIXME: access rights ?? + + snapshots_repo = SnapshotsRepository(request.app) + + @validate_arguments + async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: + try: + snapshot_orm = await snapshots_repo.get_by_index( + project_id, int(snapshot_id) + ) + except ValueError: + snapshot_orm = await snapshots_repo.get_by_name(project_id, snapshot_id) + + if not snapshot_orm: + raise web.HTTPNotFound(reason=f"snapshot {snapshot_id} not found") + + return Snapshot.from_orm(snapshot_orm) + snapshot = await get_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) - return snapshot.json() + return enveloped_response(snapshot) @routes.post( f"/{vtag}/projects/{{project_id}}/snapshots", - name="_create_snapshot_handler", ) @handle_request_errors -async def _create_snapshot_handler(request: web.Request): +async def create_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + snapshots_repo = (SnapshotsRepository(request.app),) + + @validate_arguments + async def create_snapshot( + project_id: UUID, + snapshot_label: Optional[str] = None, + ) -> Snapshot: + raise NotImplementedError snapshot = await create_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_label=request.query.get("snapshot_label"), ) - return snapshot.json() + return enveloped_response(snapshots) @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", - name="_get_snapshot_parameters_handler", + name="get_snapshot_parameters_handler", ) @handle_request_errors -async def _get_snapshot_parameters_handler( +async def get_project_snapshot_parameters_handler( request: web.Request, ): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + @validate_arguments + async def get_snapshot_parameters( + project_id: UUID, + snapshot_id: str, + ): + # + return {"x": 4, "y": "yes"} + params = await get_snapshot_parameters( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) return params - - -# API ROUTES HANDLERS --------------------------------------------------------- - - -@validate_arguments -async def list_snapshots(project_id: UUID) -> List[Dict[str, Any]]: - # project_id is param-project? - # TODO: add pagination - # TODO: optimizaiton will grow snapshots of a project with time! - # - - # snapshots: - # - ordered (iterations!) - # - have a parent project with all the parametrization - # - snapshot_info_0 = { - "id": 0, - "display_name": "snapshot 0", - "parent_id": project_id, - "parameters": get_snapshot_parameters(project_id, snapshot_id=str(id)), - } - - return [ - snapshot_info_0, - ] - - -@validate_arguments -async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: - # TODO: create a fake project - # - generate project_id - # - define what changes etc... - pass - - -@validate_arguments -async def create_snapshot( - project_id: UUID, - snapshot_label: Optional[str] = None, -) -> Snapshot: - pass - - -@validate_arguments -async def get_snapshot_parameters(project_id: UUID, snapshot_id: str): - # - return {"x": 4, "y": "yes"} diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py new file mode 100644 index 00000000000..7fb2bc84759 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -0,0 +1,57 @@ +from typing import List, Optional +from uuid import UUID + +from aiopg.sa.result import RowProxy +from pydantic import PositiveInt +from simcore_postgres_database.models.snapshots import snapshots + +from .db_base_repository import BaseRepository + +# alias for readability +# SEE https://pydantic-docs.helpmanual.io/usage/models/#orm-mode-aka-arbitrary-class-instances +SnapshotOrm = RowProxy + + +class SnapshotsRepository(BaseRepository): + """ + Abstracts access to snapshots database table + + Gets primitive/standard parameters and returns valid orm objects + """ + + async def list(self, projec_uuid: UUID) -> List[SnapshotOrm]: + result = [] + async with self.engine.acquire() as conn: + stmt = ( + snapshots.select() + .where(snapshots.c.parent_uuid == projec_uuid) + .order_by(snapshots.c.child_index) + ) + async for row in conn.execute(stmt): + result.append(row) + return result + + async def _get(self, stmt) -> Optional[SnapshotOrm]: + async with self.engine.acquire() as conn: + return await (await conn.execute(stmt)).first() + + async def get_by_index( + self, project_uuid: UUID, snapshot_index: PositiveInt + ) -> Optional[SnapshotOrm]: + stmt = snapshots.select().where( + (snapshots.c.parent_uuid == project_uuid) + & (snapshots.c.child_index == snapshot_index) + ) + return await self._get(stmt) + + async def get_by_name( + self, project_uuid: UUID, snapshot_name: str + ) -> Optional[SnapshotOrm]: + stmt = snapshots.select().where( + (snapshots.c.parent_uuid == project_uuid) + & (snapshots.c.name == snapshot_name) + ) + return await self._get(stmt) + + async def create(self, project_uuid: UUID) -> SnapshotOrm: + pass diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index f1d3ad54885..573b6e50154 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -2,6 +2,7 @@ from typing import Callable, Optional, Union from uuid import UUID +from aiohttp import web from models_library.projects_nodes import OutputID from pydantic import ( AnyUrl, @@ -15,6 +16,7 @@ BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] + ## Domain models -------- class Parameter(BaseModel): name: str @@ -36,6 +38,9 @@ class Snapshot(BaseModel): parent_id: UUID = Field(..., description="Parent's project uuid") project_id: UUID = Field(..., description="Current project's uuid") + class Config: + orm_mode = True + ## API models ---------- @@ -45,24 +50,29 @@ class ParameterApiModel(Parameter): # url_output: AnyUrl -class SnapshotApiModel(Snapshot): +class SnapshotItem(Snapshot): + """ API model for an array item of snapshots """ + url: AnyUrl url_parent: AnyUrl url_project: AnyUrl url_parameters: Optional[AnyUrl] = None @classmethod - def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiModel": + def from_snapshot(cls, snapshot: Snapshot, app: web.Application) -> "SnapshotItem": + def url_for(router_name: str, **params): + return app.router[router_name].url_for(**params) + return cls( url=url_for( - "get_snapshot", + "get_project_snapshot_handler", project_id=snapshot.project_id, snapshot_id=snapshot.id, ), url_parent=url_for("get_project", project_id=snapshot.parent_id), url_project=url_for("get_project", project_id=snapshot.project_id), url_parameters=url_for( - "get_snapshot_parameters", + "get_project_snapshot_parameters_handler", project_id=snapshot.parent_id, snapshot_id=snapshot.id, ), diff --git a/services/web/server/tests/unit/isolated/test_snapshots_models.py b/services/web/server/tests/unit/isolated/test_snapshots_models.py index 21933a6d667..87be075f0a5 100644 --- a/services/web/server/tests/unit/isolated/test_snapshots_models.py +++ b/services/web/server/tests/unit/isolated/test_snapshots_models.py @@ -3,5 +3,5 @@ Parameter, ParameterApiModel, Snapshot, - SnapshotApiModel, + SnapshotItem, ) From 50ec88c4efdc88a614f142154c5fdda7a98363f6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:17:42 +0200 Subject: [PATCH 094/137] fixes OAS --- api/specs/webserver/openapi-projects.yaml | 47 ++++++++++--------- .../api/v0/openapi.yaml | 42 +++++++++-------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index c310d1c7977..8a2acab3f89 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -439,7 +439,7 @@ paths: projects_snapshots: get: summary: List Snapshots - operationId: list_snapshots + operationId: list_project_snapshots_handler tags: - project parameters: @@ -480,20 +480,12 @@ paths: type: array items: $ref: "#/components/schemas/Snapshot" - - projects_id_snapshots_id: - get: - summary: Get Snapshot - operationId: get_snapshot + post: + summary: Create Snapshot + operationId: create_project_snapshot_handler tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -501,6 +493,12 @@ paths: format: uuid name: project_id in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query responses: "200": description: Successful Response @@ -509,10 +507,19 @@ paths: schema: $ref: "#/components/schemas/Snapshot" - post: - summary: Create Snapshot - operationId: create_snapshot + projects_id_snapshots_id: + get: + summary: Get Snapshot + operationId: get_project_snapshot_handler + tags: + - project parameters: + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path - required: true schema: title: Project Id @@ -520,12 +527,6 @@ paths: format: uuid name: project_id in: path - - required: false - schema: - title: Snapshot Label - type: string - name: snapshot_label - in: query responses: "200": description: Successful Response @@ -534,7 +535,6 @@ paths: schema: $ref: "#/components/schemas/Snapshot" - components: schemas: ClientSessionId: @@ -599,7 +599,8 @@ components: created_at: title: Created At type: string - description: Timestamp of the time snapshot was taken from parent. Notice that + description: + Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time parent_id: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 5e07b331655..6a97055351d 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13247,7 +13247,7 @@ paths: '/projects/{project_id}/snapshots': get: summary: List Snapshots - operationId: list_snapshots_projects__project_id__snapshots_get + operationId: list_project_snapshots_handler tags: - project parameters: @@ -13344,19 +13344,12 @@ paths: minLength: 1 type: string format: uri - '/projects/{project_id}/snapshots/{snapshot_id}': - get: - summary: Get Snapshot - operationId: get_snapshot_projects__project_id__snapshots__snapshot_id__get + post: + summary: Create Snapshot + operationId: create_project_snapshot_handler tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -13364,6 +13357,12 @@ paths: format: uuid name: project_id in: path + - required: false + schema: + title: Snapshot Label + type: string + name: snapshot_label + in: query responses: '200': description: Successful Response @@ -13427,10 +13426,19 @@ paths: minLength: 1 type: string format: uri - post: - summary: Create Snapshot - operationId: create_snapshot_projects__project_id__snapshots_post + '/projects/{project_id}/snapshots/{snapshot_id}': + get: + summary: Get Snapshot + operationId: get_project_snapshot_handler + tags: + - project parameters: + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path - required: true schema: title: Project Id @@ -13438,12 +13446,6 @@ paths: format: uuid name: project_id in: path - - required: false - schema: - title: Snapshot Label - type: string - name: snapshot_label - in: query responses: '200': description: Successful Response From 4d35ea1e7d240dee92222d7918f7de7a2c02bce7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:18:38 +0200 Subject: [PATCH 095/137] setup snapshot app module and connects handlers to routes --- .../application_config.py | 1 + .../projects/module_setup.py | 5 +++- .../snapshots_api_handlers.py | 24 +++++++++++++------ .../simcore_service_webserver/snapshots_db.py | 6 ++--- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/application_config.py b/services/web/server/src/simcore_service_webserver/application_config.py index d35d7974fd0..8187ffdd069 100644 --- a/services/web/server/src/simcore_service_webserver/application_config.py +++ b/services/web/server/src/simcore_service_webserver/application_config.py @@ -94,6 +94,7 @@ def create_schema() -> T.Dict: addon_section("studies_access", optional=True): minimal_addon_schema(), addon_section("studies_dispatcher", optional=True): minimal_addon_schema(), addon_section("exporter", optional=True): minimal_addon_schema(), + addon_section("snapshots", optional=True): minimal_addon_schema(), } ) diff --git a/services/web/server/src/simcore_service_webserver/projects/module_setup.py b/services/web/server/src/simcore_service_webserver/projects/module_setup.py index c723f4e8abe..188d660a164 100644 --- a/services/web/server/src/simcore_service_webserver/projects/module_setup.py +++ b/services/web/server/src/simcore_service_webserver/projects/module_setup.py @@ -42,7 +42,10 @@ def _create_routes(tag, specs, *handlers_module, disable_login: bool = False): routes = map_handlers_with_operations( handlers, - filter(lambda o: tag in o[3], iter_path_operations(specs)), + filter( + lambda o: tag in o[3] and "snapshot" not in o[2], + iter_path_operations(specs), + ), strict=True, ) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 4db60275c85..74cb099b9eb 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -10,6 +10,8 @@ from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY +from .login.decorators import login_required +from .security_decorators import permission_required from .snapshots_db import SnapshotsRepository from .snapshots_models import Snapshot, SnapshotItem @@ -25,8 +27,8 @@ def json_dumps(v) -> str: return orjson.dumps(v, default=_default).decode() -def enveloped_response(data: Any) -> web.Response: - enveloped: str = json_dumps({"data": data}) +def enveloped_response(data: Any, **extra) -> web.Response: + enveloped: str = json_dumps({"data": data, **extra}) return web.Response(text=enveloped, content_type="application/json") @@ -43,7 +45,7 @@ async def wrapped(request: web.Request): except KeyError as err: # NOTE: handles required request.match_info[*] or request.query[*] - raise web.HTTPBadRequest(reason="Expected parameter {err}") from err + raise web.HTTPBadRequest(reason=f"Expected parameter {err}") from err except ValidationError as err: # NOTE: pydantic.validate_arguments parses and validates -> ValidationError @@ -63,6 +65,8 @@ async def wrapped(request: web.Request): f"/{vtag}/projects/{{project_id}}/snapshots", name="list_project_snapshots_handler", ) +@login_required +@permission_required("project.read") @handle_request_errors async def list_project_snapshots_handler(request: web.Request): """ @@ -103,6 +107,8 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}", name="get_project_snapshot_handler", ) +@login_required +@permission_required("project.read") @handle_request_errors async def get_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] @@ -133,32 +139,36 @@ async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: @routes.post( - f"/{vtag}/projects/{{project_id}}/snapshots", + f"/{vtag}/projects/{{project_id}}/snapshots", name="create_project_snapshot_handler" ) +@login_required +@permission_required("project.create") @handle_request_errors async def create_project_snapshot_handler(request: web.Request): user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] snapshots_repo = (SnapshotsRepository(request.app),) @validate_arguments - async def create_snapshot( + async def _create_snapshot( project_id: UUID, snapshot_label: Optional[str] = None, ) -> Snapshot: raise NotImplementedError - snapshot = await create_snapshot( + snapshot = await _create_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_label=request.query.get("snapshot_label"), ) - return enveloped_response(snapshots) + return enveloped_response(snapshot) @routes.get( f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", name="get_snapshot_parameters_handler", ) +@login_required +@permission_required("project.read") @handle_request_errors async def get_project_snapshot_parameters_handler( request: web.Request, diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 7fb2bc84759..3903dd37b60 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -24,7 +24,7 @@ async def list(self, projec_uuid: UUID) -> List[SnapshotOrm]: async with self.engine.acquire() as conn: stmt = ( snapshots.select() - .where(snapshots.c.parent_uuid == projec_uuid) + .where(snapshots.c.parent_uuid == str(projec_uuid)) .order_by(snapshots.c.child_index) ) async for row in conn.execute(stmt): @@ -39,7 +39,7 @@ async def get_by_index( self, project_uuid: UUID, snapshot_index: PositiveInt ) -> Optional[SnapshotOrm]: stmt = snapshots.select().where( - (snapshots.c.parent_uuid == project_uuid) + (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.child_index == snapshot_index) ) return await self._get(stmt) @@ -48,7 +48,7 @@ async def get_by_name( self, project_uuid: UUID, snapshot_name: str ) -> Optional[SnapshotOrm]: stmt = snapshots.select().where( - (snapshots.c.parent_uuid == project_uuid) + (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.name == snapshot_name) ) return await self._get(stmt) From bf8cc5dcb8c69a8c57531545506dc3ed67bf3a24 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:39:23 +0200 Subject: [PATCH 096/137] cleanup --- .../db_base_repository.py | 25 +++++++++++++++-- .../snapshots_api_handlers.py | 28 ++++++++----------- .../snapshots_models.py | 2 +- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/db_base_repository.py b/services/web/server/src/simcore_service_webserver/db_base_repository.py index 28012be4fe0..95c960c29fb 100644 --- a/services/web/server/src/simcore_service_webserver/db_base_repository.py +++ b/services/web/server/src/simcore_service_webserver/db_base_repository.py @@ -2,11 +2,30 @@ # TODO: a repo: some members acquire and retrieve connection # TODO: a repo: any validation error in a repo is due to corrupt data in db! +from typing import Optional + from aiohttp import web +from aiopg.sa.engine import Engine -from .constants import APP_DB_ENGINE_KEY +from .constants import APP_DB_ENGINE_KEY, RQT_USERID_KEY class BaseRepository: - def __init__(self, app: web.Application): - self.engine = app[APP_DB_ENGINE_KEY] + """ + Shall be created on every request + + """ + + def __init__(self, request: web.Request): + # user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + self._engine: Engine = request.app[APP_DB_ENGINE_KEY] + self._user_id: Optional[int] = request.get(RQT_USERID_KEY) + + @property + def engine(self) -> Engine: + return self._engine + + @property + def user_id(self) -> Optional[int]: + return self._user_id diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 74cb099b9eb..f22759a84d6 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -57,6 +57,10 @@ async def wrapped(request: web.Request): return wrapped +# FIXME: access rights using same approach as in access_layer.py in storage. +# A user can only check snapshots (subresource) of its project (parent resource) + + # API ROUTES HANDLERS --------------------------------------------------------- routes = web.RouteTableDef() @@ -72,12 +76,10 @@ async def list_project_snapshots_handler(request: web.Request): """ Lists references on project snapshots """ - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - snapshots_repo = SnapshotsRepository(request.app) + snapshots_repo = SnapshotsRepository(request) @validate_arguments - async def list_snapshots(project_id: UUID) -> List[Snapshot]: + async def _list_snapshots(project_id: UUID) -> List[Snapshot]: # project_id is param-project? # TODO: add pagination # TODO: optimizaiton will grow snapshots of a project with time! @@ -89,7 +91,7 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: return [Snapshot.from_orm(obj) for obj in snapshots_orm] - snapshots: List[Snapshot] = await list_snapshots( + snapshots: List[Snapshot] = await _list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) @@ -97,8 +99,7 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: data = [] for snapshot in snapshots: - # FIXME: raw dict - data.append(SnapshotItem.from_snapshot(snapshot, request.app).dict()) + data.append(SnapshotItem.from_snapshot(snapshot, request.app)) return enveloped_response(data) @@ -111,14 +112,10 @@ async def list_snapshots(project_id: UUID) -> List[Snapshot]: @permission_required("project.read") @handle_request_errors async def get_project_snapshot_handler(request: web.Request): - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - # FIXME: access rights ?? - - snapshots_repo = SnapshotsRepository(request.app) + snapshots_repo = SnapshotsRepository(request) @validate_arguments - async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: + async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: try: snapshot_orm = await snapshots_repo.get_by_index( project_id, int(snapshot_id) @@ -131,7 +128,7 @@ async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: return Snapshot.from_orm(snapshot_orm) - snapshot = await get_snapshot( + snapshot = await _get_snapshot( project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) @@ -145,8 +142,7 @@ async def get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: @permission_required("project.create") @handle_request_errors async def create_project_snapshot_handler(request: web.Request): - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - snapshots_repo = (SnapshotsRepository(request.app),) + snapshots_repo = SnapshotsRepository(request) @validate_arguments async def _create_snapshot( diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index 573b6e50154..80a3b4d7464 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Callable, Optional, Union +from typing import Optional, Union from uuid import UUID from aiohttp import web From b0442f83a0baeab9608a5178d008806701a4768e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:43:23 +0200 Subject: [PATCH 097/137] format oas --- api/specs/webserver/openapi-projects.yaml | 12 ++++++------ .../simcore_service_webserver/api/v0/openapi.yaml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 8a2acab3f89..260d3965ebf 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -514,12 +514,6 @@ paths: tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -527,6 +521,12 @@ paths: format: uuid name: project_id in: path + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path responses: "200": description: Successful Response diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 6a97055351d..9aec6d0e910 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13433,12 +13433,6 @@ paths: tags: - project parameters: - - required: true - schema: - title: Snapshot Id - type: integer - name: snapshot_id - in: path - required: true schema: title: Project Id @@ -13446,6 +13440,12 @@ paths: format: uuid name: project_id in: path + - required: true + schema: + title: Snapshot Id + type: integer + name: snapshot_id + in: path responses: '200': description: Successful Response From 1668af078205359208caa80235a24dec266c52a1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:45:15 +0200 Subject: [PATCH 098/137] db migration: adds snapshots table --- .../5860ac6ad178_adds_snapshots_table.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py new file mode 100644 index 00000000000..8b93a8ec5d0 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py @@ -0,0 +1,50 @@ +"""adds snapshots table + +Revision ID: 5860ac6ad178 +Revises: c2d3acc313e1 +Create Date: 2021-08-11 13:21:55.415592+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "5860ac6ad178" +down_revision = "c2d3acc313e1" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "snapshots", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("parent_uuid", sa.String(), nullable=False), + sa.Column("child_index", sa.Integer(), nullable=False), + sa.Column("project_uuid", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["parent_uuid"], + ["projects.uuid"], + name="fk_snapshots_parent_uuid_projects", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_snapshots_project_uuid_projects", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("child_index"), + sa.UniqueConstraint("project_uuid"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("snapshots") + # ### end Alembic commands ### From 21d71973fa2de9470a5f763c80457417e90dbb4b Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 19:00:50 +0200 Subject: [PATCH 099/137] Drops child index from snapshot --- .../src/simcore_postgres_database/errors.py | 47 +++++++++++++++++++ .../5860ac6ad178_adds_snapshots_table.py | 5 +- .../models/snapshots.py | 11 ++--- .../postgres-database/tests/test_snapshots.py | 44 ++++++++++++++--- 4 files changed, 90 insertions(+), 17 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/errors.py diff --git a/packages/postgres-database/src/simcore_postgres_database/errors.py b/packages/postgres-database/src/simcore_postgres_database/errors.py new file mode 100644 index 00000000000..4d8c8e87a53 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/errors.py @@ -0,0 +1,47 @@ +# pylint: disable=unused-import + +# +# StandardError +# |__ Warning +# |__ Error +# |__ InterfaceError +# |__ DatabaseError +# |__ DataError +# |__ OperationalError +# |__ IntegrityError +# |__ InternalError +# |__ ProgrammingError +# |__ NotSupportedError +# +# aiopg reuses DBAPI exceptions +# SEE https://aiopg.readthedocs.io/en/stable/core.html?highlight=Exception#exceptions +# SEE http://initd.org/psycopg/docs/module.html#dbapi-exceptions + + +from psycopg2 import DatabaseError, DataError +from psycopg2 import Error as DBAPIError +from psycopg2 import ( + IntegrityError, + InterfaceError, + InternalError, + NotSupportedError, + OperationalError, + ProgrammingError, +) + +# pylint: disable=no-name-in-module +from psycopg2.errors import UniqueViolation + +# pylint: enable=no-name-in-module + +assert issubclass(UniqueViolation, IntegrityError) # nosec + +# TODO: see https://stackoverflow.com/questions/58740043/how-do-i-catch-a-psycopg2-errors-uniqueviolation-error-in-a-python-flask-app +# from sqlalchemy.exc import IntegrityError +# +# from psycopg2.errors import UniqueViolation +# +# try: +# s.commit() +# except IntegrityError as e: +# assert isinstance(e.orig, UniqueViolation) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py index 8b93a8ec5d0..23699028a2a 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py @@ -23,7 +23,6 @@ def upgrade(): sa.Column("name", sa.String(), nullable=False), sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("parent_uuid", sa.String(), nullable=False), - sa.Column("child_index", sa.Integer(), nullable=False), sa.Column("project_uuid", sa.String(), nullable=False), sa.ForeignKeyConstraint( ["parent_uuid"], @@ -38,8 +37,10 @@ def upgrade(): ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("child_index"), sa.UniqueConstraint("project_uuid"), + sa.UniqueConstraint( + "parent_uuid", "created_at", name="snapshot_from_project_uniqueness" + ), ) # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py index f2dda1ce4a4..db5ac215031 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py @@ -33,14 +33,6 @@ unique=False, doc="UUID of the parent project", ), - sa.Column( - "child_index", - sa.Integer, - nullable=False, - unique=True, - doc="0-based index in order of creation (i.e. 0 being the oldest and N-1 the latest)" - "from the same parent_id", - ), sa.Column( "project_uuid", sa.String, @@ -53,6 +45,9 @@ unique=True, doc="UUID of the project associated to this snapshot", ), + sa.UniqueConstraint( + "parent_uuid", "created_at", name="snapshot_from_project_uniqueness" + ), ) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 12879727072..95d943c3557 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -4,13 +4,14 @@ # pylint: disable=unused-variable from copy import deepcopy -from typing import Optional +from typing import Callable, Optional, Set from uuid import UUID, uuid3 import pytest from aiopg.sa.engine import Engine from aiopg.sa.result import ResultProxy, RowProxy from pytest_simcore.helpers.rawdata_fakers import random_project, random_user +from simcore_postgres_database.errors import UniqueViolation from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.snapshots import snapshots from simcore_postgres_database.models.users import users @@ -36,8 +37,9 @@ async def engine(pg_engine: Engine): yield pg_engine -async def test_creating_snapshots(engine: Engine): - exclude = { +@pytest.fixture +def exclude(): + return { "id", "uuid", "creation_date", @@ -46,8 +48,11 @@ async def test_creating_snapshots(engine: Engine): "published", } + +@pytest.fixture +def create_snapshot(exclude) -> Callable: async def _create_snapshot(child_index: int, parent_prj, conn) -> int: - # NOTE: used as prototype + # NOTE: used as FAKE prototype # create project-snapshot prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} @@ -80,13 +85,19 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: name=f"Snapshot {child_index} [{parent_prj.name}]", created_at=parent_prj.last_change_date, parent_uuid=parent_prj.uuid, - child_index=child_index, project_uuid=project_uuid, ) .returning(snapshots.c.id) ) return snapshot_id + return _create_snapshot + + +async def test_creating_snapshots( + engine: Engine, create_snapshot: Callable, exclude: Set +): + async with engine.acquire() as conn: # get parent res: ResultProxy = await conn.execute( @@ -97,7 +108,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert parent_prj # take one snapshot - first_snapshot_id = await _create_snapshot(0, parent_prj, conn) + first_snapshot_id = await create_snapshot(0, parent_prj, conn) # modify parent updated_parent_prj = await ( @@ -115,7 +126,7 @@ async def _create_snapshot(child_index: int, parent_prj, conn) -> int: assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date # take another snapshot - second_snapshot_id = await _create_snapshot(1, updated_parent_prj, conn) + second_snapshot_id = await create_snapshot(1, updated_parent_prj, conn) second_snapshot = await ( await conn.execute( @@ -150,6 +161,25 @@ def extract(t): # return existing +async def test_multiple_snapshots_of_same_project( + engine: Engine, create_snapshot: Callable +): + async with engine.acquire() as conn: + # get parent + res: ResultProxy = await conn.execute( + projects.select().where(projects.c.name == PARENT_PROJECT_NAME) + ) + parent_prj: Optional[RowProxy] = await res.first() + assert parent_prj + + # take first snapshot + await create_snapshot(0, parent_prj, conn) + + # no changes in the parent! + with pytest.raises(UniqueViolation): + await create_snapshot(1, parent_prj, conn) + + def test_deleting_snapshots(): # test delete child project -> deletes snapshot # test delete snapshot -> deletes child project From b53a9a5e1a77355dde1a648c7943dc1c607c792f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 19:51:36 +0200 Subject: [PATCH 100/137] implements create snapshot --- .../snapshots_api_handlers.py | 46 +++++++++++--- .../snapshots_core.py | 49 +++++++++++---- .../simcore_service_webserver/snapshots_db.py | 63 ++++++++++++++----- 3 files changed, 123 insertions(+), 35 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index f22759a84d6..c4ce9cdc3bd 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -11,8 +11,11 @@ from ._meta import api_version_prefix as vtag from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from .login.decorators import login_required +from .projects import projects_api +from .projects.projects_exceptions import ProjectNotFoundError from .security_decorators import permission_required -from .snapshots_db import SnapshotsRepository +from .snapshots_core import ProjectDict, take_snapshot +from .snapshots_db import ProjectsRepository, SnapshotsRepository from .snapshots_models import Snapshot, SnapshotItem @@ -54,6 +57,11 @@ async def wrapped(request: web.Request): content_type="application/json", ) from err + except ProjectNotFoundError as err: + raise web.HTTPNotFound( + reason=f"Project not found {err.project_uuid} or not accessible. Skipping snapshot" + ) from err + return wrapped @@ -94,13 +102,9 @@ async def _list_snapshots(project_id: UUID) -> List[Snapshot]: snapshots: List[Snapshot] = await _list_snapshots( project_id=request.match_info["project_id"], # type: ignore ) - # TODO: async for snapshot in await list_snapshot is the same? - data = [] - for snapshot in snapshots: - data.append(SnapshotItem.from_snapshot(snapshot, request.app)) - + data = [SnapshotItem.from_snapshot(snp, request.app) for snp in snapshots] return enveloped_response(data) @@ -143,13 +147,41 @@ async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: @handle_request_errors async def create_project_snapshot_handler(request: web.Request): snapshots_repo = SnapshotsRepository(request) + projects_repo = ProjectsRepository(request) + user_id = request[RQT_USERID_KEY] @validate_arguments async def _create_snapshot( project_id: UUID, snapshot_label: Optional[str] = None, ) -> Snapshot: - raise NotImplementedError + + snapshot_orm = None + if snapshot_label: + snapshot_orm = snapshots_repo.get_by_name(project_id, snapshot_label) + + if not snapshot_orm: + parent: ProjectDict = await projects_api.get_project_for_user( + request.app, + str(project_id), + user_id, + include_templates=False, + include_state=False, + ) + + # pylint: disable=unused-variable + project: ProjectDict + snapshot: Snapshot + project, snapshot = await take_snapshot( + parent, + snapshot_label=snapshot_label, + ) + + # FIXME: Atomic?? project and snapshot shall be created in the same transaction!! + await projects_repo.create(project) + snapshot_orm = await snapshots_repo.create(snapshot.dict()) + + return Snapshot.from_orm(snapshot_orm) snapshot = await _create_snapshot( project_id=request.match_info["project_id"], # type: ignore diff --git a/services/web/server/src/simcore_service_webserver/snapshots_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py index 9a77d8dd7be..5bd450cb6ef 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -8,15 +8,17 @@ # -from typing import Iterator, Tuple +from typing import Any, Dict, Iterator, Optional, Tuple from uuid import UUID, uuid3 from models_library.projects_nodes import Node +from .projects import projects_utils from .projects.projects_db import ProjectAtDB -from .projects.projects_utils import clone_project_document from .snapshots_models import Snapshot +ProjectDict = Dict[str, Any] + def is_parametrized(node: Node) -> bool: try: @@ -35,20 +37,45 @@ def is_parametrized_project(project: ProjectAtDB) -> bool: return any(is_parametrized(node) for node in project.workbench.values()) -def snapshot_project(parent: ProjectAtDB, snapshot_label: str): +async def take_snapshot( + parent: ProjectDict, + snapshot_label: Optional[str] = None, +) -> Tuple[ProjectDict, Snapshot]: + + assert ProjectAtDB.parse_obj(parent) # nosec + + # FIXME: + # if is_parametrized_project(parent): + # raise NotImplementedError( + # "Only non-parametrized projects can be snapshot right now" + # ) - if is_parametrized_project(parent): - raise NotImplementedError( - "Only non-parametrized projects can be snapshot right now" - ) + # Clones parent's project document + snapshot_timestamp = parent["last_change_date"] - project, nodes_map = clone_project_document( - parent.dict(), - forced_copy_project_id=str(uuid3(namespace=parent.uuid, name=snapshot_label)), + child: ProjectDict + child, nodes_map = projects_utils.clone_project_document( + project=parent, + forced_copy_project_id=uuid3( + UUID(parent["uuid"]), f"snapshot.{snapshot_timestamp}" + ), ) + assert child # nosec assert nodes_map # nosec + assert ProjectAtDB.parse_obj(child) # nosec + + child["name"] += snapshot_label or f" [snapshot {snapshot_timestamp}]" + # creation_data = state of parent upon copy! WARNING: changes can be state changes and not project definition? + child["creation_date"] = parent["last_change_date"] + child["hidden"] = True + child["published"] = False snapshot = Snapshot( - id, label=snapshot_label, parent_id=parent.id, project_id=project.id + name=f"Snapshot {snapshot_timestamp} [{parent['name']}]", + created_at=snapshot_timestamp, + parent_uuid=parent["uuid"], + project_uuid=child["uuid"], ) + + return (child, snapshot) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 3903dd37b60..ccb6f2c7f42 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -1,15 +1,26 @@ -from typing import List, Optional +from typing import Dict, List, Optional from uuid import UUID +from aiohttp import web from aiopg.sa.result import RowProxy from pydantic import PositiveInt from simcore_postgres_database.models.snapshots import snapshots +from simcore_service_catalog.services.access_rights import OLD_SERVICES_DATE +from simcore_service_director_v2.modules.dynamic_sidecar.docker_compose_specs import ( + BASE_SERVICE_SPEC, +) +from simcore_service_webserver.snapshots_models import Snapshot from .db_base_repository import BaseRepository +from .projects.projects_db import APP_PROJECT_DBAPI # alias for readability # SEE https://pydantic-docs.helpmanual.io/usage/models/#orm-mode-aka-arbitrary-class-instances SnapshotOrm = RowProxy +SnapshotDict = Dict + +ProjectOrm = RowProxy +ProjectDict = Dict class SnapshotsRepository(BaseRepository): @@ -19,39 +30,57 @@ class SnapshotsRepository(BaseRepository): Gets primitive/standard parameters and returns valid orm objects """ - async def list(self, projec_uuid: UUID) -> List[SnapshotOrm]: - result = [] + async def list( + self, project_uuid: UUID, limit: Optional[int] = None + ) -> List[SnapshotOrm]: + """ Returns sorted list of snapshots in project""" + # TODO: add pagination + async with self.engine.acquire() as conn: - stmt = ( + query = ( snapshots.select() - .where(snapshots.c.parent_uuid == str(projec_uuid)) - .order_by(snapshots.c.child_index) + .where(snapshots.c.parent_uuid == str(project_uuid)) + .order_by(snapshots.c.id) ) - async for row in conn.execute(stmt): - result.append(row) - return result + if limit and limit > 0: + query = query.limit(limit) + + return await (await conn.execute(query)).fetchall() - async def _get(self, stmt) -> Optional[SnapshotOrm]: + async def _first(self, query) -> Optional[SnapshotOrm]: async with self.engine.acquire() as conn: - return await (await conn.execute(stmt)).first() + return await (await conn.execute(query)).first() async def get_by_index( self, project_uuid: UUID, snapshot_index: PositiveInt ) -> Optional[SnapshotOrm]: - stmt = snapshots.select().where( + query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.child_index == snapshot_index) ) - return await self._get(stmt) + return await self._first(query) async def get_by_name( self, project_uuid: UUID, snapshot_name: str ) -> Optional[SnapshotOrm]: - stmt = snapshots.select().where( + query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.name == snapshot_name) ) - return await self._get(stmt) + return await self._first(query) + + async def create(self, snapshot: SnapshotDict) -> SnapshotOrm: + # pylint: disable=no-value-for-parameter + query = snapshots.insert().values(**snapshot).returning(snapshots) + row = await self._first(query) + assert row # nosec + return row + + +class ProjectsRepository(BaseRepository): + def __init__(self, request: web.Request): + super().__init__(request) + self._dbapi = request.config_dict[APP_PROJECT_DBAPI] - async def create(self, project_uuid: UUID) -> SnapshotOrm: - pass + async def create(self, project: ProjectDict): + await self._dbapi.add_project(project, self.user_id, force_project_uuid=True) From 635825397cd0824ee6e15e2a7ebb209bb2f76878 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 11 Aug 2021 21:40:30 +0200 Subject: [PATCH 101/137] auto-doc --- .../doc/img/postgres-database-models.svg | 169 +++++++++--------- 1 file changed, 83 insertions(+), 86 deletions(-) diff --git a/packages/postgres-database/doc/img/postgres-database-models.svg b/packages/postgres-database/doc/img/postgres-database-models.svg index 368427dacb8..38603b2e51a 100644 --- a/packages/postgres-database/doc/img/postgres-database-models.svg +++ b/packages/postgres-database/doc/img/postgres-database-models.svg @@ -361,64 +361,64 @@ comp_runs - -comp_runs - -run_id - [BIGINT] - -project_uuid - [VARCHAR] - -user_id - [BIGINT] - -iteration - [BIGINT] - -result - [VARCHAR(11)] - -created - [DATETIME] - -modified - [DATETIME] - -started - [DATETIME] - -ended - [DATETIME] + +comp_runs + +run_id + [BIGINT] + +project_uuid + [VARCHAR] + +user_id + [BIGINT] + +iteration + [BIGINT] + +result + [VARCHAR(11)] + +created + [DATETIME] + +modified + [DATETIME] + +started + [DATETIME] + +ended + [DATETIME] users--comp_runs - -{0,1} -0..N + +{0,1} +0..N user_to_projects - -user_to_projects - -id - [BIGINT] - -user_id - [BIGINT] - -project_id - [BIGINT] + +user_to_projects + +id + [BIGINT] + +user_id + [BIGINT] + +project_id + [BIGINT] users--user_to_projects - -{0,1} -0..N + +{0,1} +0..N @@ -587,45 +587,42 @@ study_tags - -study_tags - -study_id - [BIGINT] - -tag_id - [BIGINT] + +study_tags + +study_id + [BIGINT] + +tag_id + [BIGINT] projects--study_tags - -{0,1} -0..N + +{0,1} +0..N snapshots - -snapshots - -id - [BIGINT] - -name - [VARCHAR] - -created_at - [DATETIME] - -parent_uuid - [VARCHAR] - -child_index - [INTEGER] - -project_uuid - [VARCHAR] + +snapshots + +id + [BIGINT] + +name + [VARCHAR] + +created_at + [DATETIME] + +parent_uuid + [VARCHAR] + +project_uuid + [VARCHAR] @@ -644,23 +641,23 @@ projects--comp_runs - -{0,1} -0..N + +{0,1} +0..N projects--user_to_projects - -{0,1} -0..N + +{0,1} +0..N tags--study_tags - -{0,1} -0..N + +{0,1} +0..N From 0f741e2dc5eb14294e5d1a547918403f66048335 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:14:02 +0200 Subject: [PATCH 102/137] fixes dependency --- packages/postgres-database/requirements/ci.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/postgres-database/requirements/ci.txt b/packages/postgres-database/requirements/ci.txt index 8fc611bc091..b12bf394c2e 100644 --- a/packages/postgres-database/requirements/ci.txt +++ b/packages/postgres-database/requirements/ci.txt @@ -9,7 +9,6 @@ # installs base + tests requirements --requirement _base.txt --requirement _migration.txt ---requirement _pydantic.txt --requirement _test.txt # installs this repo's packages From 00bf30f1e496e7cf222fad0eb935fdad0fd2cd07 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:31:29 +0200 Subject: [PATCH 103/137] fixes linter on too many statements --- .../src/servicelib/application_setup.py | 28 +++++++++++-------- .../tests/test_application_setup.py | 1 - 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 89377797728..09f651f0d7a 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -32,6 +32,18 @@ class DependencyError(ApplicationSetupError): pass +def _is_app_module_enabled(cfg: Dict, parts: List[str], section) -> bool: + # navigates app_config (cfg) searching for section + for part in parts: + if section and part == "enabled": + # if section exists, no need to explicitly enable it + cfg = cfg.get(part, True) + else: + cfg = cfg[part] + assert isinstance(cfg, bool) # nosec + return cfg + + def app_module_setup( module_name: str, category: ModuleCategory, @@ -57,7 +69,7 @@ def app_module_setup( :param config_enabled: option in config to enable, defaults to None which is '$(module-section).enabled' (config_section and config_enabled are mutually exclusive) :raises DependencyError :raises ApplicationSetupError - :return: False if setup was skipped + :return: True if setup was completed or False if setup was skipped :rtype: bool :Example: @@ -112,18 +124,10 @@ def setup_wrapper(app: web.Application, *args, **kargs) -> bool: # TODO: sometimes section is optional, check in config schema cfg = app[APP_CONFIG_KEY] - def _get(cfg_, parts): - for part in parts: - if ( - section and part == "enabled" - ): # if section exists, no need to explicitly enable it - cfg_ = cfg_.get(part, True) - else: - cfg_ = cfg_[part] - return cfg_ - try: - is_enabled = _get(cfg, config_enabled.split(".")) + is_enabled = _is_app_module_enabled( + cfg, config_enabled.split("."), section + ) except KeyError as ee: raise ApplicationSetupError( f"Cannot find required option '{config_enabled}' in app config's section '{ee}'" diff --git a/packages/service-library/tests/test_application_setup.py b/packages/service-library/tests/test_application_setup.py index a37b6268c17..8ca80c37dc0 100644 --- a/packages/service-library/tests/test_application_setup.py +++ b/packages/service-library/tests/test_application_setup.py @@ -2,7 +2,6 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import logging from typing import Dict from unittest.mock import Mock From 8782760247ea20ebf0259e94adb0fb753692b522 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 09:38:32 +0200 Subject: [PATCH 104/137] fixes wrong imports --- .../web/server/src/simcore_service_webserver/snapshots_db.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index ccb6f2c7f42..c8d206990f2 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -5,11 +5,6 @@ from aiopg.sa.result import RowProxy from pydantic import PositiveInt from simcore_postgres_database.models.snapshots import snapshots -from simcore_service_catalog.services.access_rights import OLD_SERVICES_DATE -from simcore_service_director_v2.modules.dynamic_sidecar.docker_compose_specs import ( - BASE_SERVICE_SPEC, -) -from simcore_service_webserver.snapshots_models import Snapshot from .db_base_repository import BaseRepository from .projects.projects_db import APP_PROJECT_DBAPI From 61656e7c4edcb89f5b6b456c45adea1316b4a0c3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 14:31:21 +0200 Subject: [PATCH 105/137] Fixes setup of snapshot app module --- .../src/servicelib/application_setup.py | 2 +- .../snapshots_api_handlers.py | 55 ++++++++++--------- .../tests/unit/with_dbs/config-devel.yml | 2 + .../server/tests/unit/with_dbs/config.yaml | 2 + 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/packages/service-library/src/servicelib/application_setup.py b/packages/service-library/src/servicelib/application_setup.py index 09f651f0d7a..2b77212222d 100644 --- a/packages/service-library/src/servicelib/application_setup.py +++ b/packages/service-library/src/servicelib/application_setup.py @@ -145,7 +145,7 @@ def setup_wrapper(app: web.Application, *args, **kargs) -> bool: dep for dep in depends if dep not in app[APP_SETUP_KEY] ] if uninitialized: - msg = f"The following '{module_name}'' dependencies are still uninitialized: {uninitialized}" + msg = f"Cannot setup app module '{module_name}' because the following dependencies are still uninitialized: {uninitialized}" log.error(msg) raise DependencyError(msg) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index c4ce9cdc3bd..b7f1ea6994f 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -9,7 +9,7 @@ from pydantic.main import BaseModel from ._meta import api_version_prefix as vtag -from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY +from .constants import RQT_USERID_KEY from .login.decorators import login_required from .projects import projects_api from .projects.projects_exceptions import ProjectNotFoundError @@ -191,29 +191,30 @@ async def _create_snapshot( return enveloped_response(snapshot) -@routes.get( - f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", - name="get_snapshot_parameters_handler", -) -@login_required -@permission_required("project.read") -@handle_request_errors -async def get_project_snapshot_parameters_handler( - request: web.Request, -): - user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - - @validate_arguments - async def get_snapshot_parameters( - project_id: UUID, - snapshot_id: str, - ): - # - return {"x": 4, "y": "yes"} - - params = await get_snapshot_parameters( - project_id=request.match_info["project_id"], # type: ignore - snapshot_id=request.match_info["snapshot_id"], - ) - - return params +# @routes.get( +# f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", +# name="get_snapshot_parameters_handler", +# ) +# @login_required +# @permission_required("project.read") +# @handle_request_errors +# async def get_project_snapshot_parameters_handler( +# request: web.Request, +# ): +# import .constants import RQ_PRODUCT_KEY +# user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + +# @validate_arguments +# async def get_snapshot_parameters( +# project_id: UUID, +# snapshot_id: str, +# ): +# # +# return {"x": 4, "y": "yes"} + +# params = await get_snapshot_parameters( +# project_id=request.match_info["project_id"], # type: ignore +# snapshot_id=request.match_info["snapshot_id"], +# ) + +# return params diff --git a/services/web/server/tests/unit/with_dbs/config-devel.yml b/services/web/server/tests/unit/with_dbs/config-devel.yml index 72140b88a6f..4aaa5c15de9 100644 --- a/services/web/server/tests/unit/with_dbs/config-devel.yml +++ b/services/web/server/tests/unit/with_dbs/config-devel.yml @@ -36,6 +36,8 @@ main: testing: true projects: enabled: false +snapshots: + enabled: false resource_manager: enabled: false garbage_collection_interval_seconds: 30 diff --git a/services/web/server/tests/unit/with_dbs/config.yaml b/services/web/server/tests/unit/with_dbs/config.yaml index c95d0bf5115..64607a2c925 100644 --- a/services/web/server/tests/unit/with_dbs/config.yaml +++ b/services/web/server/tests/unit/with_dbs/config.yaml @@ -50,6 +50,8 @@ rest: version: v0 projects: enabled: False +snapshots: + enabled: False session: # python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())" secret_key: "tjwiMSLe0Xd9dwMlAVQT9pYY9JEnr7rcH05fkUcukVs=" From 050d59ba1b24da7620fc990e9b0a27dc62a8a4b7 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Thu, 12 Aug 2021 15:42:29 +0200 Subject: [PATCH 106/137] getSnapshots added --- .../source/class/osparc/data/model/Study.js | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 89897dc662d..3c1dff5ac79 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -252,7 +252,7 @@ qx.Class.define("osparc.data.model.Study", { return false; }, - hasSnapshots: function() { + getSnapshots: function() { return new Promise((resolve, reject) => { const params = { url: { @@ -261,7 +261,25 @@ qx.Class.define("osparc.data.model.Study", { }; osparc.data.Resources.get("snapshots", params) .then(snapshots => { - resolve(snapshots.length); + console.log(snapshots); + resolve(snapshots); + }) + .catch(() => { + // FIXME + resolve([]); + }); + }); + }, + + hasSnapshots: function() { + return new Promise((resolve, reject) => { + this.getSnapshots() + .then(snapshots => { + resolve(Boolean(snapshots.length)); + }) + .catch(() => { + // FIXME + resolve(true); }); }); }, From 61651f9a9ffad653b817cbc14bb199e34c8c8dfe Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Thu, 12 Aug 2021 15:42:58 +0200 Subject: [PATCH 107/137] show snapshots button if the study has snapshots --- .../client/source/class/osparc/desktop/WorkbenchToolbar.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js index 57c30dc2395..975e2756eac 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js @@ -151,14 +151,15 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { } }, - evalSnapshotsBtn: function() { + evalSnapshotsBtn: async function() { const study = this.getStudy(); if (study) { const allNodes = study.getWorkbench().getNodes(true); const hasIterators = Object.values(allNodes).some(node => node.isIterator()); const isSnapshot = study.isSnapshot(); + const hasSnapshots = await study.hasSnapshots(); const snapshotsBtn = this.getChildControl("snapshots-btn"); - (hasIterators && !isSnapshot) ? snapshotsBtn.show() : snapshotsBtn.exclude(); + (hasSnapshots || (hasIterators && !isSnapshot)) ? snapshotsBtn.show() : snapshotsBtn.exclude(); } }, From 117b4654152bb6e84972ede9a6958a1b526c9f2c Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 18:20:08 +0200 Subject: [PATCH 108/137] fixes field name suffix 'uuid' --- api/specs/webserver/openapi-projects.yaml | 8 +++---- .../api/v0/openapi.yaml | 24 +++++++++---------- .../snapshots_models.py | 24 +++++++++++-------- .../sandbox/projects_openapi_generator.py | 14 +++++------ .../unit/isolated/test_snapshots_models.py | 22 +++++++++++++++++ 5 files changed, 59 insertions(+), 33 deletions(-) diff --git a/api/specs/webserver/openapi-projects.yaml b/api/specs/webserver/openapi-projects.yaml index 260d3965ebf..c6d229c0668 100644 --- a/api/specs/webserver/openapi-projects.yaml +++ b/api/specs/webserver/openapi-projects.yaml @@ -581,8 +581,8 @@ components: title: Snapshot required: - id - - parent_id - - project_id + - parent_uuid + - project_uuid - url - url_parent - url_project @@ -603,12 +603,12 @@ components: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time - parent_id: + parent_uuid: title: Parent Id type: string description: Parent's project uuid format: uuid - project_id: + project_uuid: title: Project Id type: string description: Current project's uuid diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 9aec6d0e910..5b3185954f6 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13290,8 +13290,8 @@ paths: title: Snapshot required: - id - - parent_id - - project_id + - parent_uuid + - project_uuid - url - url_parent - url_project @@ -13310,12 +13310,12 @@ paths: type: string description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time - parent_id: + parent_uuid: title: Parent Id type: string description: Parent's project uuid format: uuid - project_id: + project_uuid: title: Project Id type: string description: Current project's uuid @@ -13372,8 +13372,8 @@ paths: title: Snapshot required: - id - - parent_id - - project_id + - parent_uuid + - project_uuid - url - url_parent - url_project @@ -13392,12 +13392,12 @@ paths: type: string description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time - parent_id: + parent_uuid: title: Parent Id type: string description: Parent's project uuid format: uuid - project_id: + project_uuid: title: Project Id type: string description: Current project's uuid @@ -13455,8 +13455,8 @@ paths: title: Snapshot required: - id - - parent_id - - project_id + - parent_uuid + - project_uuid - url - url_parent - url_project @@ -13475,12 +13475,12 @@ paths: type: string description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time - parent_id: + parent_uuid: title: Parent Id type: string description: Parent's project uuid format: uuid - project_id: + project_uuid: title: Project Id type: string description: Current project's uuid diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index 80a3b4d7464..3390ad489fb 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -28,15 +28,17 @@ class Parameter(BaseModel): class Snapshot(BaseModel): - id: PositiveInt = Field(..., description="Unique snapshot identifier") - label: Optional[str] = Field(None, description="Unique human readable display name") + id: PositiveInt = Field(None, description="Unique snapshot identifier") + label: Optional[str] = Field( + None, description="Unique human readable display name", alias="name" + ) created_at: datetime = Field( default_factory=datetime.utcnow, description="Timestamp of the time snapshot was taken from parent. Notice that parent might change with time", ) - parent_id: UUID = Field(..., description="Parent's project uuid") - project_id: UUID = Field(..., description="Current project's uuid") + parent_uuid: UUID = Field(..., description="Parent's project uuid") + project_uuid: UUID = Field(..., description="Current project's uuid") class Config: orm_mode = True @@ -51,7 +53,7 @@ class ParameterApiModel(Parameter): class SnapshotItem(Snapshot): - """ API model for an array item of snapshots """ + """API model for an array item of snapshots""" url: AnyUrl url_parent: AnyUrl @@ -61,19 +63,21 @@ class SnapshotItem(Snapshot): @classmethod def from_snapshot(cls, snapshot: Snapshot, app: web.Application) -> "SnapshotItem": def url_for(router_name: str, **params): - return app.router[router_name].url_for(**params) + return app.router[router_name].url_for( + **{k: str(v) for k, v in params.items()} + ) return cls( url=url_for( "get_project_snapshot_handler", - project_id=snapshot.project_id, + project_id=snapshot.project_uuid, snapshot_id=snapshot.id, ), - url_parent=url_for("get_project", project_id=snapshot.parent_id), - url_project=url_for("get_project", project_id=snapshot.project_id), + url_parent=url_for("get_project", project_id=snapshot.parent_uuid), + url_project=url_for("get_project", project_id=snapshot.project_uuid), url_parameters=url_for( "get_project_snapshot_parameters_handler", - project_id=snapshot.parent_id, + project_id=snapshot.parent_uuid, snapshot_id=snapshot.id, ), **snapshot.dict(), diff --git a/services/web/server/tests/sandbox/projects_openapi_generator.py b/services/web/server/tests/sandbox/projects_openapi_generator.py index 5c0baf38cf5..c1c998edf9f 100644 --- a/services/web/server/tests/sandbox/projects_openapi_generator.py +++ b/services/web/server/tests/sandbox/projects_openapi_generator.py @@ -83,8 +83,8 @@ class Snapshot(BaseModel): description="Timestamp of the time snapshot was taken from parent. Notice that parent might change with time", ) - parent_id: UUID = Field(..., description="Parent's project uuid") - project_id: UUID = Field(..., description="Current project's uuid") + parent_uuid: UUID = Field(..., description="Parent's project uuid") + project_uuid: UUID = Field(..., description="Current project's uuid") class ParameterApiModel(Parameter): @@ -106,11 +106,11 @@ def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotApiMod project_id=snapshot.project_id, snapshot_id=snapshot.id, ), - url_parent=url_for("get_project", project_id=snapshot.parent_id), + url_parent=url_for("get_project", project_id=snapshot.parent_uuid), url_project=url_for("get_project", project_id=snapshot.project_id), url_parameters=url_for( "get_snapshot_parameters", - project_id=snapshot.parent_id, + project_id=snapshot.parent_uuid, snapshot_id=snapshot.id, ), **snapshot.dict(), @@ -258,7 +258,7 @@ async def create_snapshot( project_id = uuid3(namespace=parent_project.id, name=snapshot_label) project = parent_project.copy(update={"id": project_id}) # THIS IS WRONG - snapshot = Snapshot(id=index, parent_id=pid, project_id=project_id) + snapshot = Snapshot(id=index, parent_uuid=pid, project_id=project_id) _PROJECTS[project_id] = project @@ -270,7 +270,7 @@ async def create_snapshot( return SnapshotApiModel( url=url_for( - "get_snapshot", project_id=snapshot.parent_id, snapshot_id=snapshot.id + "get_snapshot", project_id=snapshot.parent_uuid, snapshot_id=snapshot.id ), **snapshot.dict(), ) @@ -292,7 +292,7 @@ async def get_snapshot( return SnapshotApiModel( url=url_for( - "get_snapshot", project_id=snapshot.parent_id, snapshot_id=snapshot.id + "get_snapshot", project_id=snapshot.parent_uuid, snapshot_id=snapshot.id ), **snapshot.dict(), ) diff --git a/services/web/server/tests/unit/isolated/test_snapshots_models.py b/services/web/server/tests/unit/isolated/test_snapshots_models.py index 87be075f0a5..2e0f99b901d 100644 --- a/services/web/server/tests/unit/isolated/test_snapshots_models.py +++ b/services/web/server/tests/unit/isolated/test_snapshots_models.py @@ -1,7 +1,29 @@ +from uuid import uuid4 + +from faker import Faker +from models_library.utils.database_models_factory import sa_table_to_pydantic_model from simcore_postgres_database.models.snapshots import snapshots +from simcore_service_webserver.snapshots_db import snapshots from simcore_service_webserver.snapshots_models import ( Parameter, ParameterApiModel, Snapshot, SnapshotItem, ) + +SnapshotORM = sa_table_to_pydantic_model(snapshots) + + +def test_snapshot_orm_to_domain_model(faker: Faker): + + snapshot_orm = SnapshotORM( + id=faker.random_int(min=0), + name=faker.name(), + created_at=faker.date_time(), + parent_uuid=faker.uuid4(), + project_uuid=faker.uuid4(), + ) + + snapshot = Snapshot.from_orm(snapshot_orm) + + assert snapshot.dict(by_alias=True) == snapshot_orm.dict() From 46de9b599b21f828fdc7e2f8a31beeb645fd59cf Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 18:20:38 +0200 Subject: [PATCH 109/137] minor --- services/web/server/requirements/_test.in | 14 +++++++------- .../src/simcore_service_webserver/snapshots_db.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/services/web/server/requirements/_test.in b/services/web/server/requirements/_test.in index dc6d78c0de4..69f857c6941 100644 --- a/services/web/server/requirements/_test.in +++ b/services/web/server/requirements/_test.in @@ -8,6 +8,8 @@ --constraint _base.txt # testing + +# fixtures coverage pytest pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 @@ -20,21 +22,19 @@ pytest-mock pytest-runner pytest-sugar websockets - -# fixtures aioresponses alembic click +docker Faker +jsonschema openapi-spec-validator python-dotenv -jsonschema -tenacity -docker redis +tenacity # tools -pylint==2.5.0 # 2.5.3 fails to run in parallel. SEE https://github.com/PyCQA/pylint/releases for updates -coveralls codecov +coveralls ptvsd +pylint==2.5.0 # 2.5.3 fails to run in parallel. SEE https://github.com/PyCQA/pylint/releases for updates diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index c8d206990f2..2166ddc46fe 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -11,10 +11,10 @@ # alias for readability # SEE https://pydantic-docs.helpmanual.io/usage/models/#orm-mode-aka-arbitrary-class-instances -SnapshotOrm = RowProxy +SnapshotRow = RowProxy SnapshotDict = Dict -ProjectOrm = RowProxy +ProjectRow = RowProxy ProjectDict = Dict @@ -27,7 +27,7 @@ class SnapshotsRepository(BaseRepository): async def list( self, project_uuid: UUID, limit: Optional[int] = None - ) -> List[SnapshotOrm]: + ) -> List[SnapshotRow]: """ Returns sorted list of snapshots in project""" # TODO: add pagination @@ -42,13 +42,13 @@ async def list( return await (await conn.execute(query)).fetchall() - async def _first(self, query) -> Optional[SnapshotOrm]: + async def _first(self, query) -> Optional[SnapshotRow]: async with self.engine.acquire() as conn: return await (await conn.execute(query)).first() async def get_by_index( self, project_uuid: UUID, snapshot_index: PositiveInt - ) -> Optional[SnapshotOrm]: + ) -> Optional[SnapshotRow]: query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.child_index == snapshot_index) @@ -57,14 +57,14 @@ async def get_by_index( async def get_by_name( self, project_uuid: UUID, snapshot_name: str - ) -> Optional[SnapshotOrm]: + ) -> Optional[SnapshotRow]: query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) & (snapshots.c.name == snapshot_name) ) return await self._first(query) - async def create(self, snapshot: SnapshotDict) -> SnapshotOrm: + async def create(self, snapshot: SnapshotDict) -> SnapshotRow: # pylint: disable=no-value-for-parameter query = snapshots.insert().values(**snapshot).returning(snapshots) row = await self._first(query) From f7f7d8fe17214cd198b6b2f65a6987593d2041a6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Thu, 12 Aug 2021 18:21:12 +0200 Subject: [PATCH 110/137] fixes on models projection --- .../snapshots_api_handlers.py | 37 ++++++++++++++++--- .../snapshots_core.py | 19 +++++----- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index b7f1ea6994f..937e2dfc59f 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -1,3 +1,4 @@ +import logging from functools import wraps from typing import Any, Callable, List, Optional from uuid import UUID @@ -18,6 +19,8 @@ from .snapshots_db import ProjectsRepository, SnapshotsRepository from .snapshots_models import Snapshot, SnapshotItem +logger = logging.getLogger(__name__) + def _default(obj): if isinstance(obj, BaseModel): @@ -48,16 +51,19 @@ async def wrapped(request: web.Request): except KeyError as err: # NOTE: handles required request.match_info[*] or request.query[*] + logger.debug(err, stack_info=True) raise web.HTTPBadRequest(reason=f"Expected parameter {err}") from err except ValidationError as err: # NOTE: pydantic.validate_arguments parses and validates -> ValidationError + logger.debug(err, stack_info=True) raise web.HTTPUnprocessableEntity( text=json_dumps({"error": err.errors()}), content_type="application/json", ) from err except ProjectNotFoundError as err: + logger.debug(err, stack_info=True) raise web.HTTPNotFound( reason=f"Project not found {err.project_uuid} or not accessible. Skipping snapshot" ) from err @@ -136,7 +142,9 @@ async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: project_id=request.match_info["project_id"], # type: ignore snapshot_id=request.match_info["snapshot_id"], ) - return enveloped_response(snapshot) + + data = SnapshotItem.from_snapshot(snapshot, request.app) + return enveloped_response(data) @routes.post( @@ -156,9 +164,18 @@ async def _create_snapshot( snapshot_label: Optional[str] = None, ) -> Snapshot: + # validate parents! + + # already exists! + # - check parent_uuid + # - check + + # yes: get and return + # no: create and return + snapshot_orm = None if snapshot_label: - snapshot_orm = snapshots_repo.get_by_name(project_id, snapshot_label) + snapshot_orm = await snapshots_repo.get_by_name(project_id, snapshot_label) if not snapshot_orm: parent: ProjectDict = await projects_api.get_project_for_user( @@ -177,9 +194,16 @@ async def _create_snapshot( snapshot_label=snapshot_label, ) - # FIXME: Atomic?? project and snapshot shall be created in the same transaction!! - await projects_repo.create(project) - snapshot_orm = await snapshots_repo.create(snapshot.dict()) + snapshot_orm = await snapshots_repo.search( + **snapshot.dict(include={"created_at", "parent_uuid", "project_uuid"}) + ) + if not snapshot_orm: + # FIXME: Atomic?? project and snapshot shall be created in the same transaction!! + # FIXME: project returned might already exist, then return same snaphot + await projects_repo.create(project) + snapshot_orm = await snapshots_repo.create( + snapshot.dict(by_alias=True, exclude_none=True) + ) return Snapshot.from_orm(snapshot_orm) @@ -188,7 +212,8 @@ async def _create_snapshot( snapshot_label=request.query.get("snapshot_label"), ) - return enveloped_response(snapshot) + data = SnapshotItem.from_snapshot(snapshot, request.app) + return enveloped_response(data) # @routes.get( diff --git a/services/web/server/src/simcore_service_webserver/snapshots_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py index 5bd450cb6ef..d0114d2a881 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -11,10 +11,10 @@ from typing import Any, Dict, Iterator, Optional, Tuple from uuid import UUID, uuid3 +from models_library.projects import Project from models_library.projects_nodes import Node from .projects import projects_utils -from .projects.projects_db import ProjectAtDB from .snapshots_models import Snapshot ProjectDict = Dict[str, Any] @@ -27,13 +27,13 @@ def is_parametrized(node: Node) -> bool: return False -def iter_param_nodes(project: ProjectAtDB) -> Iterator[Tuple[UUID, Node]]: +def iter_param_nodes(project: Project) -> Iterator[Tuple[UUID, Node]]: for node_id, node in project.workbench.items(): if is_parametrized(node): yield UUID(node_id), node -def is_parametrized_project(project: ProjectAtDB) -> bool: +def is_parametrized_project(project: Project) -> bool: return any(is_parametrized(node) for node in project.workbench.values()) @@ -42,7 +42,7 @@ async def take_snapshot( snapshot_label: Optional[str] = None, ) -> Tuple[ProjectDict, Snapshot]: - assert ProjectAtDB.parse_obj(parent) # nosec + assert Project.parse_obj(parent) # nosec # FIXME: # if is_parametrized_project(parent): @@ -51,10 +51,10 @@ async def take_snapshot( # ) # Clones parent's project document - snapshot_timestamp = parent["last_change_date"] + snapshot_timestamp = parent["lastChangeDate"] child: ProjectDict - child, nodes_map = projects_utils.clone_project_document( + child, _ = projects_utils.clone_project_document( project=parent, forced_copy_project_id=uuid3( UUID(parent["uuid"]), f"snapshot.{snapshot_timestamp}" @@ -62,12 +62,11 @@ async def take_snapshot( ) assert child # nosec - assert nodes_map # nosec - assert ProjectAtDB.parse_obj(child) # nosec + assert Project.parse_obj(child) # nosec child["name"] += snapshot_label or f" [snapshot {snapshot_timestamp}]" - # creation_data = state of parent upon copy! WARNING: changes can be state changes and not project definition? - child["creation_date"] = parent["last_change_date"] + # creation_date = state of parent upon copy! WARNING: changes can be state changes and not project definition? + child["creationDate"] = parent["lastChangeDate"] child["hidden"] = True child["published"] = False From eae8728a495ff1c922695c49c5bc3b1d791dfda8 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 13 Aug 2021 07:56:50 +0200 Subject: [PATCH 111/137] minor --- .../osparc/component/snapshots/Snapshots.js | 50 +++++++++++++++++-- .../source/class/osparc/data/model/Study.js | 8 +-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js index 54716f7edd7..f21a409beb7 100644 --- a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -32,7 +32,8 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { this.setColumnWidth(this.self().T_POS.NAME.col, 220); this.setColumnWidth(this.self().T_POS.DATE.col, 130); - this.__populateTable(); + // this.__populateTable(); + this.__populateSnapshotsTable(); }, statics: { @@ -48,6 +49,14 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { DATE: { col: 2, label: qx.locale.Manager.tr("Created At") + }, + PARAMETERS: { + col: 3, + label: qx.locale.Manager.tr("Parameters") + }, + ACTIONS: { + col: 4, + label: "" } } }, @@ -67,6 +76,7 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { this.__cols[colKey] = this.self().T_POS[colKey]; }); + /* // add data-iterators to columns const nextCol = Object.keys(this.__cols).length; const iterators = this.__primaryStudy.getIterators(); @@ -77,6 +87,7 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { label: dataIterator.getLabel() }; } + */ const cols = []; Object.keys(this.__cols).forEach(colKey => { @@ -86,6 +97,13 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { }); model.setColumns(cols); + const columnModel = this.getTableColumnModel(); + const initCols = Object.keys(this.self().T_POS).length; + const totalCols = Object.keys(this.__cols).length; + for (let i=initCols; i { + const rows = []; + for (let i=0; i study.uuid === secondaryStudyId); + if (!secondaryStudy) { + console.error("Secondary study not found", secondaryStudyId); + continue; + } + const row = []; + row[this.self().T_POS.ID.col] = secondaryStudy.uuid; + row[this.self().T_POS.NAME.col] = secondaryStudy.name; + const date = new Date(secondaryStudy.creationDate); + row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); + row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); rows.push(row); } this.getTableModel().setData(rows, false); diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 3c1dff5ac79..e15afbbe5c0 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -275,11 +275,13 @@ qx.Class.define("osparc.data.model.Study", { return new Promise((resolve, reject) => { this.getSnapshots() .then(snapshots => { - resolve(Boolean(snapshots.length)); - }) - .catch(() => { // FIXME + // resolve(Boolean(snapshots.length)); resolve(true); + }) + .catch(err => { + console.error(err); + reject(); }); }); }, From 2d062b9d35ce43b9cfbdf1c79d825a5173fb190f Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 13 Aug 2021 08:13:07 +0200 Subject: [PATCH 112/137] bad merge --- .../api/v0/openapi.yaml | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d95adc65e1c..5b3185954f6 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13290,13 +13290,8 @@ paths: title: Snapshot required: - id -<<<<<<< HEAD - - parent_id - - project_id -======= - parent_uuid - project_uuid ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 - url - url_parent - url_project @@ -13315,20 +13310,12 @@ paths: type: string description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time -<<<<<<< HEAD - parent_id: -======= parent_uuid: ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 title: Parent Id type: string description: Parent's project uuid format: uuid -<<<<<<< HEAD - project_id: -======= project_uuid: ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 title: Project Id type: string description: Current project's uuid @@ -13405,20 +13392,12 @@ paths: type: string description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time -<<<<<<< HEAD - parent_id: -======= parent_uuid: ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 title: Parent Id type: string description: Parent's project uuid format: uuid -<<<<<<< HEAD - project_id: -======= project_uuid: ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 title: Project Id type: string description: Current project's uuid @@ -13476,13 +13455,8 @@ paths: title: Snapshot required: - id -<<<<<<< HEAD - - parent_id - - project_id -======= - parent_uuid - project_uuid ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 - url - url_parent - url_project @@ -13501,20 +13475,12 @@ paths: type: string description: Timestamp of the time snapshot was taken from parent. Notice that parent might change with time format: date-time -<<<<<<< HEAD - parent_id: -======= parent_uuid: ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 title: Parent Id type: string description: Parent's project uuid format: uuid -<<<<<<< HEAD - project_id: -======= project_uuid: ->>>>>>> f7f7d8fe17214cd198b6b2f65a6987593d2041a6 title: Project Id type: string description: Current project's uuid From 557ebe3384db4a1eb9d8fc3e873ab08a09b2c9ba Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 13 Aug 2021 10:32:20 +0200 Subject: [PATCH 113/137] get snapshots --- .../client/source/class/osparc/data/model/Study.js | 12 +++++------- .../web/client/source/class/osparc/store/Store.js | 4 ++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 31a5019fe59..cf81ac7de8f 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -264,9 +264,9 @@ qx.Class.define("osparc.data.model.Study", { console.log(snapshots); resolve(snapshots); }) - .catch(() => { - // FIXME - resolve([]); + .catch(err => { + console.error(err); + reject(err); }); }); }, @@ -275,13 +275,11 @@ qx.Class.define("osparc.data.model.Study", { return new Promise((resolve, reject) => { this.getSnapshots() .then(snapshots => { - // FIXME - // resolve(Boolean(snapshots.length)); - resolve(true); + resolve(Boolean(snapshots.length)); }) .catch(err => { console.error(err); - reject(); + reject(err); }); }); }, diff --git a/services/web/client/source/class/osparc/store/Store.js b/services/web/client/source/class/osparc/store/Store.js index 264b5f05317..a8389c51839 100644 --- a/services/web/client/source/class/osparc/store/Store.js +++ b/services/web/client/source/class/osparc/store/Store.js @@ -62,6 +62,10 @@ qx.Class.define("osparc.store.Store", { check: "Array", init: [] }, + snapshots: { + check: "Array", + init: [] + }, config: { check: "Object", init: {} From 5f6c48738ef7c2c86f417c7cbbc835101f1d2da0 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 13 Aug 2021 11:37:41 +0200 Subject: [PATCH 114/137] [skip ci] label -> snapshot_label --- .../web/client/source/class/osparc/desktop/WorkbenchView.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 0872439f186..43c64c42a7c 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -204,7 +204,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { const win = osparc.ui.window.Window.popUpInWindow(takeSnapshotView, title, 400, 140); takeSnapshotView.addListener("takeSnapshot", () => { const label = takeSnapshotView.getLabel(); - const saveData = takeSnapshotView.getSaveData(); + // const saveData = takeSnapshotView.getSaveData(); const workbenchToolbar = this.__mainPanel.getToolbar(); const takeSnapshotBtn = workbenchToolbar.getChildControl("take-snapshot-btn"); takeSnapshotBtn.setFetching(true); @@ -213,8 +213,8 @@ qx.Class.define("osparc.desktop.WorkbenchView", { "studyId": study.getUuid() }, data: { - "label": label, - "save_data": saveData + // "save_data": saveData + "snapshot_label": label } }; osparc.data.Resources.fetch("snapshots", "takeSnapshot", params) From aabae99d73a6d97e953c22f9c346f359d15f4997 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:45:42 +0200 Subject: [PATCH 115/137] fixes creating snapshots --- .../snapshots_api_handlers.py | 55 ++++++++----------- .../snapshots_core.py | 12 ++-- .../simcore_service_webserver/snapshots_db.py | 24 +++++++- .../snapshots_models.py | 8 ++- 4 files changed, 59 insertions(+), 40 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 937e2dfc59f..98aa42fe548 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime from functools import wraps from typing import Any, Callable, List, Optional from uuid import UUID @@ -51,19 +52,19 @@ async def wrapped(request: web.Request): except KeyError as err: # NOTE: handles required request.match_info[*] or request.query[*] - logger.debug(err, stack_info=True) + logger.debug(err, exc_info=True) raise web.HTTPBadRequest(reason=f"Expected parameter {err}") from err except ValidationError as err: # NOTE: pydantic.validate_arguments parses and validates -> ValidationError - logger.debug(err, stack_info=True) + logger.debug(err, exc_info=True) raise web.HTTPUnprocessableEntity( text=json_dumps({"error": err.errors()}), content_type="application/json", ) from err except ProjectNotFoundError as err: - logger.debug(err, stack_info=True) + logger.debug(err, exc_info=True) raise web.HTTPNotFound( reason=f"Project not found {err.project_uuid} or not accessible. Skipping snapshot" ) from err @@ -164,29 +165,27 @@ async def _create_snapshot( snapshot_label: Optional[str] = None, ) -> Snapshot: - # validate parents! + # fetch parent's project + parent: ProjectDict = await projects_api.get_project_for_user( + request.app, + str(project_id), + user_id, + include_templates=False, + include_state=False, + ) - # already exists! - # - check parent_uuid - # - check + # fetch snapshot if any + parent_uuid: UUID = parent["uuid"] + snapshot_timestamp: datetime = parent["lastChangeDate"] - # yes: get and return - # no: create and return + snapshot_orm = await snapshots_repo.get( + parent_uuid=parent_uuid, created_at=snapshot_timestamp + ) - snapshot_orm = None - if snapshot_label: - snapshot_orm = await snapshots_repo.get_by_name(project_id, snapshot_label) + # FIXME: if exists but different name? if not snapshot_orm: - parent: ProjectDict = await projects_api.get_project_for_user( - request.app, - str(project_id), - user_id, - include_templates=False, - include_state=False, - ) - - # pylint: disable=unused-variable + # take a snapshot of the parent project and commit to db project: ProjectDict snapshot: Snapshot project, snapshot = await take_snapshot( @@ -194,16 +193,10 @@ async def _create_snapshot( snapshot_label=snapshot_label, ) - snapshot_orm = await snapshots_repo.search( - **snapshot.dict(include={"created_at", "parent_uuid", "project_uuid"}) - ) - if not snapshot_orm: - # FIXME: Atomic?? project and snapshot shall be created in the same transaction!! - # FIXME: project returned might already exist, then return same snaphot - await projects_repo.create(project) - snapshot_orm = await snapshots_repo.create( - snapshot.dict(by_alias=True, exclude_none=True) - ) + # FIXME: Atomic?? project and snapshot shall be created in the same transaction!! + # FIXME: project returned might already exist, then return same snaphot + await projects_repo.create(project) + snapshot_orm = await snapshots_repo.create(snapshot) return Snapshot.from_orm(snapshot_orm) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py index d0114d2a881..2eba8d96c21 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -8,6 +8,7 @@ # +from datetime import datetime from typing import Any, Dict, Iterator, Optional, Tuple from uuid import UUID, uuid3 @@ -51,14 +52,15 @@ async def take_snapshot( # ) # Clones parent's project document - snapshot_timestamp = parent["lastChangeDate"] + snapshot_timestamp: datetime = parent["lastChangeDate"] + snapshot_project_uuid: UUID = Snapshot.compose_project_uuid( + parent["uuid"], snapshot_timestamp + ) child: ProjectDict child, _ = projects_utils.clone_project_document( project=parent, - forced_copy_project_id=uuid3( - UUID(parent["uuid"]), f"snapshot.{snapshot_timestamp}" - ), + forced_copy_project_id=snapshot_project_uuid, ) assert child # nosec @@ -66,7 +68,7 @@ async def take_snapshot( child["name"] += snapshot_label or f" [snapshot {snapshot_timestamp}]" # creation_date = state of parent upon copy! WARNING: changes can be state changes and not project definition? - child["creationDate"] = parent["lastChangeDate"] + child["creationDate"] = snapshot_timestamp child["hidden"] = True child["published"] = False diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 2166ddc46fe..175e1c149ee 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Dict, List, Optional from uuid import UUID @@ -5,6 +6,7 @@ from aiopg.sa.result import RowProxy from pydantic import PositiveInt from simcore_postgres_database.models.snapshots import snapshots +from simcore_service_webserver.snapshots_models import Snapshot from .db_base_repository import BaseRepository from .projects.projects_db import APP_PROJECT_DBAPI @@ -28,7 +30,7 @@ class SnapshotsRepository(BaseRepository): async def list( self, project_uuid: UUID, limit: Optional[int] = None ) -> List[SnapshotRow]: - """ Returns sorted list of snapshots in project""" + """Returns sorted list of snapshots in project""" # TODO: add pagination async with self.engine.acquire() as conn: @@ -64,9 +66,25 @@ async def get_by_name( ) return await self._first(query) - async def create(self, snapshot: SnapshotDict) -> SnapshotRow: + async def get( + self, parent_uuid: UUID, created_at: datetime + ) -> Optional[SnapshotRow]: + snapshot_project_uuid: UUID = Snapshot.compose_project_uuid( + parent_uuid, created_at + ) + query = snapshots.select().where( + (snapshots.c.parent_uuid == str(parent_uuid)) + & (snapshots.c.project_uuid == snapshot_project_uuid) + ) + return await self._first(query) + + async def create(self, snapshot: Snapshot) -> SnapshotRow: # pylint: disable=no-value-for-parameter - query = snapshots.insert().values(**snapshot).returning(snapshots) + query = ( + snapshots.insert() + .values(**snapshot.dict(by_alias=True, exclude={"id"})) + .returning(snapshots) + ) row = await self._first(query) assert row # nosec return row diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index 3390ad489fb..22d241560d6 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -1,6 +1,6 @@ from datetime import datetime from typing import Optional, Union -from uuid import UUID +from uuid import UUID, uuid3 from aiohttp import web from models_library.projects_nodes import OutputID @@ -43,6 +43,12 @@ class Snapshot(BaseModel): class Config: orm_mode = True + # TODO: can project_uuid be frozen property?? + + @staticmethod + def compose_project_uuid(parent_uuid: UUID, snapshot_timestamp: datetime): + return uuid3(parent_uuid, f"snapshot.{snapshot_timestamp}") + ## API models ---------- From 6b4838954f972c2846e9908ec21d47d096b602d8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 13 Aug 2021 11:56:34 +0200 Subject: [PATCH 116/137] minor --- packages/postgres-database/tests/test_snapshots.py | 4 ++-- .../server/src/simcore_service_webserver/snapshots_core.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py index 95d943c3557..594ca3ec0f2 100644 --- a/packages/postgres-database/tests/test_snapshots.py +++ b/packages/postgres-database/tests/test_snapshots.py @@ -38,7 +38,7 @@ async def engine(pg_engine: Engine): @pytest.fixture -def exclude(): +def exclude() -> Set: return { "id", "uuid", @@ -50,7 +50,7 @@ def exclude(): @pytest.fixture -def create_snapshot(exclude) -> Callable: +def create_snapshot(exclude: Set) -> Callable: async def _create_snapshot(child_index: int, parent_prj, conn) -> int: # NOTE: used as FAKE prototype diff --git a/services/web/server/src/simcore_service_webserver/snapshots_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py index 2eba8d96c21..0f432595c7b 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Dict, Iterator, Optional, Tuple -from uuid import UUID, uuid3 +from uuid import UUID from models_library.projects import Project from models_library.projects_nodes import Node From 59f69234355bd847b7435e603007ec4d4993afa9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:26:05 +0200 Subject: [PATCH 117/137] fixing url_for --- .../snapshots_api_handlers.py | 93 +++++++++++-------- .../simcore_service_webserver/snapshots_db.py | 26 ++++-- .../snapshots_models.py | 16 ++-- 3 files changed, 80 insertions(+), 55 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index 98aa42fe548..e833f7e9a5a 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -9,9 +9,10 @@ from pydantic.decorator import validate_arguments from pydantic.error_wrappers import ValidationError from pydantic.main import BaseModel +from yarl import URL from ._meta import api_version_prefix as vtag -from .constants import RQT_USERID_KEY +from .constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from .login.decorators import login_required from .projects import projects_api from .projects.projects_exceptions import ProjectNotFoundError @@ -72,6 +73,22 @@ async def wrapped(request: web.Request): return wrapped +def create_url_for(request: web.Request): + app = request.app + + def url_for(router_name: str, **params) -> Optional[str]: + try: + rel_url: URL = app.router[router_name].url_for( + **{k: str(v) for k, v in params.items()} + ) + url = request.url.origin().with_path(str(rel_url)) + return str(url) + except KeyError: + return None + + return url_for + + # FIXME: access rights using same approach as in access_layer.py in storage. # A user can only check snapshots (subresource) of its project (parent resource) @@ -92,6 +109,7 @@ async def list_project_snapshots_handler(request: web.Request): Lists references on project snapshots """ snapshots_repo = SnapshotsRepository(request) + url_for = create_url_for(request) @validate_arguments async def _list_snapshots(project_id: UUID) -> List[Snapshot]: @@ -111,7 +129,7 @@ async def _list_snapshots(project_id: UUID) -> List[Snapshot]: ) # TODO: async for snapshot in await list_snapshot is the same? - data = [SnapshotItem.from_snapshot(snp, request.app) for snp in snapshots] + data = [SnapshotItem.from_snapshot(snp, url_for) for snp in snapshots] return enveloped_response(data) @@ -124,18 +142,16 @@ async def _list_snapshots(project_id: UUID) -> List[Snapshot]: @handle_request_errors async def get_project_snapshot_handler(request: web.Request): snapshots_repo = SnapshotsRepository(request) + url_for = create_url_for(request) @validate_arguments async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: - try: - snapshot_orm = await snapshots_repo.get_by_index( - project_id, int(snapshot_id) - ) - except ValueError: - snapshot_orm = await snapshots_repo.get_by_name(project_id, snapshot_id) + snapshot_orm = await snapshots_repo.get_by_id(project_id, int(snapshot_id)) if not snapshot_orm: - raise web.HTTPNotFound(reason=f"snapshot {snapshot_id} not found") + raise web.HTTPNotFound( + reason=f"snapshot {snapshot_id} for project {project_id} not found" + ) return Snapshot.from_orm(snapshot_orm) @@ -144,7 +160,7 @@ async def _get_snapshot(project_id: UUID, snapshot_id: str) -> Snapshot: snapshot_id=request.match_info["snapshot_id"], ) - data = SnapshotItem.from_snapshot(snapshot, request.app) + data = SnapshotItem.from_snapshot(snapshot, url_for) return enveloped_response(data) @@ -158,6 +174,7 @@ async def create_project_snapshot_handler(request: web.Request): snapshots_repo = SnapshotsRepository(request) projects_repo = ProjectsRepository(request) user_id = request[RQT_USERID_KEY] + url_for = create_url_for(request) @validate_arguments async def _create_snapshot( @@ -205,34 +222,34 @@ async def _create_snapshot( snapshot_label=request.query.get("snapshot_label"), ) - data = SnapshotItem.from_snapshot(snapshot, request.app) + data = SnapshotItem.from_snapshot(snapshot, url_for) return enveloped_response(data) -# @routes.get( -# f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", -# name="get_snapshot_parameters_handler", -# ) -# @login_required -# @permission_required("project.read") -# @handle_request_errors -# async def get_project_snapshot_parameters_handler( -# request: web.Request, -# ): -# import .constants import RQ_PRODUCT_KEY -# user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] - -# @validate_arguments -# async def get_snapshot_parameters( -# project_id: UUID, -# snapshot_id: str, -# ): -# # -# return {"x": 4, "y": "yes"} - -# params = await get_snapshot_parameters( -# project_id=request.match_info["project_id"], # type: ignore -# snapshot_id=request.match_info["snapshot_id"], -# ) - -# return params +@routes.get( + f"/{vtag}/projects/{{project_id}}/snapshots/{{snapshot_id}}/parameters", + name="get_snapshot_parameters_handler", +) +@login_required +@permission_required("project.read") +@handle_request_errors +async def get_project_snapshot_parameters_handler( + request: web.Request, +): + # pylint: disable=unused-variable + # pylint: disable=unused-argument + user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] + + @validate_arguments + async def get_snapshot_parameters( + project_id: UUID, + snapshot_id: str, + ): + return {"x": 4, "y": "yes"} + + params = await get_snapshot_parameters( + project_id=request.match_info["project_id"], # type: ignore + snapshot_id=request.match_info["snapshot_id"], + ) + + return params diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 175e1c149ee..53e8624634c 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -1,7 +1,8 @@ from datetime import datetime -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from uuid import UUID +import sqlalchemy as sa from aiohttp import web from aiopg.sa.result import RowProxy from pydantic import PositiveInt @@ -48,21 +49,21 @@ async def _first(self, query) -> Optional[SnapshotRow]: async with self.engine.acquire() as conn: return await (await conn.execute(query)).first() - async def get_by_index( - self, project_uuid: UUID, snapshot_index: PositiveInt + async def get_by_name( + self, project_uuid: UUID, snapshot_name: str ) -> Optional[SnapshotRow]: query = snapshots.select().where( (snapshots.c.parent_uuid == str(project_uuid)) - & (snapshots.c.child_index == snapshot_index) + & (snapshots.c.name == snapshot_name) ) return await self._first(query) - async def get_by_name( - self, project_uuid: UUID, snapshot_name: str + async def get_by_id( + self, parent_uuid: UUID, snapshot_id: int ) -> Optional[SnapshotRow]: query = snapshots.select().where( - (snapshots.c.parent_uuid == str(project_uuid)) - & (snapshots.c.name == snapshot_name) + (snapshots.c.parent_uuid == str(parent_uuid)) + & (snapshots.c.id == snapshot_id) ) return await self._first(query) @@ -78,6 +79,15 @@ async def get( ) return await self._first(query) + async def list_snapshot_names(self, parent_uuid: UUID) -> List[Tuple[str, int]]: + query = ( + sa.select([snapshots.c.name, snapshots.c.id]) + .where(snapshots.c.parent_uuid == str(parent_uuid)) + .order_by(snapshots.c.id) + ) + async with self.engine.acquire() as conn: + return await (await conn.execute(query)).fetchall() + async def create(self, snapshot: Snapshot) -> SnapshotRow: # pylint: disable=no-value-for-parameter query = ( diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index 22d241560d6..d3cbbbec04b 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Optional, Union +from typing import Callable, Optional, Union from uuid import UUID, uuid3 from aiohttp import web @@ -13,6 +13,7 @@ StrictFloat, StrictInt, ) +from yarl import URL BuiltinTypes = Union[StrictBool, StrictInt, StrictFloat, str] @@ -67,12 +68,9 @@ class SnapshotItem(Snapshot): url_parameters: Optional[AnyUrl] = None @classmethod - def from_snapshot(cls, snapshot: Snapshot, app: web.Application) -> "SnapshotItem": - def url_for(router_name: str, **params): - return app.router[router_name].url_for( - **{k: str(v) for k, v in params.items()} - ) - + def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotItem": + # TODO: is this the right place? requires pre-defined routes + # how to guarantee routes names return cls( url=url_for( "get_project_snapshot_handler", @@ -82,9 +80,9 @@ def url_for(router_name: str, **params): url_parent=url_for("get_project", project_id=snapshot.parent_uuid), url_project=url_for("get_project", project_id=snapshot.project_uuid), url_parameters=url_for( - "get_project_snapshot_parameters_handler", + "get_snapshot_parameters_handler", project_id=snapshot.parent_uuid, snapshot_id=snapshot.id, ), - **snapshot.dict(), + **snapshot.dict(by_alias=True), ) From a7fad980c90f3de20a907968806498f7ba9d9e54 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:48:53 +0200 Subject: [PATCH 118/137] fixes from manual exploratory testing --- .../snapshots_api_handlers.py | 2 +- .../src/simcore_service_webserver/snapshots_core.py | 2 +- .../src/simcore_service_webserver/snapshots_db.py | 3 +-- .../src/simcore_service_webserver/snapshots_models.py | 11 ++++++++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py index e833f7e9a5a..24622b1b8a4 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_api_handlers.py @@ -192,7 +192,7 @@ async def _create_snapshot( ) # fetch snapshot if any - parent_uuid: UUID = parent["uuid"] + parent_uuid: UUID = UUID(parent["uuid"]) snapshot_timestamp: datetime = parent["lastChangeDate"] snapshot_orm = await snapshots_repo.get( diff --git a/services/web/server/src/simcore_service_webserver/snapshots_core.py b/services/web/server/src/simcore_service_webserver/snapshots_core.py index 0f432595c7b..e2a8d636e45 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_core.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_core.py @@ -73,7 +73,7 @@ async def take_snapshot( child["published"] = False snapshot = Snapshot( - name=f"Snapshot {snapshot_timestamp} [{parent['name']}]", + name=snapshot_label or f"Snapshot {snapshot_timestamp} [{parent['name']}]", created_at=snapshot_timestamp, parent_uuid=parent["uuid"], project_uuid=child["uuid"], diff --git a/services/web/server/src/simcore_service_webserver/snapshots_db.py b/services/web/server/src/simcore_service_webserver/snapshots_db.py index 53e8624634c..5f7ef8b7afe 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_db.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_db.py @@ -5,7 +5,6 @@ import sqlalchemy as sa from aiohttp import web from aiopg.sa.result import RowProxy -from pydantic import PositiveInt from simcore_postgres_database.models.snapshots import snapshots from simcore_service_webserver.snapshots_models import Snapshot @@ -75,7 +74,7 @@ async def get( ) query = snapshots.select().where( (snapshots.c.parent_uuid == str(parent_uuid)) - & (snapshots.c.project_uuid == snapshot_project_uuid) + & (snapshots.c.project_uuid == str(snapshot_project_uuid)) ) return await self._first(query) diff --git a/services/web/server/src/simcore_service_webserver/snapshots_models.py b/services/web/server/src/simcore_service_webserver/snapshots_models.py index d3cbbbec04b..01021539adf 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_models.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_models.py @@ -2,7 +2,6 @@ from typing import Callable, Optional, Union from uuid import UUID, uuid3 -from aiohttp import web from models_library.projects_nodes import OutputID from pydantic import ( AnyUrl, @@ -47,7 +46,11 @@ class Config: # TODO: can project_uuid be frozen property?? @staticmethod - def compose_project_uuid(parent_uuid: UUID, snapshot_timestamp: datetime): + def compose_project_uuid( + parent_uuid: Union[UUID, str], snapshot_timestamp: datetime + ) -> UUID: + if isinstance(parent_uuid, str): + parent_uuid = UUID(parent_uuid) return uuid3(parent_uuid, f"snapshot.{snapshot_timestamp}") @@ -68,7 +71,9 @@ class SnapshotItem(Snapshot): url_parameters: Optional[AnyUrl] = None @classmethod - def from_snapshot(cls, snapshot: Snapshot, url_for: Callable) -> "SnapshotItem": + def from_snapshot( + cls, snapshot: Snapshot, url_for: Callable[..., URL] + ) -> "SnapshotItem": # TODO: is this the right place? requires pre-defined routes # how to guarantee routes names return cls( From 8146419a013a56066aa358cb0895a2c3c947a4b2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:49:57 +0200 Subject: [PATCH 119/137] wip tests --- .../unit/isolated/test_snapshots_models.py | 18 +++++++++++------- .../with_dbs/11/test_snapshots_api_handlers.py | 0 .../unit/with_dbs/11/test_snapshots_core.py | 2 -- .../unit/with_dbs/11/test_snapshots_db.py | 0 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/11/test_snapshots_api_handlers.py create mode 100644 services/web/server/tests/unit/with_dbs/11/test_snapshots_db.py diff --git a/services/web/server/tests/unit/isolated/test_snapshots_models.py b/services/web/server/tests/unit/isolated/test_snapshots_models.py index 2e0f99b901d..c8e3d182461 100644 --- a/services/web/server/tests/unit/isolated/test_snapshots_models.py +++ b/services/web/server/tests/unit/isolated/test_snapshots_models.py @@ -1,15 +1,10 @@ +from datetime import datetime from uuid import uuid4 from faker import Faker from models_library.utils.database_models_factory import sa_table_to_pydantic_model from simcore_postgres_database.models.snapshots import snapshots -from simcore_service_webserver.snapshots_db import snapshots -from simcore_service_webserver.snapshots_models import ( - Parameter, - ParameterApiModel, - Snapshot, - SnapshotItem, -) +from simcore_service_webserver.snapshots_models import Snapshot SnapshotORM = sa_table_to_pydantic_model(snapshots) @@ -27,3 +22,12 @@ def test_snapshot_orm_to_domain_model(faker: Faker): snapshot = Snapshot.from_orm(snapshot_orm) assert snapshot.dict(by_alias=True) == snapshot_orm.dict() + + +def test_compose_project_uuid(): + + prj_id1 = Snapshot.compose_project_uuid(uuid4(), datetime.now()) + assert prj_id1 + + prj_id2 = Snapshot.compose_project_uuid(str(uuid4()), datetime.now()) + assert prj_id2 diff --git a/services/web/server/tests/unit/with_dbs/11/test_snapshots_api_handlers.py b/services/web/server/tests/unit/with_dbs/11/test_snapshots_api_handlers.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py b/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py index d4f14e54043..ed2cbc39d33 100644 --- a/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py +++ b/services/web/server/tests/unit/with_dbs/11/test_snapshots_core.py @@ -10,11 +10,9 @@ from models_library.projects import Project # , ProjectAtDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID -from simcore_service_webserver.constants import APP_PROJECT_DBAPI from simcore_service_webserver.projects.projects_api import get_project_for_user from simcore_service_webserver.projects.projects_db import APP_PROJECT_DBAPI from simcore_service_webserver.projects.projects_utils import clone_project_document -from simcore_service_webserver.snapshots_core import snapshot_project # is parametrized project? diff --git a/services/web/server/tests/unit/with_dbs/11/test_snapshots_db.py b/services/web/server/tests/unit/with_dbs/11/test_snapshots_db.py new file mode 100644 index 00000000000..e69de29bb2d From d90659e2f82311caea807cb14e62c8910485d497 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 13 Aug 2021 16:02:26 +0200 Subject: [PATCH 120/137] snapshot with label --- .../class/osparc/component/snapshots/TakeSnapshotView.js | 3 --- services/web/client/source/class/osparc/data/Resources.js | 2 +- .../web/client/source/class/osparc/desktop/WorkbenchView.js | 6 +----- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js b/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js index c4176439535..4c91fb8669f 100644 --- a/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js +++ b/services/web/client/source/class/osparc/component/snapshots/TakeSnapshotView.js @@ -95,9 +95,6 @@ qx.Class.define("osparc.component.snapshots.TakeSnapshotView", { form.add(label, "Label", null, "label"); label.setValue(study.getName()); - const saveWData = this.getChildControl("save-data"); - form.add(saveWData, "Save with Data", null, "save-data"); - // buttons const cancelButton = this.getChildControl("cancel-button"); form.addButton(cancelButton); diff --git a/services/web/client/source/class/osparc/data/Resources.js b/services/web/client/source/class/osparc/data/Resources.js index 42ffc1a79e1..217e003c07c 100644 --- a/services/web/client/source/class/osparc/data/Resources.js +++ b/services/web/client/source/class/osparc/data/Resources.js @@ -177,7 +177,7 @@ qx.Class.define("osparc.data.Resources", { }, takeSnapshot: { method: "POST", - url: statics.API + "/projects/{studyId}/snapshots" + url: statics.API + "/projects/{studyId}/snapshots?snapshot_label={snapshot_label}" } } }, diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 43c64c42a7c..197dba55d5e 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -204,16 +204,12 @@ qx.Class.define("osparc.desktop.WorkbenchView", { const win = osparc.ui.window.Window.popUpInWindow(takeSnapshotView, title, 400, 140); takeSnapshotView.addListener("takeSnapshot", () => { const label = takeSnapshotView.getLabel(); - // const saveData = takeSnapshotView.getSaveData(); const workbenchToolbar = this.__mainPanel.getToolbar(); const takeSnapshotBtn = workbenchToolbar.getChildControl("take-snapshot-btn"); takeSnapshotBtn.setFetching(true); const params = { url: { - "studyId": study.getUuid() - }, - data: { - // "save_data": saveData + "studyId": study.getUuid(), "snapshot_label": label } }; From cab75c488476a89f9aefaa2cf143bfbb5dcbc5d8 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 13 Aug 2021 16:24:23 +0200 Subject: [PATCH 121/137] [skip ci] populateSnapshotsTable --- .../osparc/component/snapshots/Snapshots.js | 63 ++++++------------- .../source/class/osparc/data/model/Study.js | 53 +++++++++------- 2 files changed, 50 insertions(+), 66 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js index f21a409beb7..40cb8177699 100644 --- a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -76,19 +76,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { this.__cols[colKey] = this.self().T_POS[colKey]; }); - /* - // add data-iterators to columns - const nextCol = Object.keys(this.__cols).length; - const iterators = this.__primaryStudy.getIterators(); - for (let i=0; i { const idx = this.__cols[colKey].col; @@ -97,13 +84,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { }); model.setColumns(cols); - const columnModel = this.getTableColumnModel(); - const initCols = Object.keys(this.self().T_POS).length; - const totalCols = Object.keys(this.__cols).length; - for (let i=initCols; i { - const rows = []; - for (let i=0; i study.uuid === secondaryStudyId); - if (!secondaryStudy) { - console.error("Secondary study not found", secondaryStudyId); - continue; - } - const row = []; - row[this.self().T_POS.ID.col] = secondaryStudy.uuid; - row[this.self().T_POS.NAME.col] = secondaryStudy.name; - const date = new Date(secondaryStudy.creationDate); - row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); - row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); - rows.push(row); - } - this.getTableModel().setData(rows, false); - }); + const columnModel = this.getTableColumnModel(); + const initCols = Object.keys(this.self().T_POS).length; + const totalCols = Object.keys(this.__cols).length; + for (let i=initCols; i { + const rows = []; + snapshots.reverse().forEach(snapshot => { + const row = []; + row[this.self().T_POS.ID.col] = snapshot["7aed0b07-99a9-3552-9126-ed05b508e9f5"]; + row[this.self().T_POS.NAME.col] = snapshot["label"]; + const date = new Date(snapshot["created_at"]); + row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); + // row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); + rows.push(row); + }); + this.getTableModel().setData(rows, false); + }); } } }); diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index cf81ac7de8f..7d5bc1154d5 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -232,36 +232,17 @@ qx.Class.define("osparc.data.model.Study", { return true; } return false; - } - }, - - members: { - buildWorkbench: function() { - this.getWorkbench().buildWorkbench(); - }, - - initStudy: function() { - this.getWorkbench().initWorkbench(); - }, - - isSnapshot: function() { - if (this.getSweeper()) { - const primaryStudyId = this.getSweeper().getPrimaryStudyId(); - return primaryStudyId !== null; - } - return false; }, - getSnapshots: function() { + getSnapshots: function(studyId) { return new Promise((resolve, reject) => { const params = { url: { - "studyId": this.getUuid() + "studyId": studyId } }; osparc.data.Resources.get("snapshots", params) .then(snapshots => { - console.log(snapshots); resolve(snapshots); }) .catch(err => { @@ -271,9 +252,9 @@ qx.Class.define("osparc.data.model.Study", { }); }, - hasSnapshots: function() { + hasSnapshots: function(studyId) { return new Promise((resolve, reject) => { - this.getSnapshots() + this.self().getSnapshots(studyId) .then(snapshots => { resolve(Boolean(snapshots.length)); }) @@ -282,6 +263,32 @@ qx.Class.define("osparc.data.model.Study", { reject(err); }); }); + } + }, + + members: { + buildWorkbench: function() { + this.getWorkbench().buildWorkbench(); + }, + + initStudy: function() { + this.getWorkbench().initWorkbench(); + }, + + isSnapshot: function() { + if (this.getSweeper()) { + const primaryStudyId = this.getSweeper().getPrimaryStudyId(); + return primaryStudyId !== null; + } + return false; + }, + + getSnapshots: function() { + return this.self().getSnapshots(this.getUuid()); + }, + + hasSnapshots: function() { + return this.self().hasSnapshots(this.getUuid()); }, __applyAccessRights: function(value) { From e92f416a34322d565f1889158965eb2650673f0d Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 16 Aug 2021 12:39:33 +0200 Subject: [PATCH 122/137] clean up --- .../osparc/component/snapshots/Snapshots.js | 69 ++++--------------- 1 file changed, 12 insertions(+), 57 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js index 40cb8177699..9753bd09d00 100644 --- a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -21,7 +21,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { construct: function(primaryStudy) { this.__primaryStudy = primaryStudy; - this.__cols = {}; const model = this.__initModel(); this.base(arguments, model, { @@ -32,7 +31,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { this.setColumnWidth(this.self().T_POS.NAME.col, 220); this.setColumnWidth(this.self().T_POS.DATE.col, 130); - // this.__populateTable(); this.__populateSnapshotsTable(); }, @@ -50,20 +48,15 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { col: 2, label: qx.locale.Manager.tr("Created At") }, - PARAMETERS: { - col: 3, - label: qx.locale.Manager.tr("Parameters") - }, ACTIONS: { - col: 4, - label: "" + col: 3, + label: qx.locale.Manager.tr("Actions") } } }, members: { __primaryStudy: null, - __cols: null, getRowData: function(rowIdx) { return this.getTableModel().getRowDataAsMap(rowIdx); @@ -72,14 +65,10 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { __initModel: function() { const model = new qx.ui.table.model.Simple(); - Object.keys(this.self().T_POS).forEach(colKey => { - this.__cols[colKey] = this.self().T_POS[colKey]; - }); - const cols = []; - Object.keys(this.__cols).forEach(colKey => { - const idx = this.__cols[colKey].col; - const label = this.__cols[colKey].label; + Object.keys(this.self().T_POS).forEach(colKey => { + const idx = this.self().T_POS[colKey].col; + const label = this.self().T_POS[colKey].label; cols.splice(idx, 0, label); }); model.setColumns(cols); @@ -87,58 +76,24 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { return model; }, - __populateTable: function() { - const columnModel = this.getTableColumnModel(); - const initCols = Object.keys(this.self().T_POS).length; - const totalCols = Object.keys(this.__cols).length; - for (let i=initCols; i { - const rows = []; - for (let i=0; i study.uuid === secondaryStudyId); - if (!secondaryStudy) { - console.error("Secondary study not found", secondaryStudyId); - continue; - } - const row = []; - row[this.self().T_POS.ID.col] = secondaryStudy.uuid; - row[this.self().T_POS.NAME.col] = secondaryStudy.name; - const date = new Date(secondaryStudy.creationDate); - row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); - row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); - rows.push(row); - } - this.getTableModel().setData(rows, false); - }); - } - }, - __populateSnapshotsTable: function() { const columnModel = this.getTableColumnModel(); - const initCols = Object.keys(this.self().T_POS).length; - const totalCols = Object.keys(this.__cols).length; - for (let i=initCols; i { const rows = []; snapshots.reverse().forEach(snapshot => { const row = []; - row[this.self().T_POS.ID.col] = snapshot["7aed0b07-99a9-3552-9126-ed05b508e9f5"]; + row[this.self().T_POS.ID.col] = snapshot["project_uuid"]; row[this.self().T_POS.NAME.col] = snapshot["label"]; const date = new Date(snapshot["created_at"]); row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); - // row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); + row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); rows.push(row); }); this.getTableModel().setData(rows, false); From 6ada25244454e6505be7fa8ef440f87fe5f99ba0 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 16 Aug 2021 17:39:44 +0200 Subject: [PATCH 123/137] minor --- .../source/class/osparc/component/snapshots/Snapshots.js | 6 ------ services/web/client/source/class/osparc/data/Resources.js | 1 - 2 files changed, 7 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js index 9753bd09d00..df72d5431d5 100644 --- a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -47,10 +47,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { DATE: { col: 2, label: qx.locale.Manager.tr("Created At") - }, - ACTIONS: { - col: 3, - label: qx.locale.Manager.tr("Actions") } } }, @@ -82,7 +78,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { columnModel.setDataCellRenderer(this.self().T_POS.NAME.col, new qx.ui.table.cellrenderer.String()); columnModel.setDataCellRenderer(this.self().T_POS.DATE.col, new qx.ui.table.cellrenderer.Date()); columnModel.setDataCellRenderer(this.self().T_POS.ID.col, new qx.ui.table.cellrenderer.String()); - columnModel.setDataCellRenderer(this.self().T_POS.ACTIONS.col, new qx.ui.table.cellrenderer.Default()); osparc.data.model.Study.getSnapshots() .then(snapshots => { @@ -93,7 +88,6 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { row[this.self().T_POS.NAME.col] = snapshot["label"]; const date = new Date(snapshot["created_at"]); row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); - row[this.self().T_POS.ACTIONS.col] = new qx.ui.form.Button(); rows.push(row); }); this.getTableModel().setData(rows, false); diff --git a/services/web/client/source/class/osparc/data/Resources.js b/services/web/client/source/class/osparc/data/Resources.js index 217e003c07c..cc2f68ba5fb 100644 --- a/services/web/client/source/class/osparc/data/Resources.js +++ b/services/web/client/source/class/osparc/data/Resources.js @@ -154,7 +154,6 @@ qx.Class.define("osparc.data.Resources", { * SNAPSHOTS */ "snapshots": { - useCache: true, idField: "uuid", endpoints: { get: { From 78e0b3e426792f9fed8e8e661d82c62676f29ee6 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Wed, 18 Aug 2021 14:53:00 +0200 Subject: [PATCH 124/137] [skip ci] remove delete and recreate snapshots --- .../osparc/component/snapshots/Snapshots.js | 2 +- .../component/snapshots/SnapshotsView.js | 66 ------------------- 2 files changed, 1 insertion(+), 67 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js index df72d5431d5..59cb41af9ad 100644 --- a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -79,7 +79,7 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { columnModel.setDataCellRenderer(this.self().T_POS.DATE.col, new qx.ui.table.cellrenderer.Date()); columnModel.setDataCellRenderer(this.self().T_POS.ID.col, new qx.ui.table.cellrenderer.String()); - osparc.data.model.Study.getSnapshots() + this.__primaryStudy.getSnapshots() .then(snapshots => { const rows = []; snapshots.reverse().forEach(snapshot => { diff --git a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js index 6d5bead62e8..4ba1164e056 100644 --- a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js +++ b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js @@ -57,13 +57,6 @@ qx.Class.define("osparc.component.snapshots.SnapshotsView", { layout: new qx.ui.layout.VBox(5) }); - const snapshotBtns = new qx.ui.container.Composite(new qx.ui.layout.HBox(10)); - const deleteSnapshotsBtn = this.__deleteSnapshotsBtn(); - snapshotBtns.add(deleteSnapshotsBtn); - const recreateSnapshotsBtn = this.__recreateSnapshotsBtn(); - snapshotBtns.add(recreateSnapshotsBtn); - snapshotsSection.addAt(snapshotBtns, 0); - this.__rebuildSnapshotsTable(); const openSnapshotBtn = this.__openSnapshotBtn = this.__createOpenSnapshotBtn(); @@ -99,65 +92,6 @@ qx.Class.define("osparc.component.snapshots.SnapshotsView", { return snapshotsTable; }, - __deleteSnapshotsBtn: function() { - const deleteSnapshotsBtn = new osparc.ui.form.FetchButton(this.tr("Delete Snapshots")).set({ - alignX: "left", - allowGrowX: false - }); - deleteSnapshotsBtn.addListener("execute", () => { - deleteSnapshotsBtn.setFetching(true); - this.__deleteSnapshots(deleteSnapshotsBtn) - .then(() => { - this.__rebuildSnapshotsTable(); - }) - .finally(() => { - deleteSnapshotsBtn.setFetching(false); - }); - }, this); - return deleteSnapshotsBtn; - }, - - __recreateSnapshotsBtn: function() { - const recreateSnapshotsBtn = new osparc.ui.form.FetchButton(this.tr("Recreate Snapshots")).set({ - alignX: "right", - allowGrowX: false - }); - recreateSnapshotsBtn.addListener("execute", () => { - recreateSnapshotsBtn.setFetching(true); - this.__recreateSnapshots(recreateSnapshotsBtn) - .then(() => { - this.__rebuildSnapshotsTable(); - }) - .finally(() => { - recreateSnapshotsBtn.setFetching(false); - }); - }, this); - return recreateSnapshotsBtn; - }, - - __deleteSnapshots: function() { - return new Promise((resolve, reject) => { - this.__primaryStudy.getSweeper().removeSecondaryStudies() - .then(() => { - const msg = this.tr("Snapshots Deleted"); - osparc.component.message.FlashMessenger.getInstance().logAs(msg); - resolve(); - }); - }); - }, - - __recreateSnapshots: function() { - return new Promise((resolve, reject) => { - const primaryStudyData = this.__primaryStudy.serialize(); - this.__primaryStudy.getSweeper().recreateSnapshots(primaryStudyData, this.__primaryStudy.getParameters()) - .then(secondaryStudyIds => { - const msg = secondaryStudyIds.length + this.tr(" Snapshots Created"); - osparc.component.message.FlashMessenger.getInstance().logAs(msg); - resolve(); - }); - }); - }, - __createOpenSnapshotBtn: function() { const openSnapshotBtn = new qx.ui.form.Button(this.tr("Open Snapshot")).set({ allowGrowX: false From 7680153fa6f191d2f531495aeddedb8378dba9b1 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Wed, 18 Aug 2021 15:12:31 +0200 Subject: [PATCH 125/137] [skip ci] bad merge --- .../5860ac6ad178_adds_snapshots_table.py | 51 ----- .../models/snapshots.py | 54 ----- .../postgres-database/tests/test_snapshots.py | 189 ------------------ .../api/v0/openapi.yaml | 4 - .../snapshots_settings.py | 1 - 5 files changed, 299 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/snapshots.py delete mode 100644 packages/postgres-database/tests/test_snapshots.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py deleted file mode 100644 index 23699028a2a..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5860ac6ad178_adds_snapshots_table.py +++ /dev/null @@ -1,51 +0,0 @@ -"""adds snapshots table - -Revision ID: 5860ac6ad178 -Revises: c2d3acc313e1 -Create Date: 2021-08-11 13:21:55.415592+00:00 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "5860ac6ad178" -down_revision = "c2d3acc313e1" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "snapshots", - sa.Column("id", sa.BigInteger(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column("created_at", sa.DateTime(), nullable=False), - sa.Column("parent_uuid", sa.String(), nullable=False), - sa.Column("project_uuid", sa.String(), nullable=False), - sa.ForeignKeyConstraint( - ["parent_uuid"], - ["projects.uuid"], - name="fk_snapshots_parent_uuid_projects", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["project_uuid"], - ["projects.uuid"], - name="fk_snapshots_project_uuid_projects", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("project_uuid"), - sa.UniqueConstraint( - "parent_uuid", "created_at", name="snapshot_from_project_uniqueness" - ), - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("snapshots") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py b/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py deleted file mode 100644 index db5ac215031..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/snapshots.py +++ /dev/null @@ -1,54 +0,0 @@ -import sqlalchemy as sa - -from .base import metadata - -snapshots = sa.Table( - "snapshots", - metadata, - sa.Column( - "id", - sa.BigInteger, - nullable=False, - primary_key=True, - doc="Global snapshot identifier index", - ), - sa.Column("name", sa.String, nullable=False, doc="Display name"), - sa.Column( - "created_at", - sa.DateTime(), - nullable=False, - doc="Timestamp for this snapshot." - "It corresponds to the last_change_date of the parent project " - "at the time the snapshot was taken.", - ), - sa.Column( - "parent_uuid", - sa.String, - sa.ForeignKey( - "projects.uuid", - name="fk_snapshots_parent_uuid_projects", - ondelete="CASCADE", - ), - nullable=False, - unique=False, - doc="UUID of the parent project", - ), - sa.Column( - "project_uuid", - sa.String, - sa.ForeignKey( - "projects.uuid", - name="fk_snapshots_project_uuid_projects", - ondelete="CASCADE", - ), - nullable=False, - unique=True, - doc="UUID of the project associated to this snapshot", - ), - sa.UniqueConstraint( - "parent_uuid", "created_at", name="snapshot_from_project_uniqueness" - ), -) - - -# Snapshot : convert_to_pydantic(snapshot) diff --git a/packages/postgres-database/tests/test_snapshots.py b/packages/postgres-database/tests/test_snapshots.py deleted file mode 100644 index 594ca3ec0f2..00000000000 --- a/packages/postgres-database/tests/test_snapshots.py +++ /dev/null @@ -1,189 +0,0 @@ -# pylint: disable=no-value-for-parameter -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -from copy import deepcopy -from typing import Callable, Optional, Set -from uuid import UUID, uuid3 - -import pytest -from aiopg.sa.engine import Engine -from aiopg.sa.result import ResultProxy, RowProxy -from pytest_simcore.helpers.rawdata_fakers import random_project, random_user -from simcore_postgres_database.errors import UniqueViolation -from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.models.snapshots import snapshots -from simcore_postgres_database.models.users import users - -USERNAME = "me" -PARENT_PROJECT_NAME = "parent" - - -@pytest.fixture -async def engine(pg_engine: Engine): - # injects ... - async with pg_engine.acquire() as conn: - # a 'me' user - user_id = await conn.scalar( - users.insert().values(**random_user(name=USERNAME)).returning(users.c.id) - ) - # has a project 'parent' - await conn.execute( - projects.insert().values( - **random_project(prj_owner=user_id, name=PARENT_PROJECT_NAME) - ) - ) - yield pg_engine - - -@pytest.fixture -def exclude() -> Set: - return { - "id", - "uuid", - "creation_date", - "last_change_date", - "hidden", - "published", - } - - -@pytest.fixture -def create_snapshot(exclude: Set) -> Callable: - async def _create_snapshot(child_index: int, parent_prj, conn) -> int: - # NOTE: used as FAKE prototype - - # create project-snapshot - prj_dict = {c: deepcopy(parent_prj[c]) for c in parent_prj if c not in exclude} - - prj_dict["name"] += f" [snapshot {child_index}]" - prj_dict["uuid"] = uuid3(UUID(parent_prj.uuid), f"snapshot.{child_index}") - # creation_data = state of parent upon copy! WARNING: changes can be state changes and not project definition? - prj_dict["creation_date"] = parent_prj.last_change_date - prj_dict["hidden"] = True - prj_dict["published"] = False - - # NOTE: a snapshot has no results but workbench stores some states, - # - input hashes - # - node ids - - # - # Define policies about changes in parent project and - # how it influence children - # - project_uuid: str = await conn.scalar( - projects.insert().values(**prj_dict).returning(projects.c.uuid) - ) - - assert UUID(project_uuid) == prj_dict["uuid"] - - # create snapshot - snapshot_id = await conn.scalar( - snapshots.insert() - .values( - name=f"Snapshot {child_index} [{parent_prj.name}]", - created_at=parent_prj.last_change_date, - parent_uuid=parent_prj.uuid, - project_uuid=project_uuid, - ) - .returning(snapshots.c.id) - ) - return snapshot_id - - return _create_snapshot - - -async def test_creating_snapshots( - engine: Engine, create_snapshot: Callable, exclude: Set -): - - async with engine.acquire() as conn: - # get parent - res: ResultProxy = await conn.execute( - projects.select().where(projects.c.name == PARENT_PROJECT_NAME) - ) - parent_prj: Optional[RowProxy] = await res.first() - - assert parent_prj - - # take one snapshot - first_snapshot_id = await create_snapshot(0, parent_prj, conn) - - # modify parent - updated_parent_prj = await ( - await conn.execute( - projects.update() - .values(description="foo") - .where(projects.c.id == parent_prj.id) - .returning(projects) - ) - ).first() - - assert updated_parent_prj - assert updated_parent_prj.id == parent_prj.id - assert updated_parent_prj.description != parent_prj.description - assert updated_parent_prj.creation_date < updated_parent_prj.last_change_date - - # take another snapshot - second_snapshot_id = await create_snapshot(1, updated_parent_prj, conn) - - second_snapshot = await ( - await conn.execute( - snapshots.select().where(snapshots.c.id == second_snapshot_id) - ) - ).first() - - assert second_snapshot - assert second_snapshot.id != first_snapshot_id - assert second_snapshot.created_at == updated_parent_prj.last_change_date - - # get project corresponding to first snapshot - j = projects.join(snapshots, projects.c.uuid == snapshots.c.project_uuid) - selected_snapshot_project = await ( - await conn.execute( - projects.select() - .select_from(j) - .where(snapshots.c.id == second_snapshot_id) - ) - ).first() - - assert selected_snapshot_project - assert selected_snapshot_project.uuid == second_snapshot.project_uuid - assert parent_prj.uuid == second_snapshot.parent_uuid - - def extract(t): - return {k: t[k] for k in t if k not in exclude.union({"name"})} - - assert extract(selected_snapshot_project) == extract(updated_parent_prj) - - # TODO: if we call to take consecutive snapshots ... of the same thing, it should - # return existing - - -async def test_multiple_snapshots_of_same_project( - engine: Engine, create_snapshot: Callable -): - async with engine.acquire() as conn: - # get parent - res: ResultProxy = await conn.execute( - projects.select().where(projects.c.name == PARENT_PROJECT_NAME) - ) - parent_prj: Optional[RowProxy] = await res.first() - assert parent_prj - - # take first snapshot - await create_snapshot(0, parent_prj, conn) - - # no changes in the parent! - with pytest.raises(UniqueViolation): - await create_snapshot(1, parent_prj, conn) - - -def test_deleting_snapshots(): - # test delete child project -> deletes snapshot - # test delete snapshot -> deletes child project - - # test delete parent project -> deletes snapshots - # test delete snapshot does NOT delete parent - pass diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index e84b842e44d..1dd84a0128e 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -13263,10 +13263,6 @@ paths: schema: title: Offset type: integer -<<<<<<< HEAD - description: index to the first item to return (pagination) -======= ->>>>>>> master default: 0 name: offset in: query diff --git a/services/web/server/src/simcore_service_webserver/snapshots_settings.py b/services/web/server/src/simcore_service_webserver/snapshots_settings.py index d3b6b48b51c..e69de29bb2d 100644 --- a/services/web/server/src/simcore_service_webserver/snapshots_settings.py +++ b/services/web/server/src/simcore_service_webserver/snapshots_settings.py @@ -1 +0,0 @@ -# TODO: do not enable From d726342cbe1856b2ef20f5e2c40c5cc2334c3a1d Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Wed, 18 Aug 2021 15:13:04 +0200 Subject: [PATCH 126/137] [skip ci] minor --- .../server/src/simcore_service_webserver/snapshots_settings.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 services/web/server/src/simcore_service_webserver/snapshots_settings.py diff --git a/services/web/server/src/simcore_service_webserver/snapshots_settings.py b/services/web/server/src/simcore_service_webserver/snapshots_settings.py deleted file mode 100644 index e69de29bb2d..00000000000 From b6fc59b9ee04e824ee60814295a8bfcfbbb6dec1 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Wed, 18 Aug 2021 16:06:22 +0200 Subject: [PATCH 127/137] [skip ci] minor --- .../web/client/source/class/osparc/desktop/StartStopButtons.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/client/source/class/osparc/desktop/StartStopButtons.js b/services/web/client/source/class/osparc/desktop/StartStopButtons.js index 5ec189242f5..a31881a97ef 100644 --- a/services/web/client/source/class/osparc/desktop/StartStopButtons.js +++ b/services/web/client/source/class/osparc/desktop/StartStopButtons.js @@ -72,7 +72,7 @@ qx.Class.define("osparc.desktop.StartStopButtons", { selectedNodeIds.forEach(selectedNodeId => { runnableNodes.push(this.getStudy().getWorkbench().getNode(selectedNodeId)); }); - const isSelectionRunnable = runnableNodes.some(node => node.isComputational()); + const isSelectionRunnable = runnableNodes.length && runnableNodes.some(node => node.isComputational()); if (isSelectionRunnable) { this.__startButton.exclude(); this.__startSelectionButton.show(); From b63ac9f853fa9172515a278234e167bc98f0db50 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Thu, 19 Aug 2021 15:30:00 +0200 Subject: [PATCH 128/137] removed Sweeper related frontend code --- .../component/form/renderer/PropForm.js | 2 +- .../component/snapshots/SnapshotsView.js | 12 +- .../osparc/component/sweeper/Parameters.js | 137 --------- .../class/osparc/component/sweeper/Sweeper.js | 92 ------ .../source/class/osparc/data/model/Study.js | 19 +- .../source/class/osparc/data/model/Sweeper.js | 287 ------------------ .../class/osparc/desktop/StudyEditor.js | 47 +-- .../class/osparc/desktop/WorkbenchToolbar.js | 41 +-- .../class/osparc/desktop/WorkbenchView.js | 58 +--- .../client/source/class/osparc/utils/Utils.js | 13 + 10 files changed, 37 insertions(+), 671 deletions(-) delete mode 100644 services/web/client/source/class/osparc/component/sweeper/Parameters.js delete mode 100644 services/web/client/source/class/osparc/component/sweeper/Sweeper.js delete mode 100644 services/web/client/source/class/osparc/data/model/Sweeper.js diff --git a/services/web/client/source/class/osparc/component/form/renderer/PropForm.js b/services/web/client/source/class/osparc/component/form/renderer/PropForm.js index cb08b88ba16..8dc41988e3f 100644 --- a/services/web/client/source/class/osparc/component/form/renderer/PropForm.js +++ b/services/web/client/source/class/osparc/component/form/renderer/PropForm.js @@ -88,7 +88,7 @@ qx.Class.define("osparc.component.form.renderer.PropForm", { const paramsMenuBtn = this.__getParamsMenuButton(field.key).set({ visibility: "excluded" }); - osparc.data.model.Sweeper.isSweeperEnabled() + osparc.utils.Utils.isSweeperEnabled() .then(isSweeperEnabled => { field.bind("visibility", paramsMenuBtn, "visibility", { converter: visibility => (visibility === "visible" && isSweeperEnabled) ? "visible" : "excluded" diff --git a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js index 4ba1164e056..4cbc17b8a53 100644 --- a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js +++ b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js @@ -23,16 +23,7 @@ qx.Class.define("osparc.component.snapshots.SnapshotsView", { this._setLayout(new qx.ui.layout.VBox(10)); - if (study.isSnapshot()) { - const primaryStudyId = study.getSweeper().getPrimaryStudyId(); - const openPrimaryStudyParamBtn = new qx.ui.form.Button(this.tr("Open Main Study")).set({ - allowGrowX: false - }); - openPrimaryStudyParamBtn.addListener("execute", () => { - this.fireDataEvent("openPrimaryStudy", primaryStudyId); - }); - this._add(openPrimaryStudyParamBtn); - } else { + if (study.hasSnapshots()) { this.__primaryStudy = study; const snapshotsSection = this.__buildSnapshotsSection(); this._add(snapshotsSection, { @@ -42,7 +33,6 @@ qx.Class.define("osparc.component.snapshots.SnapshotsView", { }, events: { - "openPrimaryStudy": "qx.event.type.Data", "openSnapshot": "qx.event.type.Data" }, diff --git a/services/web/client/source/class/osparc/component/sweeper/Parameters.js b/services/web/client/source/class/osparc/component/sweeper/Parameters.js deleted file mode 100644 index d19e7ca091a..00000000000 --- a/services/web/client/source/class/osparc/component/sweeper/Parameters.js +++ /dev/null @@ -1,137 +0,0 @@ -/* ************************************************************************ - - osparc - the simcore frontend - - https://osparc.io - - Copyright: - 2020 IT'IS Foundation, https://itis.swiss - - License: - MIT: https://opensource.org/licenses/MIT - - Authors: - * Odei Maiz (odeimaiz) - -************************************************************************ */ - -/** - * - */ - -qx.Class.define("osparc.component.sweeper.Parameters", { - extend: osparc.ui.table.Table, - - construct: function(primaryStudy) { - this.__primaryStudy = primaryStudy; - const model = this.__initModel(); - - this.base(arguments, model, { - tableColumnModel: obj => new qx.ui.table.columnmodel.Resize(obj), - statusBarVisible: false, - initiallyHiddenColumns: [0] - }); - - this.__initTable(); - this.updateTable(); - }, - - members: { // eslint-disable-line qx-rules/no-refs-in-members - __model: null, - - __cols: { - "id": { - col: 0, - label: qx.locale.Manager.tr("Id") - }, - "label": { - col: 1, - label: qx.locale.Manager.tr("Name") - }, - "low": { - col: 2, - label: qx.locale.Manager.tr("Low") - }, - "high": { - col: 3, - label: qx.locale.Manager.tr("High") - }, - "nSteps": { - col: 4, - label: qx.locale.Manager.tr("#Steps") - }, - "distribution": { - col: 5, - label: qx.locale.Manager.tr("Distribution") - } - }, - - __initModel: function() { - const model = this.__model = new qx.ui.table.model.Simple(); - - model.addListener("dataChanged", e => { - const data = e.getData(); - if (data.firstColumn === data.lastColumn && data.firstRow === data.lastRow) { - const rowData = model.getRowData(data.firstRow); - this.__dataChanged(rowData); - } - }); - - const cols = []; - Object.keys(this.__cols).forEach(colKey => { - cols.push(this.__cols[colKey].label); - }); - model.setColumns(cols); - - return model; - }, - - __initTable: function() { - const cols = this.__cols; - - const model = this.__model; - model.setColumnEditable(cols["id"].col, false); - model.setColumnEditable(cols["label"].col, true); - model.setColumnEditable(cols["low"].col, true); - model.setColumnEditable(cols["high"].col, true); - model.setColumnEditable(cols["nSteps"].col, true); - model.setColumnEditable(cols["distribution"].col, false); - - const columnModel = this.getTableColumnModel(); - columnModel.setDataCellRenderer(cols["low"].col, new qx.ui.table.cellrenderer.Number()); - columnModel.setDataCellRenderer(cols["high"].col, new qx.ui.table.cellrenderer.Number()); - - this.getSelectionModel().setSelectionMode(qx.ui.table.selection.Model.SINGLE_SELECTION); - }, - - updateTable: function() { - const parameters = this.__primaryStudy.getSweeper().getParameters(); - - const rows = []; - parameters.forEach(parameter => { - const row = []; - row[this.__cols["id"].col] = parameter["id"]; - row[this.__cols["label"].col] = parameter["label"]; - row[this.__cols["low"].col] = parameter["low"]; - row[this.__cols["high"].col] = parameter["high"]; - row[this.__cols["nSteps"].col] = parameter["nSteps"]; - row[this.__cols["distribution"].col] = parameter["distribution"]; - rows.push(row); - }); - this.getTableModel().setData(rows, false); - }, - - __dataChanged: function(rowData) { - const cols = this.__cols; - const parameters = this.__primaryStudy.getSweeper().getParameters(); - const idx = parameters.findIndex(existingParam => existingParam.id === rowData[cols["id"].col]); - if (idx !== -1) { - const parameter = parameters[idx]; - parameter.label = rowData[cols["label"].col]; - parameter.low = rowData[cols["low"].col]; - parameter.high = rowData[cols["high"].col]; - parameter.nSteps = rowData[cols["nSteps"].col]; - } - } - } -}); diff --git a/services/web/client/source/class/osparc/component/sweeper/Sweeper.js b/services/web/client/source/class/osparc/component/sweeper/Sweeper.js deleted file mode 100644 index 6df7d952e4d..00000000000 --- a/services/web/client/source/class/osparc/component/sweeper/Sweeper.js +++ /dev/null @@ -1,92 +0,0 @@ -/* ************************************************************************ - - osparc - the simcore frontend - - https://osparc.io - - Copyright: - 2020 IT'IS Foundation, https://itis.swiss - - License: - MIT: https://opensource.org/licenses/MIT - - Authors: - * Odei Maiz (odeimaiz) - -************************************************************************ */ - -qx.Class.define("osparc.component.sweeper.Sweeper", { - extend: qx.ui.core.Widget, - - construct: function(study) { - this.base(arguments); - - this._setLayout(new qx.ui.layout.VBox(10)); - - if (study.isSnapshot()) { - const primaryStudyId = study.getSweeper().getPrimaryStudyId(); - const openPrimaryStudyParamBtn = new qx.ui.form.Button(this.tr("Open Main Study")).set({ - allowGrowX: false - }); - openPrimaryStudyParamBtn.addListener("execute", () => { - this.fireDataEvent("openPrimaryStudy", primaryStudyId); - }); - this._add(openPrimaryStudyParamBtn); - } else { - this.__primaryStudy = study; - const parametersSection = this.__buildParametersSection(); - this._add(parametersSection, { - flex: 1 - }); - } - }, - - events: { - "openPrimaryStudy": "qx.event.type.Data" - }, - - members: { - __primaryStudy: null, - __parametersTable: null, - - __buildParametersSection: function() { - const parametersSection = new qx.ui.groupbox.GroupBox(this.tr("Parameters")).set({ - layout: new qx.ui.layout.VBox(5) - }); - const newParamBtn = this.__createNewParamBtn(); - parametersSection.add(newParamBtn); - - const parametersTable = this.__parametersTable = new osparc.component.sweeper.Parameters(this.__primaryStudy); - parametersSection.add(parametersTable, { - flex: 1 - }); - return parametersSection; - }, - - __createNewParamBtn: function() { - const label = this.tr("Create new Parameter"); - const newParamBtn = new qx.ui.form.Button(label).set({ - allowGrowX: false - }); - newParamBtn.addListener("execute", () => { - const newParamName = new osparc.component.widget.Renamer(null, null, label); - newParamName.addListener("labelChanged", e => { - const primaryStudy = this.__primaryStudy; - let newParameterLabel = e.getData()["newLabel"]; - newParameterLabel = newParameterLabel.replace(" ", "_"); - if (primaryStudy.getSweeper().parameterLabelExists(newParameterLabel)) { - const msg = this.tr("Parameter name already exists"); - osparc.component.message.FlashMessenger.getInstance().logAs(msg, "ERROR"); - } else { - primaryStudy.getSweeper().addNewParameter(newParameterLabel); - this.__parametersTable.updateTable(); - newParamName.close(); - } - }, this); - newParamName.center(); - newParamName.open(); - }, this); - return newParamBtn; - } - } -}); diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 7d5bc1154d5..0138e5f511b 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -60,8 +60,6 @@ qx.Class.define("osparc.data.model.Study", { const wbData = studyData.workbench || this.getWorkbench(); this.setWorkbench(new osparc.data.model.Workbench(wbData, studyData.ui)); this.setUi(new osparc.data.model.StudyUI(studyData.ui)); - - this.setSweeper(new osparc.data.model.Sweeper(studyData)); }, properties: { @@ -147,11 +145,6 @@ qx.Class.define("osparc.data.model.Study", { nullable: true }, - sweeper: { - check: "osparc.data.model.Sweeper", - nullable: true - }, - state: { check: "Object", nullable: true, @@ -276,10 +269,6 @@ qx.Class.define("osparc.data.model.Study", { }, isSnapshot: function() { - if (this.getSweeper()) { - const primaryStudyId = this.getSweeper().getPrimaryStudyId(); - return primaryStudyId !== null; - } return false; }, @@ -355,11 +344,6 @@ qx.Class.define("osparc.data.model.Study", { jsonObject[key] = this.getUi().serialize(); return; } - if (key === "sweeper") { - jsonObject["dev"] = {}; - jsonObject["dev"]["sweeper"] = this.getSweeper().serialize(); - return; - } const value = this.get(key); if (value !== null) { // only put the value in the payload if there is a value @@ -397,8 +381,7 @@ qx.Class.define("osparc.data.model.Study", { creationDate: new Date(data.creationDate), lastChangeDate: new Date(data.lastChangeDate), workbench: this.getWorkbench(), - ui: this.getUi(), - sweeper: this.getSweeper() + ui: this.getUi() }); const nodes = this.getWorkbench().getNodes(true); diff --git a/services/web/client/source/class/osparc/data/model/Sweeper.js b/services/web/client/source/class/osparc/data/model/Sweeper.js deleted file mode 100644 index 36e2e7b99b5..00000000000 --- a/services/web/client/source/class/osparc/data/model/Sweeper.js +++ /dev/null @@ -1,287 +0,0 @@ -/* ************************************************************************ - - osparc - the simcore frontend - - https://osparc.io - - Copyright: - 2020 IT'IS Foundation, https://itis.swiss - - License: - MIT: https://opensource.org/licenses/MIT - - Authors: - * Odei Maiz (odeimaiz) - -************************************************************************ */ - -/** - * Class that stores Sweeper related data. - */ - -qx.Class.define("osparc.data.model.Sweeper", { - extend: qx.core.Object, - - /** - * @param studyData {Object} Object containing the serialized Study Data - */ - construct: function(studyData = {}) { - this.base(arguments); - - this.__parameters = []; - this.__parameterValues = []; - this.__combinations = []; - this.__secondaryStudyIds = []; - this.__primaryStudyId = null; - - if ("dev" in studyData && "sweeper" in studyData["dev"]) { - this.deserialize(studyData["dev"]["sweeper"]); - } - }, - - statics: { - isSweeperEnabled: function() { - return new Promise((resolve, reject) => { - osparc.utils.LibVersions.getPlatformName() - .then(platformName => { - if (["dev", "master"].includes(platformName)) { - resolve(true); - } else { - resolve(false); - } - }); - }); - } - }, - - events: { - "changeParameters": "qx.event.type.Event" - }, - - members: { - __parameters: null, - __parameterValues: null, - __combinations: null, - __secondaryStudyIds: null, - __primaryStudyId: null, - - /* PARAMETERS */ - hasParameters: function() { - return Boolean(Object.keys(this.__parameters).length); - }, - - getParameter: function(parameterId) { - return this.__parameters.find(parameter => parameter.id === parameterId); - }, - - getParameters: function() { - return this.__parameters; - }, - - parameterLabelExists: function(parameterLabel) { - const params = this.getParameters(); - const idx = params.findIndex(param => param.label === parameterLabel); - return (idx !== -1); - }, - - __setParameters: function(parameters) { - this.__parameters = parameters; - this.fireEvent("changeParameters"); - }, - - addNewParameter: function(parameterLabel) { - if (!this.parameterLabelExists(parameterLabel)) { - const parameter = { - id: parameterLabel, - label: parameterLabel, - low: 1, - high: 2, - nSteps: 2, - distribution: "linear" - }; - this.__parameters.push(parameter); - - this.fireEvent("changeParameters"); - - return parameter; - } - return null; - }, - /* /PARAMETERS */ - - /* /PARAMETER VALUES */ - hasParameterValues: function() { - return Boolean(this.__parameterValues.length); - }, - - getParameterValues: function() { - return this.__parameterValues; - }, - - __setParameterValues: function(parameterValues) { - this.__parameterValues = parameterValues; - }, - /* /PARAMETER VALUES */ - - /* COMBINATIONS */ - __hasCombinations: function() { - return this.__combinations.length; - }, - - getCombinations: function() { - return this.__combinations; - }, - - __setCombinations: function(combinations) { - this.__combinations = combinations; - }, - /* /COMBINATIONS */ - - /* SECONDARY STUDIES */ - hasSecondaryStudies: function() { - return this.__secondaryStudyIds.length; - }, - - getSecondaryStudyIds: function() { - return this.__secondaryStudyIds; - }, - - __removeSecondaryStudy: function(secondaryStudyId) { - return new Promise((resolve, reject) => { - this.__deleteSecondaryStudy(secondaryStudyId) - .then(() => { - const idx = this.__secondaryStudyIds.findIndex(secStudyId => secStudyId === secondaryStudyId); - if (idx > -1) { - this.__secondaryStudyIds.splice(idx, 1); - } - }) - .catch(er => { - console.error(er); - }) - .finally(() => resolve()); - }); - }, - - removeSecondaryStudies: function() { - const deletePromises = []; - this.getSecondaryStudyIds().forEach(secondaryStudyId => { - deletePromises.push(this.__removeSecondaryStudy(secondaryStudyId)); - }); - return new Promise((resolve, reject) => { - Promise.all(deletePromises) - .then(() => { - resolve(); - }); - }); - }, - - __setSecondaryStudies: function(secondaryStudyIds) { - secondaryStudyIds.forEach(secondaryStudyId => { - this.__secondaryStudyIds.push(secondaryStudyId); - }); - }, - - __deleteSecondaryStudy: function(secondaryStudyId) { - return osparc.store.Store.getInstance().deleteStudy(secondaryStudyId); - }, - /* /SECONDARY STUDIES */ - - /* PRIMARY STUDY */ - __setPrimaryStudyId: function(primaryStudyId) { - this.__primaryStudyId = primaryStudyId; - }, - - getPrimaryStudyId: function() { - return this.__primaryStudyId; - }, - /* /PRIMARY STUDY */ - - recreateSnaphots: function(primaryStudyData) { - return new Promise((resolve, reject) => { - // delete previous iterations - this.removeSecondaryStudies() - .then(() => { - const usedParams = osparc.data.StudyParametrizer.getActiveParameters(primaryStudyData, this.__parameters); - - const steps = osparc.data.StudyParametrizer.calculateSteps(usedParams); - if (steps.length !== usedParams.length) { - console.error("Number of elements in the array of steps must be the same as parameters"); - reject(); - } - - const combinations = osparc.data.StudyParametrizer.calculateCombinations(steps); - this.__setCombinations(combinations); - - osparc.data.StudyParametrizer.recreateSnaphots(primaryStudyData, usedParams, combinations) - .then(secondaryStudiesData => { - secondaryStudiesData.forEach(secondaryStudyData => { - this.__secondaryStudyIds.push(secondaryStudyData.uuid); - }); - resolve(this.getSecondaryStudyIds()); - }); - }); - }); - }, - - serialize: function() { - const obj = {}; - - if (this.hasParameters()) { - obj["parameters"] = []; - this.getParameters().forEach(parameter => { - obj["parameters"].push(parameter); - }); - } - - if (this.hasParameterValues()) { - obj["parameterValues"] = []; - this.getParameterValues().forEach(parameterValue => { - obj["parameterValues"].push(parameterValue); - }); - } - - if (this.__hasCombinations()) { - obj["combinations"] = []; - this.getCombinations().forEach(combination => { - obj["combinations"].push(combination); - }); - } - - if (this.hasSecondaryStudies()) { - obj["secondaryStudyIds"] = []; - this.getSecondaryStudyIds().forEach(secondaryStudyId => { - obj["secondaryStudyIds"].push(secondaryStudyId); - }); - } - - const primaryStudyId = this.getPrimaryStudyId(); - if (primaryStudyId) { - obj["primaryStudyId"] = primaryStudyId; - } - - return obj; - }, - - deserialize: function(sweeperData) { - if ("parameters" in sweeperData) { - this.__setParameters(sweeperData["parameters"]); - } - - if ("parameterValues" in sweeperData) { - this.__setParameterValues(sweeperData["parameterValues"]); - } - - if ("combinations" in sweeperData) { - this.__setCombinations(sweeperData["combinations"]); - } - - if ("secondaryStudyIds" in sweeperData) { - this.__setSecondaryStudies(sweeperData["secondaryStudyIds"]); - } - - if ("primaryStudyId" in sweeperData) { - this.__setPrimaryStudyId(sweeperData["primaryStudyId"]); - } - } - } -}); diff --git a/services/web/client/source/class/osparc/desktop/StudyEditor.js b/services/web/client/source/class/osparc/desktop/StudyEditor.js index e4952d1c460..746c4f0449d 100644 --- a/services/web/client/source/class/osparc/desktop/StudyEditor.js +++ b/services/web/client/source/class/osparc/desktop/StudyEditor.js @@ -235,7 +235,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { startStopButtonsSS.setRunning(true); this.updateStudyDocument(true) .then(() => { - this.__doStartPipeline(partialPipeline); + this.__requestStartPipeline(this.getStudy().getUuid(), partialPipeline); }) .catch(() => { this.__getStudyLogger().error(null, "Run failed"); @@ -244,36 +244,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { }); }, - __doStartPipeline: function(partialPipeline) { - if (this.getStudy().getSweeper().hasSecondaryStudies()) { - const secondaryStudyIds = this.getStudy().getSweeper().getSecondaryStudyIds(); - secondaryStudyIds.forEach(secondaryStudyId => { - this.__requestStartPipeline(secondaryStudyId); - }); - } else { - this.__requestStartPipeline(this.getStudy().getUuid(), partialPipeline); - } - }, - __requestStartPipeline: function(studyId, partialPipeline = [], forceRestart = false) { - if (this.getStudy().isSnapshot()) { - const startPipelineView = new osparc.desktop.StartPipelineView(partialPipeline, forceRestart); - const win = osparc.ui.window.Window.popUpInWindow(startPipelineView, "Start Pipeline", 250, 290); - startPipelineView.addListener("startPipeline", e => { - const data = e.getData(); - const useCache = data["useCache"]; - this.__reallyRequestStartPipeline(studyId, partialPipeline, forceRestart, useCache); - win.close(); - }, this); - startPipelineView.addListener("cancel", () => { - win.close(); - }, this); - } else { - this.__reallyRequestStartPipeline(studyId, partialPipeline, forceRestart); - } - }, - - __reallyRequestStartPipeline: function(studyId, partialPipeline = [], forceRestart = false, useCache = false) { const url = "/computation/pipeline/" + encodeURIComponent(studyId) + ":start"; const req = new osparc.io.request.ApiRequest(url, "POST"); const startStopButtonsWB = this.__workbenchView.getStartStopButtons(); @@ -307,8 +278,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { req.setRequestData({ "subgraph": partialPipeline, - "force_restart": forceRestart, - "use_cache": useCache + "force_restart": forceRestart }); req.send(); if (partialPipeline.length) { @@ -349,18 +319,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { return; } - this.__doStopPipeline(); - }, - - __doStopPipeline: function() { - if (this.getStudy().getSweeper().hasSecondaryStudies()) { - const secondaryStudyIds = this.getStudy().getSweeper().getSecondaryStudyIds(); - secondaryStudyIds.forEach(secondaryStudyId => { - this.__requestStopPipeline(secondaryStudyId); - }); - } else { - this.__requestStopPipeline(this.getStudy().getUuid()); - } + this.__requestStopPipeline(this.getStudy().getUuid()); }, __requestStopPipeline: function(studyId) { diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js index 60a13bf1e1d..ded30194dab 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js @@ -27,7 +27,6 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { events: { "takeSnapshot": "qx.event.type.Event", "convertToStudy": "qx.event.type.Event", - "showParameters": "qx.event.type.Event", "showSnapshots": "qx.event.type.Event", "openPrimaryStudy": "qx.event.type.Data" }, @@ -85,21 +84,6 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { this._add(control); break; } - case "primary-study-btn": { - control = new qx.ui.form.Button(this.tr("Open Main Study")).set({ - icon: "@FontAwesome5Solid/external-link-alt/14", - ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, - allowGrowX: false - }); - control.addListener("execute", () => { - const primaryStudyId = this.getStudy().getSweeper().getPrimaryStudyId(); - if (primaryStudyId) { - this.fireDataEvent("openPrimaryStudy", primaryStudyId); - } - }, this); - this._add(control); - break; - } } return control || this.base(arguments, id); }, @@ -113,12 +97,6 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { const takeSnapshotBtn = this.getChildControl("take-snapshot-btn"); takeSnapshotBtn.exclude(); - const convertToStudy = this.getChildControl("convert-to-study-btn"); - convertToStudy.exclude(); - - const primaryBtn = this.getChildControl("primary-study-btn"); - primaryBtn.exclude(); - const snapshotsBtn = this.getChildControl("snapshots-btn"); snapshotsBtn.exclude(); @@ -134,34 +112,19 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { this.__navNodes.populateButtons(nodeIds); const takeSnapshotBtn = this.getChildControl("take-snapshot-btn"); - const convertToStudyBtn = this.getChildControl("convert-to-study-btn"); - const primaryBtn = this.getChildControl("primary-study-btn"); - if (study.isSnapshot()) { - takeSnapshotBtn.exclude(); - convertToStudyBtn.show(); - primaryBtn.show(); - } else { - takeSnapshotBtn.setVisibility(osparc.data.Permissions.getInstance().canDo("study.snapshot.create") ? "visible" : "excluded"); - convertToStudyBtn.exclude(); - primaryBtn.exclude(); - } + takeSnapshotBtn.setVisibility(osparc.data.Permissions.getInstance().canDo("study.snapshot.create") ? "visible" : "excluded"); study.getWorkbench().addListener("nNodesChanged", this.evalSnapshotsBtn, this); this.evalSnapshotsBtn(); - - study.isSnapshot() ? this._startStopBtns.exclude() : this._startStopBtns.show(); } }, evalSnapshotsBtn: async function() { const study = this.getStudy(); if (study) { - const allNodes = study.getWorkbench().getNodes(true); - const hasIterators = Object.values(allNodes).some(node => node.isIterator()); - const isSnapshot = study.isSnapshot(); const hasSnapshots = await study.hasSnapshots(); const snapshotsBtn = this.getChildControl("snapshots-btn"); - (hasSnapshots || (hasIterators && !isSnapshot)) ? snapshotsBtn.show() : snapshotsBtn.exclude(); + hasSnapshots ? snapshotsBtn.show() : snapshotsBtn.exclude(); } }, diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 197dba55d5e..08097912e49 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -185,18 +185,6 @@ qx.Class.define("osparc.desktop.WorkbenchView", { }); }, - __showParameters: function() { - const study = this.getStudy(); - const sweeper = new osparc.component.sweeper.Sweeper(study); - const title = this.tr("Sweeper"); - const win = osparc.ui.window.Window.popUpInWindow(sweeper, title, 400, 500); - sweeper.addListener("openPrimaryStudy", e => { - win.close(); - const primaryStudyId = e.getData(); - this.__switchStudy(primaryStudyId); - }); - }, - __takeSnapshot: function() { const study = this.getStudy(); const takeSnapshotView = new osparc.component.snapshots.TakeSnapshotView(study); @@ -233,35 +221,25 @@ qx.Class.define("osparc.desktop.WorkbenchView", { const sweeper = new osparc.component.snapshots.SnapshotsView(study); const title = this.tr("Snapshots"); const win = osparc.ui.window.Window.popUpInWindow(sweeper, title, 600, 500); - [ - "openPrimaryStudy", - "openSnapshot" - ].forEach(signalName => { - sweeper.addListener(signalName, e => { - win.close(); - const studyId = e.getData(); - this.__switchStudy(studyId); - }); + sweeper.addListener("openSnapshot", e => { + win.close(); + const studyId = e.getData(); + const params = { + url: { + "studyId": studyId + } + }; + osparc.data.Resources.getOne("studies", params) + .then(studyData => { + study.removeIFrames(); + const data = { + studyId: studyData.uuid + }; + this.fireDataEvent("startStudy", data); + }); }); }, - __switchStudy: function(studyId) { - const params = { - url: { - "studyId": studyId - } - }; - osparc.data.Resources.getOne("studies", params) - .then(studyData => { - const study = this.getStudy(); - study.removeIFrames(); - const data = { - studyId: studyData.uuid - }; - this.fireDataEvent("startStudy", data); - }); - }, - __showWorkbenchUI: function() { const workbench = this.getStudy().getWorkbench(); const currentNode = workbench.getNode(this.__currentNodeId); @@ -544,10 +522,6 @@ qx.Class.define("osparc.desktop.WorkbenchView", { this.nodeSelected(nodeId); }, this); }); - workbenchToolbar.addListener("showSweeper", this.__showParameters, this); - if (!workbenchToolbar.hasListener("showParameters")) { - workbenchToolbar.addListener("showParameters", this.__showParameters, this); - } if (!workbenchToolbar.hasListener("takeSnapshot")) { workbenchToolbar.addListener("takeSnapshot", this.__takeSnapshot, this); } diff --git a/services/web/client/source/class/osparc/utils/Utils.js b/services/web/client/source/class/osparc/utils/Utils.js index ec4d33ca53b..b0ccad32ba5 100644 --- a/services/web/client/source/class/osparc/utils/Utils.js +++ b/services/web/client/source/class/osparc/utils/Utils.js @@ -30,6 +30,19 @@ qx.Class.define("osparc.utils.Utils", { type: "static", statics: { + isSweeperEnabled: function() { + return new Promise((resolve, reject) => { + osparc.utils.LibVersions.getPlatformName() + .then(platformName => { + if (["dev", "master"].includes(platformName)) { + resolve(true); + } else { + resolve(false); + } + }); + }); + }, + getEditButton: function() { const button = new qx.ui.form.Button(null, "@FontAwesome5Solid/pencil-alt/12").set({ allowGrowY: false, From cc1a2b27af0952f3fe740baf1156c33faf4faf45 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Thu, 19 Aug 2021 15:33:56 +0200 Subject: [PATCH 129/137] [skip ci] more clean up --- .../class/osparc/desktop/WorkbenchToolbar.js | 15 +-------------- .../source/class/osparc/desktop/WorkbenchView.js | 6 ------ 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js index ded30194dab..4fa04562705 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchToolbar.js @@ -26,9 +26,7 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { events: { "takeSnapshot": "qx.event.type.Event", - "convertToStudy": "qx.event.type.Event", - "showSnapshots": "qx.event.type.Event", - "openPrimaryStudy": "qx.event.type.Data" + "showSnapshots": "qx.event.type.Event" }, members: { @@ -61,17 +59,6 @@ qx.Class.define("osparc.desktop.WorkbenchToolbar", { this._add(control); break; } - case "convert-to-study-btn": { - control = new osparc.ui.form.FetchButton(this.tr("Convert To Study")).set({ - ...osparc.navigation.NavigationBar.BUTTON_OPTIONS, - allowGrowX: false - }); - control.addListener("execute", () => { - this.fireDataEvent("convertToStudy"); - }, this); - this._add(control); - break; - } case "snapshots-btn": { control = new qx.ui.form.Button(this.tr("Snapshots")).set({ icon: "@FontAwesome5Solid/copy/14", diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 08097912e49..ccaaddc5a72 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -528,12 +528,6 @@ qx.Class.define("osparc.desktop.WorkbenchView", { if (!workbenchToolbar.hasListener("showSnapshots")) { workbenchToolbar.addListener("showSnapshots", this.__showSnapshots, this); } - if (!workbenchToolbar.hasListener("openPrimaryStudy")) { - workbenchToolbar.addListener("openPrimaryStudy", e => { - const primaryStudyId = e.getData(); - this.__switchStudy(primaryStudyId); - }, this); - } nodesTree.addListener("changeSelectedNode", e => { const node = workbenchUI.getNodeUI(e.getData()); From 1f7aa38db0fed7bb33da8a4d4b3dc459e0e37a86 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 20 Aug 2021 09:09:20 +0200 Subject: [PATCH 130/137] loadStudy refactored --- .../component/snapshots/SnapshotsView.js | 2 +- .../source/class/osparc/desktop/MainPage.js | 65 ++++++++++--------- .../class/osparc/desktop/StudyEditor.js | 7 +- .../class/osparc/desktop/WorkbenchView.js | 24 ++----- 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js index 4cbc17b8a53..f4706b2d329 100644 --- a/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js +++ b/services/web/client/source/class/osparc/component/snapshots/SnapshotsView.js @@ -72,7 +72,7 @@ qx.Class.define("osparc.component.snapshots.SnapshotsView", { this.__openSnapshotBtn.setEnabled(true); } const selectedRow = e.getRow(); - this.__selectedSnapshot = snapshotsTable.getRowData(selectedRow)["StudyId"]; + this.__selectedSnapshot = snapshotsTable.getRowData(selectedRow); }); this.__snapshotsSection.addAt(snapshotsTable, 1, { diff --git a/services/web/client/source/class/osparc/desktop/MainPage.js b/services/web/client/source/class/osparc/desktop/MainPage.js index 5e26109b0ae..f8ba240b9ad 100644 --- a/services/web/client/source/class/osparc/desktop/MainPage.js +++ b/services/web/client/source/class/osparc/desktop/MainPage.js @@ -242,40 +242,43 @@ qx.Class.define("osparc.desktop.MainPage", { } }; osparc.data.Resources.getOne("studies", params) - .then(latestStudyData => { - if (!latestStudyData) { + .then(studyData => { + if (!studyData) { const msg = this.tr("Study not found"); throw new Error(msg); } + this.__loadStudy(studyData, pageContext); + }) + .catch(err => { + osparc.component.message.FlashMessenger.getInstance().logAs(err.message, "ERROR"); + this.__showDashboard(); + return; + }); + }, - let locked = false; - let lockedBy = false; - if ("state" in latestStudyData && "locked" in latestStudyData["state"]) { - locked = latestStudyData["state"]["locked"]["value"]; - lockedBy = latestStudyData["state"]["locked"]["owner"]; - } - if (locked && lockedBy["user_id"] !== osparc.auth.Data.getInstance().getUserId()) { - const msg = this.tr("Study is already open by ") + lockedBy["first_name"]; + __loadStudy: function(studyData, pageContext) { + let locked = false; + let lockedBy = false; + if ("state" in studyData && "locked" in studyData["state"]) { + locked = studyData["state"]["locked"]["value"]; + lockedBy = studyData["state"]["locked"]["owner"]; + } + if (locked && lockedBy["user_id"] !== osparc.auth.Data.getInstance().getUserId()) { + const msg = this.tr("Study is already open by ") + lockedBy["first_name"]; + throw new Error(msg); + } + const store = osparc.store.Store.getInstance(); + store.getInaccessibleServices(studyData) + .then(inaccessibleServices => { + if (inaccessibleServices.length) { + this.__dashboard.getStudyBrowser().resetSelection(); + const msg = osparc.utils.Study.getInaccessibleServicesMsg(inaccessibleServices); throw new Error(msg); } - const store = osparc.store.Store.getInstance(); - store.getInaccessibleServices(latestStudyData) - .then(inaccessibleServices => { - if (inaccessibleServices.length) { - this.__dashboard.getStudyBrowser().resetSelection(); - const msg = osparc.utils.Study.getInaccessibleServicesMsg(inaccessibleServices); - throw new Error(msg); - } - this.__showStudyEditor(this.__getStudyEditor()); - this.__studyEditor.setStudyData(latestStudyData) - .then(() => { - this.__syncStudyEditor(pageContext); - }); - }) - .catch(err => { - osparc.component.message.FlashMessenger.getInstance().logAs(err.message, "ERROR"); - this.__showDashboard(); - return; + this.__showStudyEditor(this.__getStudyEditor()); + this.__studyEditor.setStudyData(studyData) + .then(() => { + this.__syncStudyEditor(pageContext); }); }) .catch(err => { @@ -309,9 +312,9 @@ qx.Class.define("osparc.desktop.MainPage", { __getStudyEditor: function() { const studyEditor = this.__studyEditor || new osparc.desktop.StudyEditor(); - studyEditor.addListenerOnce("startStudy", e => { - const startStudyData = e.getData(); - this.__startStudy(startStudyData); + studyEditor.addListenerOnce("startSnapshot", e => { + const snapshotData = e.getData(); + this.__loadStudy(snapshotData); }, this); return studyEditor; }, diff --git a/services/web/client/source/class/osparc/desktop/StudyEditor.js b/services/web/client/source/class/osparc/desktop/StudyEditor.js index 746c4f0449d..3db623cfe00 100644 --- a/services/web/client/source/class/osparc/desktop/StudyEditor.js +++ b/services/web/client/source/class/osparc/desktop/StudyEditor.js @@ -26,8 +26,9 @@ qx.Class.define("osparc.desktop.StudyEditor", { const viewsStack = this.__viewsStack = new qx.ui.container.Stack(); const workbenchView = this.__workbenchView = new osparc.desktop.WorkbenchView(); - workbenchView.addListener("startStudy", e => { - this.fireDataEvent("startStudy", e.getData()); + workbenchView.addListener("startSnapshot", e => { + this.getStudy().removeIFrames(); + this.fireDataEvent("startSnapshot", e.getData()); }); viewsStack.add(workbenchView); @@ -60,7 +61,7 @@ qx.Class.define("osparc.desktop.StudyEditor", { events: { "forceBackToDashboard": "qx.event.type.Event", - "startStudy": "qx.event.type.Data" + "startSnapshot": "qx.event.type.Data" }, properties: { diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index ccaaddc5a72..fa10cfed26c 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -44,7 +44,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { }, events: { - "startStudy": "qx.event.type.Data" + "startSnapshot": "qx.event.type.Data" }, properties: { @@ -218,25 +218,13 @@ qx.Class.define("osparc.desktop.WorkbenchView", { __showSnapshots: function() { const study = this.getStudy(); - const sweeper = new osparc.component.snapshots.SnapshotsView(study); + const snapshots = new osparc.component.snapshots.SnapshotsView(study); const title = this.tr("Snapshots"); - const win = osparc.ui.window.Window.popUpInWindow(sweeper, title, 600, 500); - sweeper.addListener("openSnapshot", e => { + const win = osparc.ui.window.Window.popUpInWindow(snapshots, title, 600, 500); + snapshots.addListener("openSnapshot", e => { win.close(); - const studyId = e.getData(); - const params = { - url: { - "studyId": studyId - } - }; - osparc.data.Resources.getOne("studies", params) - .then(studyData => { - study.removeIFrames(); - const data = { - studyId: studyData.uuid - }; - this.fireDataEvent("startStudy", data); - }); + const snapshot = e.getData(); + this.fireDataEvent("startSnapshot", snapshot); }); }, From 03580b73b22f3749b59079f92239a355751e0a48 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 20 Aug 2021 11:34:10 +0200 Subject: [PATCH 131/137] load snapshot --- .../osparc/component/snapshots/Snapshots.js | 27 +++++++++++---- .../source/class/osparc/data/model/Study.js | 1 + .../source/class/osparc/desktop/MainPage.js | 34 +++++++++++++++++-- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js index 59cb41af9ad..3b870929732 100644 --- a/services/web/client/source/class/osparc/component/snapshots/Snapshots.js +++ b/services/web/client/source/class/osparc/component/snapshots/Snapshots.js @@ -24,7 +24,11 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { const model = this.__initModel(); this.base(arguments, model, { - initiallyHiddenColumns: [this.self().T_POS.ID.col], + initiallyHiddenColumns: [ + this.self().T_POS.STUDY_ID.col, + this.self().T_POS.SNAPSHOT_ID.col, + this.self().T_POS.PARENT_ID.col + ], statusBarVisible: false }); @@ -36,9 +40,9 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { statics: { T_POS: { - ID: { + STUDY_ID: { col: 0, - label: qx.locale.Manager.tr("StudyId") + label: "StudyId" }, NAME: { col: 1, @@ -47,6 +51,14 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { DATE: { col: 2, label: qx.locale.Manager.tr("Created At") + }, + SNAPSHOT_ID: { + col: 3, + label: "SnapshotId" + }, + PARENT_ID: { + col: 4, + label: "ParentId" } } }, @@ -74,20 +86,23 @@ qx.Class.define("osparc.component.snapshots.Snapshots", { __populateSnapshotsTable: function() { const columnModel = this.getTableColumnModel(); - columnModel.setDataCellRenderer(this.self().T_POS.ID.col, new qx.ui.table.cellrenderer.String()); + columnModel.setDataCellRenderer(this.self().T_POS.STUDY_ID.col, new qx.ui.table.cellrenderer.String()); columnModel.setDataCellRenderer(this.self().T_POS.NAME.col, new qx.ui.table.cellrenderer.String()); columnModel.setDataCellRenderer(this.self().T_POS.DATE.col, new qx.ui.table.cellrenderer.Date()); - columnModel.setDataCellRenderer(this.self().T_POS.ID.col, new qx.ui.table.cellrenderer.String()); + columnModel.setDataCellRenderer(this.self().T_POS.SNAPSHOT_ID.col, new qx.ui.table.cellrenderer.String()); + columnModel.setDataCellRenderer(this.self().T_POS.PARENT_ID.col, new qx.ui.table.cellrenderer.String()); this.__primaryStudy.getSnapshots() .then(snapshots => { const rows = []; snapshots.reverse().forEach(snapshot => { const row = []; - row[this.self().T_POS.ID.col] = snapshot["project_uuid"]; + row[this.self().T_POS.STUDY_ID.col] = snapshot["project_uuid"]; row[this.self().T_POS.NAME.col] = snapshot["label"]; const date = new Date(snapshot["created_at"]); row[this.self().T_POS.DATE.col] = osparc.utils.Utils.formatDateAndTime(date); + row[this.self().T_POS.SNAPSHOT_ID.col] = snapshot["id"]; + row[this.self().T_POS.PARENT_ID.col] = snapshot["parent_uuid"]; rows.push(row); }); this.getTableModel().setData(rows, false); diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 0138e5f511b..6c4481eacfa 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -236,6 +236,7 @@ qx.Class.define("osparc.data.model.Study", { }; osparc.data.Resources.get("snapshots", params) .then(snapshots => { + console.log(snapshots); resolve(snapshots); }) .catch(err => { diff --git a/services/web/client/source/class/osparc/desktop/MainPage.js b/services/web/client/source/class/osparc/desktop/MainPage.js index f8ba240b9ad..0754a223c8f 100644 --- a/services/web/client/source/class/osparc/desktop/MainPage.js +++ b/services/web/client/source/class/osparc/desktop/MainPage.js @@ -256,6 +256,33 @@ qx.Class.define("osparc.desktop.MainPage", { }); }, + __startSnapshot: function(snapshotData) { + this.__showLoadingPage(this.tr("Loading Snapshot")); + const params = { + url: { + "studyId": snapshotData["ParentId"], + "snapshotId": snapshotData["SnapshotId"] + } + }; + osparc.data.Resources.getOne("snapshots", params) + .then(snapshotResp => { + if (!snapshotResp) { + const msg = this.tr("Snapshot not found"); + throw new Error(msg); + } + fetch(snapshotResp["url_project"]) + .then(response => response.json()) + .then(data => { + this.__loadStudy(data["data"]); + }); + }) + .catch(err => { + osparc.component.message.FlashMessenger.getInstance().logAs(err.message, "ERROR"); + this.__showDashboard(); + return; + }); + }, + __loadStudy: function(studyData, pageContext) { let locked = false; let lockedBy = false; @@ -311,10 +338,13 @@ qx.Class.define("osparc.desktop.MainPage", { }, __getStudyEditor: function() { - const studyEditor = this.__studyEditor || new osparc.desktop.StudyEditor(); + if (this.__studyEditor) { + return this.__studyEditor; + } + const studyEditor = new osparc.desktop.StudyEditor(); studyEditor.addListenerOnce("startSnapshot", e => { const snapshotData = e.getData(); - this.__loadStudy(snapshotData); + this.__startSnapshot(snapshotData); }, this); return studyEditor; }, From d0a9f1a952e69bbbe84e9843877817a583b9513f Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 20 Aug 2021 11:39:39 +0200 Subject: [PATCH 132/137] cleanup --- services/web/client/source/class/osparc/data/model/Study.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/client/source/class/osparc/data/model/Study.js b/services/web/client/source/class/osparc/data/model/Study.js index 6c4481eacfa..0138e5f511b 100644 --- a/services/web/client/source/class/osparc/data/model/Study.js +++ b/services/web/client/source/class/osparc/data/model/Study.js @@ -236,7 +236,6 @@ qx.Class.define("osparc.data.model.Study", { }; osparc.data.Resources.get("snapshots", params) .then(snapshots => { - console.log(snapshots); resolve(snapshots); }) .catch(err => { From b78301f11b8547230d91402f2fac2d39e14d33f7 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 20 Aug 2021 17:09:18 +0200 Subject: [PATCH 133/137] fixed weird bug --- services/web/client/source/class/osparc/data/Resources.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/web/client/source/class/osparc/data/Resources.js b/services/web/client/source/class/osparc/data/Resources.js index cc2f68ba5fb..b57f218459d 100644 --- a/services/web/client/source/class/osparc/data/Resources.js +++ b/services/web/client/source/class/osparc/data/Resources.js @@ -128,6 +128,7 @@ qx.Class.define("osparc.data.Resources", { url: statics.API + "/projects/{studyId}" }, addNode: { + useCache: false, method: "POST", url: statics.API + "/projects/{studyId}/nodes" }, @@ -137,14 +138,17 @@ qx.Class.define("osparc.data.Resources", { url: statics.API + "/projects/{studyId}/nodes/{nodeId}" }, deleteNode: { + useCache: false, method: "DELETE", url: statics.API + "/projects/{studyId}/nodes/{nodeId}" }, addTag: { + useCache: false, method: "PUT", url: statics.API + "/projects/{studyId}/tags/{tagId}" }, removeTag: { + useCache: false, method: "DELETE", url: statics.API + "/projects/{studyId}/tags/{tagId}" } From 3eb5150242a4db33443f32c3ab177f5ca7ac5b62 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 20 Aug 2021 17:09:28 +0200 Subject: [PATCH 134/137] minor --- services/web/client/source/class/osparc/desktop/MainPage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/client/source/class/osparc/desktop/MainPage.js b/services/web/client/source/class/osparc/desktop/MainPage.js index 0754a223c8f..0763de40670 100644 --- a/services/web/client/source/class/osparc/desktop/MainPage.js +++ b/services/web/client/source/class/osparc/desktop/MainPage.js @@ -342,7 +342,7 @@ qx.Class.define("osparc.desktop.MainPage", { return this.__studyEditor; } const studyEditor = new osparc.desktop.StudyEditor(); - studyEditor.addListenerOnce("startSnapshot", e => { + studyEditor.addListener("startSnapshot", e => { const snapshotData = e.getData(); this.__startSnapshot(snapshotData); }, this); From 6dfffff20a328687631f112678647bd27cc98fb0 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Fri, 20 Aug 2021 17:13:10 +0200 Subject: [PATCH 135/137] minor --- services/web/client/source/class/osparc/desktop/MainPage.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/web/client/source/class/osparc/desktop/MainPage.js b/services/web/client/source/class/osparc/desktop/MainPage.js index 0763de40670..7c731df3046 100644 --- a/services/web/client/source/class/osparc/desktop/MainPage.js +++ b/services/web/client/source/class/osparc/desktop/MainPage.js @@ -14,6 +14,9 @@ * Odei Maiz (odeimaiz) ************************************************************************ */ +/** + * @ignore(fetch) + */ /** * Widget managing the layout once the user is logged in. From cc2887122a89547861c7396be6ee89d8a37717b2 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 23 Aug 2021 10:23:44 +0200 Subject: [PATCH 136/137] minor renaming --- .../source/class/osparc/component/form/renderer/PropForm.js | 6 +++--- services/web/client/source/class/osparc/utils/Utils.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/client/source/class/osparc/component/form/renderer/PropForm.js b/services/web/client/source/class/osparc/component/form/renderer/PropForm.js index 8dc41988e3f..5e41f6b9dd3 100644 --- a/services/web/client/source/class/osparc/component/form/renderer/PropForm.js +++ b/services/web/client/source/class/osparc/component/form/renderer/PropForm.js @@ -88,10 +88,10 @@ qx.Class.define("osparc.component.form.renderer.PropForm", { const paramsMenuBtn = this.__getParamsMenuButton(field.key).set({ visibility: "excluded" }); - osparc.utils.Utils.isSweeperEnabled() - .then(isSweeperEnabled => { + osparc.utils.Utils.isDevelopmentPlatform() + .then(areParamsEnabled => { field.bind("visibility", paramsMenuBtn, "visibility", { - converter: visibility => (visibility === "visible" && isSweeperEnabled) ? "visible" : "excluded" + converter: visibility => (visibility === "visible" && areParamsEnabled) ? "visible" : "excluded" }); }); return paramsMenuBtn; diff --git a/services/web/client/source/class/osparc/utils/Utils.js b/services/web/client/source/class/osparc/utils/Utils.js index b0ccad32ba5..0cd46e77270 100644 --- a/services/web/client/source/class/osparc/utils/Utils.js +++ b/services/web/client/source/class/osparc/utils/Utils.js @@ -30,7 +30,7 @@ qx.Class.define("osparc.utils.Utils", { type: "static", statics: { - isSweeperEnabled: function() { + isDevelopmentPlatform: function() { return new Promise((resolve, reject) => { osparc.utils.LibVersions.getPlatformName() .then(platformName => { From f2a0fb7f9c6d0828781405e98df90dade2204716 Mon Sep 17 00:00:00 2001 From: odeimaiz Date: Mon, 23 Aug 2021 10:25:05 +0200 Subject: [PATCH 137/137] clean up --- services/web/client/source/class/osparc/data/model/Node.js | 1 - services/web/client/source/class/osparc/desktop/WorkbenchView.js | 1 - 2 files changed, 2 deletions(-) diff --git a/services/web/client/source/class/osparc/data/model/Node.js b/services/web/client/source/class/osparc/data/model/Node.js index 6f913c121eb..ba8954f1ea7 100644 --- a/services/web/client/source/class/osparc/data/model/Node.js +++ b/services/web/client/source/class/osparc/data/model/Node.js @@ -972,7 +972,6 @@ qx.Class.define("osparc.data.model.Node", { const sizeBytes = (data && ("size_bytes" in data)) ? data["size_bytes"] : 0; this.getPropsForm().retrievedPortData(portKey, true, sizeBytes); } - console.log(data); }, this); updReq.addListener("fail", e => { const { diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index fa10cfed26c..0eab23699cc 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -203,7 +203,6 @@ qx.Class.define("osparc.desktop.WorkbenchView", { }; osparc.data.Resources.fetch("snapshots", "takeSnapshot", params) .then(data => { - console.log(data); workbenchToolbar.evalSnapshotsBtn(); }) .catch(err => osparc.component.message.FlashMessenger.getInstance().logAs(err.message, "ERROR"))