✨ project version control (database only) (#2519)

Implementing metamodeling #2392. This PR address only the database layer

✨Introduces a new version control system for projects (or eventually any table's row) based on git internals: ``simcore_postgres_database.models.projects_version_control``
⚗️ ``simcore_postgres_database.utils_aiopg_orm`` is a simple ORM to wrap aiopg/sqlalchemy queries
🔨 make auto-doc uses now containers and does not require having graphviz installed

.dockerignore

@@ -19,3 +19,12 @@ ops/
 
 # coverage files
 .coverage*
+
+# pytest-fixture-tools output
+artifacts
+
+# dask artifacts in devel mode
+dask*-space/
+
+# produced when mounting volumes on docker and pip installing
+.local/

.gitignore

@@ -166,3 +166,6 @@ artifacts
 
 # dask artifacts in devel mode
 dask*-space/
+
+# produced when mounting volumes on docker and pip installing
+.local/

packages/postgres-database/Makefile

@@ -102,8 +102,5 @@ down-pg down-prod: $(docker-compose-configs) ## stops pg server
 
 
 .PHONY: auto-doc
-auto-doc: install-dev ## Creates entity relationship diagram (ERD) defined under ``simcore_postgres_database.models``
-	# installing doc dependencies (install-doc)
-	pip install eralchemy
-	# running script
-	python scripts/create_erd.py
+auto-doc: ## Creates entity relationship diagram (ERD) defined under ``simcore_postgres_database.models``
+	$(MAKE) --directory=scripts/erd run

packages/postgres-database/scripts/erd/Dockerfile

@@ -0,0 +1,28 @@
+ARG PYTHON_VERSION="3.8.10"
+FROM python:${PYTHON_VERSION}-slim-buster as base
+
+RUN apt-get update \
+  && apt-get -y install --no-install-recommends\
+  make \
+  libc-dev \
+  graphviz-dev \
+  git \
+  gcc \
+  gawk \
+  graphviz \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get clean
+
+
+RUN pip --no-cache-dir install --upgrade \
+  pip~=21.2.3  \
+  wheel \
+  setuptools
+
+
+# devenv
+RUN pip install --no-cache-dir \
+    pyparsing \
+    pydot \
+    eralchemy \
+    sqlalchemy_schemadisplay

packages/postgres-database/scripts/erd/Makefile

@@ -0,0 +1,62 @@
+#
+# ERD (Entity Relationship Diagram) is used to visualize these relationships
+#
+.DEFAULT_GOAL := help
+
+PYTHON_VERSION=3.8.10
+
+# locations
+REPODIR := $(shell git rev-parse --show-toplevel)
+PACKAGES_DIR := $(abspath $(REPODIR)/packages)
+SERVICES_DIR := $(abspath $(REPODIR)/services)
+PG_DIR := $(abspath $(PACKAGES_DIR)/postgres-database)
+
+# tools
+MAKE_C := $(MAKE) --directory
+
+
+IMAGE_NAME:=local/postgres-database-scripts-erd:${PYTHON_VERSION}
+
+# SEE https://medium.com/faun/set-current-host-user-for-docker-container-4e521cef9ffc
+.PHONY: build
+build build-nc: ## builds tooling image ${IMAGE_NAME}
+	docker build $(if $(findstring -nc,$@),--no-cache,) \
+		--build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
+		--tag ${IMAGE_NAME} .
+
+
+.PHONY: shell
+shell: build ## Opens shell in ${IMAGE_NAME}
+	docker run -it \
+		--workdir="/home/$(USER)" \
+    --volume="/etc/group:/etc/group:ro" \
+    --volume="/etc/passwd:/etc/passwd:ro" \
+		--volume="/etc/shadow:/etc/shadow:ro" \
+		--volume=$(PG_DIR):/home/$(USER) \
+		--user=$(shell id -u):$(shell id -g) \
+		--entrypoint=/bin/bash \
+		${IMAGE_NAME}
+
+
+.PHONY: run
+run: build ## Runs upgrade in a container [WARNING! UNDER DEV. USE CAREFULY]
+	docker run -it \
+		--workdir="/home/$(USER)" \
+    --volume="/etc/group:/etc/group:ro" \
+    --volume="/etc/passwd:/etc/passwd:ro" \
+    --volume="/etc/shadow:/etc/shadow:ro" \
+		--volume=$(PG_DIR):/home/$(USER) \
+		--user=$(shell id -u):$(shell id -g) \
+		--entrypoint=/bin/bash \
+		${IMAGE_NAME} \
+		-c "pip install -e .; python scripts/erd/main.py"
+
+
+
+.PHONY: help
+# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+help: ## this colorful help
+	@echo "Recipes for '$(notdir $(CURDIR))':"
+	@echo ""
+	@awk --posix 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@echo ""

packages/postgres-database/scripts/erd/main.py

@@ -0,0 +1,50 @@
+#
+# ERD (Entity Relationship Diagram) is used to visualize these relationships
+#
+
+
+# pylint: disable=wildcard-import
+# pylint: disable=unused-wildcard-import
+#
+import sys
+from pathlib import Path
+
+from simcore_postgres_database.models import *  # registers all schemas in metadata
+from simcore_postgres_database.models.base import metadata
+
+CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
+
+
+def create_with_sqlalchemy_schemadisplay(svg_path: Path):
+    # SEE https://github.com/sqlalchemy/sqlalchemy/wiki/SchemaDisplay
+
+    from sqlalchemy_schemadisplay import create_schema_graph
+
+    # create the pydot graph object by autoloading all tables via a bound metadata object
+
+    graph = create_schema_graph(
+        metadata=metadata,
+        show_datatypes=True,  # The image would get nasty big if we'd show the datatypes
+        show_indexes=False,  # ditto for indexes
+        rankdir="LR",  # From left to right (instead of top to bottom)
+        concentrate=False,  # Don't try to join the relation lines together
+    )
+    graph.write_svg(str(svg_path))  # write out the file
+
+
+def create_with_eralchemy(svg_path: Path):
+    # SEE https://github.com/Alexis-benoist/eralchemy
+    from eralchemy import render_er
+
+    render_er(metadata, str(svg_path))
+
+
+if __name__ == "__main__":
+
+    output_dir = (CURRENT_DIR / "../../doc/img").resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # FIXME: sqlalchemy_schemadisplay failes with json columns
+    # create_with_sqlalchemy_schemadisplay( output_dir / "postgres-database-models.svg")
+
+    create_with_eralchemy(output_dir / "postgres-database-models.svg")

packages/postgres-database/src/simcore_postgres_database/migration/versions/0208f6b32f32_adds_version_control_tables.py

@@ -0,0 +1,170 @@
+"""Adds version control tables
+
+Revision ID: 0208f6b32f32
+Revises: d10c53a5bea6
+Create Date: 2021-09-06 14:19:42.599645+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0208f6b32f32"
+down_revision = "d10c53a5bea6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "projects_vc_snapshots",
+        sa.Column("checksum", sa.String(), nullable=False),
+        sa.Column(
+            "content",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default=sa.text("'{}'::jsonb"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("checksum"),
+    )
+    op.create_table(
+        "projects_vc_repos",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("project_uuid", sa.String(), nullable=False),
+        sa.Column("project_checksum", sa.String(), nullable=True),
+        sa.Column(
+            "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column(
+            "modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_uuid"],
+            ["projects.uuid"],
+            name="fk_projects_vc_repos_project_uuid",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("project_uuid"),
+    )
+    op.create_table(
+        "projects_vc_commits",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("repo_id", sa.BigInteger(), nullable=False),
+        sa.Column("parent_commit_id", sa.BigInteger(), nullable=True),
+        sa.Column("snapshot_checksum", sa.String(), nullable=False),
+        sa.Column("message", sa.String(), nullable=True),
+        sa.Column(
+            "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_commit_id"],
+            ["projects_vc_commits.id"],
+            name="fk_projects_vc_commits_parent_commit_id",
+            onupdate="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["repo_id"],
+            ["projects_vc_repos.id"],
+            name="fk_projects_vc_commits_repo_id",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["snapshot_checksum"],
+            ["projects_vc_snapshots.checksum"],
+            name="fk_projects_vc_commits_snapshot_checksum",
+            ondelete="RESTRICT",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "projects_vc_branches",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("repo_id", sa.BigInteger(), nullable=False),
+        sa.Column("head_commit_id", sa.BigInteger(), nullable=True),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column(
+            "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column(
+            "modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["head_commit_id"],
+            ["projects_vc_commits.id"],
+            name="fk_projects_vc_branches_head_commit_id",
+            ondelete="RESTRICT",
+        ),
+        sa.ForeignKeyConstraint(
+            ["repo_id"],
+            ["projects_vc_repos.id"],
+            name="projects_vc_branches_repo_id",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "repo_id", name="repo_branch_uniqueness"),
+    )
+    op.create_table(
+        "projects_vc_tags",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("repo_id", sa.BigInteger(), nullable=False),
+        sa.Column("commit_id", sa.BigInteger(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("message", sa.String(), nullable=True),
+        sa.Column("hidden", sa.Boolean(), nullable=True),
+        sa.Column(
+            "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column(
+            "modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["commit_id"],
+            ["projects_vc_commits.id"],
+            name="fk_projects_vc_tags_commit_id",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["repo_id"],
+            ["projects_vc_repos.id"],
+            name="fk_projects_vc_tags_repo_id",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "repo_id", name="repo_tag_uniqueness"),
+    )
+    op.create_table(
+        "projects_vc_heads",
+        sa.Column("repo_id", sa.BigInteger(), nullable=False),
+        sa.Column("head_branch_id", sa.BigInteger(), nullable=True),
+        sa.Column(
+            "modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["head_branch_id"],
+            ["projects_vc_branches.id"],
+            name="fk_projects_vc_heads_head_branch_id",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["repo_id"],
+            ["projects_vc_repos.id"],
+            name="projects_vc_branches_repo_id",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("repo_id"),
+        sa.UniqueConstraint("head_branch_id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("projects_vc_heads")
+    op.drop_table("projects_vc_tags")
+    op.drop_table("projects_vc_branches")
+    op.drop_table("projects_vc_commits")
+    op.drop_table("projects_vc_repos")
+    op.drop_table("projects_vc_snapshots")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/projects.py

@@ -4,16 +4,13 @@
 
 """
 import enum
-import logging
 
 import sqlalchemy as sa
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.sql import func
 
 from .base import metadata
 
-log = logging.getLogger(__name__)
-
 
 class ProjectType(enum.Enum):
     """