✨ Is765/tags for groups (⚠️ devops) (#3584)

docs/coding-conventions.md

@@ -15,7 +15,7 @@ Have a look at `ESLint`'s configuration files [.eslintrc.json](.eslintrc.json) a
 Have a look at `Pylint`'s configuration file [.pylintrc](.pylintrc).
 
 
-## Posgres
+## Postgres
 
 ### Foreign keys
 

packages/postgres-database/scripts/copy_database_volume.sh

@@ -53,13 +53,13 @@ IFS=$(printf '\n\t')
 
 
 if [ ! -z $folder ]; then
-  #folder mode
+  #from folder to target volume
   ssh $host \
     "tar -cf - $folder " \
     | \
     docker run --rm -i -v "$target":/to alpine ash -c "cd /to ; tar -xpvf - "
 else
-  #docker mode
+  #from docker volume to target volume
   ssh $host \
     "docker run --rm -v $volume:/from alpine ash -c 'cd /from ; tar -cf - . '" \
     | \

packages/postgres-database/scripts/erd/Dockerfile

@@ -24,5 +24,4 @@ RUN pip --no-cache-dir install --upgrade \
 RUN pip install --no-cache-dir \
   pyparsing \
   pydot \
-  eralchemy \
   sqlalchemy_schemadisplay

packages/postgres-database/scripts/erd/main.py

@@ -1,50 +1,47 @@
 #
 # ERD (Entity Relationship Diagram) is used to visualize these relationships
 #
-
+# - Using sqlalchemy_schemadisplay which is maintained by sqlalchemy
+# - Already tried 'eralchemy' but fails with latest version and not maintained anymore
+#
+# SEE https://github.com/sqlalchemy/sqlalchemy/wiki/SchemaDisplay
+# SEE https://github.com/Alexis-benoist/eralchemy
 
 # pylint: disable=wildcard-import
 # pylint: disable=unused-wildcard-import
-#
-import sys
+
 from pathlib import Path
+from typing import Any, Optional
 
 from simcore_postgres_database.models import *  # registers all schemas in metadata
 from simcore_postgres_database.models.base import metadata
+from sqlalchemy_schemadisplay import create_schema_graph
 
-CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
-
-
-def create_with_sqlalchemy_schemadisplay(image_path: Path):
-    # SEE https://github.com/sqlalchemy/sqlalchemy/wiki/SchemaDisplay
 
-    from sqlalchemy_schemadisplay import create_schema_graph
+def create_erd(image_path: Path, tables: Optional[list[str]] = None):
+    """
+    create the pydot graph object by autoloading all tables via a bound metadata object
+    """
 
-    # create the pydot graph object by autoloading all tables via a bound metadata object
-
-    graph = create_schema_graph(
-        metadata=metadata,
+    kwargs: dict[str, Any] = dict(
         show_datatypes=True,  # The image would get nasty big if we'd show the datatypes
         show_indexes=False,  # ditto for indexes
         rankdir="LR",  # From left to right (instead of top to bottom)
         concentrate=False,  # Don't try to join the relation lines together
     )
-    graph.write_svg(str(image_path.with_suffix(".svg")))
-    graph.write_png(str(image_path.with_suffix(".png")))
-
-
-def create_with_eralchemy(image_path: Path):
-    # SEE https://github.com/Alexis-benoist/eralchemy
 
-    from eralchemy import render_er
+    if tables:
+        kwargs["tables"] = [metadata.tables[t] for t in tables]
+    else:
+        kwargs["metadata"] = metadata
 
-    for ext in (".png", ".svg"):
-        render_er(metadata, str(image_path.with_suffix(ext)))
+    graph = create_schema_graph(kwargs)
+    # pylint: disable=no-member
+    graph.write_svg(f'{image_path.with_suffix(".svg")}')
+    graph.write_png(f'{image_path.with_suffix(".png")}')
 
 
 if __name__ == "__main__":
-
-    # FIXME: sqlalchemy_schemadisplay failes with json columns
-    # create_with_sqlalchemy_schemadisplay( output_dir / "postgres-database-models.svg")
-
-    create_with_eralchemy(Path.cwd() / "postgres-database-models.ignore.ext")
+    path = Path("postgres-database-erd-ignore.svg")
+    create_erd(path)
+    print("Created", path)

packages/postgres-database/src/simcore_postgres_database/migration/versions/3aa309471ff8_add_tag_to_groups_table_and_rm_tag_user_.py

@@ -0,0 +1,104 @@
+"""add tag_to_groups table and rm tag.user_id
+
+Revision ID: 3aa309471ff8
+Revises: c3c564121364
+Create Date: 2022-11-17 23:21:49.290958+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3aa309471ff8"
+down_revision = "c3c564121364"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "tags_to_groups",
+        sa.Column("tag_id", sa.BigInteger(), nullable=False),
+        sa.Column("group_id", sa.BigInteger(), nullable=False),
+        sa.Column("read", sa.Boolean(), server_default=sa.text("true"), nullable=False),
+        sa.Column(
+            "write", sa.Boolean(), server_default=sa.text("false"), nullable=False
+        ),
+        sa.Column(
+            "delete", sa.Boolean(), server_default=sa.text("false"), nullable=False
+        ),
+        sa.Column(
+            "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column(
+            "modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["group_id"],
+            ["groups.gid"],
+            name="fk_tag_to_group_group_id",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["tag_id"],
+            ["tags.id"],
+            name="fk_tag_to_group_tag_id",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint("tag_id", "group_id"),
+    )
+
+    # transform every tags.user_id to its PRIMARY group_id in tags_to_groups
+    op.execute(
+        sa.DDL(
+            """
+INSERT INTO tags_to_groups (tag_id, group_id) SELECT tags.id, user_to_groups.gid
+FROM tags JOIN users ON users.id = tags.user_id JOIN user_to_groups ON users.id = user_to_groups.uid JOIN groups ON groups.gid = user_to_groups.gid AND groups.type = 'PRIMARY'
+            """
+        )
+    )
+
+    # set  full access for PRIMARY group_id
+    op.execute(
+        sa.DDL(
+            """
+UPDATE tags_to_groups SET write='True', delete='True', modified=now()
+            """
+        )
+    )
+
+    # Drop old tags.user_id
+    op.drop_constraint("tags_user_id_fkey", "tags", type_="foreignkey")
+    op.drop_column("tags", "user_id")
+
+
+def downgrade():
+    op.add_column(
+        "tags",
+        sa.Column("user_id", sa.BIGINT(), autoincrement=False, default=1),
+    )
+    op.create_foreign_key(
+        "tags_user_id_fkey",
+        "tags",
+        "users",
+        ["user_id"],
+        ["id"],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+    )
+
+    # Sets tags.user_id
+    op.execute(
+        sa.DDL(
+            """
+    UPDATE tags SET user_id=(SELECT users.id
+    FROM users JOIN tags_to_groups ON tags_to_groups.group_id = users.primary_gid
+    WHERE tags.id = tags_to_groups.tag_id)
+            """
+        )
+    )
+    op.alter_column("tags", "user_id", nullable=False)
+
+    op.drop_table("tags_to_groups")

packages/postgres-database/src/simcore_postgres_database/models/_common.py

@@ -0,0 +1,22 @@
+import sqlalchemy as sa
+
+
+def column_created_datetime() -> sa.Column:
+    return sa.Column(
+        "created",
+        sa.DateTime(),
+        nullable=False,
+        server_default=sa.sql.func.now(),
+        doc="Timestamp auto-generated upon creation",
+    )
+
+
+def column_modified_datetime() -> sa.Column:
+    return sa.Column(
+        "modified",
+        sa.DateTime(),
+        nullable=False,
+        server_default=sa.sql.func.now(),
+        onupdate=sa.sql.func.now(),
+        doc="Timestamp with last row update",
+    )

packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py

@@ -1,5 +1,6 @@
 import sqlalchemy as sa
 
+from ._common import column_created_datetime, column_modified_datetime
 from .base import metadata
 
 projects_to_products = sa.Table(
@@ -29,21 +30,8 @@
         nullable=False,
         doc="Products unique name",
     ),
-    # -----
-    sa.Column(
-        "created",
-        sa.DateTime(),
-        nullable=False,
-        server_default=sa.sql.func.now(),
-        doc="Timestamp auto-generated upon creation",
-    ),
-    sa.Column(
-        "modified",
-        sa.DateTime(),
-        nullable=False,
-        server_default=sa.sql.func.now(),
-        onupdate=sa.sql.func.now(),
-        doc="Timestamp with last row update",
-    ),
+    # TIME STAMPS ----
+    column_created_datetime(),
+    column_modified_datetime(),
     sa.UniqueConstraint("project_uuid", "product_name"),
 )