feat(ci): enable database restore for feature branches (#254)

* feat(ci): enable database restore for feature branches * ci(db): reset user after restore * feat(ci): add post-restore script * fix(ci): fix post-restore * fix(ci): fix post-restore * fix(ci): fix post-restore * fix: snaps Co-authored-by: LionelB <[email protected]>
SocialGouv · Jan 14, 2021 · 1a5b9a4 · 1a5b9a4
1 parent c64f916
commit 1a5b9a4
Show file tree

Hide file tree

Showing 6 changed files with 351 additions and 6 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -118,15 +118,18 @@ Restore container:
     - .base_deploy_kosko_stage
   environment:
     name: prod2
+  needs:
+    - job: Create Azure DB (dev)
+  dependencies: null
   variables:
     # storage copy
     SOURCE_CONTAINER: "cdtn"
     SOURCE_SERVER: "dev"
     DESTINATION_CONTAINER: "cdtn-dev"
     DESTINATION_SERVER: "dev"
     # db restore
-    # BACKUP_DB_NAME: "db_${CI_COMMIR_SHORT_SHA}"
-    # BACKUP_DB_OWNER: "user_${CI_COMMIR_SHORT_SHA}"
-    # BACKUP_DB_FILE: "hasura_prod_db.psql.gz"
+    BACKUP_DB_NAME: "db_${CI_COMMIT_SHORT_SHA}"
+    BACKUP_DB_OWNER: "user_${CI_COMMIT_SHORT_SHA}"
+    BACKUP_DB_FILE: "hasura_prod_db.psql.gz"
     # kosko options
     KOSKO_GENERATE_ARGS: --env prod restore
diff --git a/.k8s/__tests__/__snapshots__/kosko generate --env prod restore.ts.snap b/.k8s/__tests__/__snapshots__/kosko generate --env prod restore.ts.snap
@@ -1,7 +1,8 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`kosko generate --prod restore 1`] = `
-"metadata:
+"---
+metadata:
   name: restore-container-8843083e
   namespace: cdtn-admin-secret
 spec:
@@ -103,5 +104,128 @@ spec:
       restartPolicy: Never
 apiVersion: batch/v1
 kind: Job
+---
+data:
+  post-restore.sql: |
+    TRUNCATE TABLE \\"auth\\".\\"users\\" CASCADE;
+
+    WITH admin_row AS (
+    INSERT INTO auth.users (email, PASSWORD, name, default_role, active)
+        VALUES ('[email protected]', '$argon2i$v=19$m=4096,t=3,p=1$n9eoWSv+5sCgc7SjB5hLig$iBQ7NzrHHLkJSku/dCetNs+n/JI1CMdkWaoZsUekLU8', 'Administrateur', 'admin', TRUE)
+      RETURNING
+        id, default_role)
+      INSERT INTO auth.user_roles (ROLE, user_id)
+      SELECT
+        default_role,
+        id
+      FROM
+        admin_row;
+
+    WITH admin_row AS (
+    INSERT INTO auth.users (email, PASSWORD, name, default_role, active)
+        VALUES ('[email protected]', '$argon2i$v=19$m=4096,t=3,p=1$PqKPf9cxunVLLtEcINHhWQ$CwHKhk71fc8LGp6BWbcFPzQ2ftOiHa7vUkp1eAqVHSM', 'Utilisateur', 'user', TRUE)
+      RETURNING
+        id, default_role)
+      INSERT INTO auth.user_roles (ROLE, user_id)
+      SELECT
+        default_role,
+        id
+      FROM
+        admin_row;
+metadata:
+  name: post-restore-script-configmap-8843083e
+  namespace: cdtn-admin-secret
+apiVersion: v1
+kind: ConfigMap
+---
+metadata:
+  name: restore-db-8843083e
+  namespace: cdtn-admin-secret
+spec:
+  backoffLimit: 0
+  template:
+    metadata: {}
+    spec:
+      containers:
+        - command:
+            - sh
+            - '-c'
+            - >
+
+
+              echo \\"starting restore into $PGHOST/$PGDATABASE\\"
+
+
+              [ ! -z $PGDATABASE ] || (echo \\"No PGDATABASE\\"; exit 1)
+
+              [ ! -z $PGHOST ] || (echo \\"No PGHOST\\"; exit 1)
+
+              [ ! -z $PGUSER ] || (echo \\"No PGUSER\\"; exit 1)
+
+              [ ! -z $PGPASSWORD ] || (echo \\"No PGPASSWORD\\"; exit 1)
+
+              [ ! -z $OWNER ] || (echo \\"No OWNER\\"; exit 1)
+
+
+              # get latest backup folder
+
+              LATEST=$(ls -1Fr /mnt/data | head -n 1);
+
+              DUMP=\\"/mnt/data/\${LATEST}\${FILE}\\"
+
+              echo \\"Restore \${DUMP} into \${PGDATABASE}\\";
+
+
+              pg_isready;
+
+
+              pg_restore   --dbname \${PGDATABASE}   --clean --if-exists  
+              --no-owner   --role \${OWNER}   --no-acl   --verbose   \${DUMP};
+
+
+              psql -v ON_ERROR_STOP=1 \${PGDATABASE} -c \\"ALTER SCHEMA public
+              owner to \${OWNER};\\"
+
+
+              [ -f \\"/mnt/scripts/post-restore.sql\\" ] && psql -v ON_ERROR_STOP=1
+              -a < /mnt/scripts/post-restore.sql
+          env:
+            - name: PGDATABASE
+              value: some-database
+            - name: OWNER
+              value: some-owner
+            - name: FILE
+              value: some-backup.sql.gz
+          envFrom:
+            - secretRef:
+                name: azure-pg-admin-user-dev
+          image: >-
+            registry.gitlab.factory.social.gouv.fr/socialgouv/docker/azure-db:2.6.1
+          imagePullPolicy: IfNotPresent
+          name: restore-db
+          resources:
+            limits:
+              cpu: 300m
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 64Mi
+          volumeMounts:
+            - mountPath: /mnt/data
+              name: backups
+            - mountPath: /mnt/scripts
+              name: scripts
+      restartPolicy: OnFailure
+      volumes:
+        - azureFile:
+            readOnly: true
+            secretName: azure-cdtnadminprod-volume
+            shareName: cdtn-admin-backup-restore
+          name: backups
+        - configMap:
+            name: post-restore-script-configmap-8843083e
+          name: scripts
+apiVersion: batch/v1
+kind: Job
 "
 `;
diff --git a/.k8s/components/restore/db.ts b/.k8s/components/restore/db.ts
@@ -1,8 +1,10 @@
+import fs from "fs";
+import path from "path";
 import env from "@kosko/env";
 import { ok } from "assert";
 
 import { EnvVar } from "kubernetes-models/v1/EnvVar";
-import { restoreDbJob } from "@socialgouv/kosko-charts/components/azure-pg/restore-db.job";
+import { restoreDbJob } from "../../restore-db.job";
 
 ok(process.env.BACKUP_DB_NAME);
 ok(process.env.BACKUP_DB_OWNER);
@@ -24,6 +26,9 @@ const manifests = restoreDbJob({
       value: process.env.BACKUP_DB_FILE,
     }),
   ],
+  postRestoreScript: fs
+    .readFileSync(path.join(__dirname, "./post-restore.sql"))
+    .toString(),
 });
 
 export default manifests;
diff --git a/.k8s/components/restore/index.ts b/.k8s/components/restore/index.ts
@@ -1,4 +1,4 @@
 import containerJob from "./container";
 import dbJob from "./db";
 
-export default [containerJob /*, dbJob*/];
+export default [containerJob, dbJob];
diff --git a/.k8s/components/restore/post-restore.sql b/.k8s/components/restore/post-restore.sql
@@ -0,0 +1,25 @@
+TRUNCATE TABLE "auth"."users" CASCADE;
+
+WITH admin_row AS (
+INSERT INTO auth.users (email, PASSWORD, name, default_role, active)
+    VALUES ('[email protected]', '$argon2i$v=19$m=4096,t=3,p=1$n9eoWSv+5sCgc7SjB5hLig$iBQ7NzrHHLkJSku/dCetNs+n/JI1CMdkWaoZsUekLU8', 'Administrateur', 'admin', TRUE)
+  RETURNING
+    id, default_role)
+  INSERT INTO auth.user_roles (ROLE, user_id)
+  SELECT
+    default_role,
+    id
+  FROM
+    admin_row;
+
+WITH admin_row AS (
+INSERT INTO auth.users (email, PASSWORD, name, default_role, active)
+    VALUES ('[email protected]', '$argon2i$v=19$m=4096,t=3,p=1$PqKPf9cxunVLLtEcINHhWQ$CwHKhk71fc8LGp6BWbcFPzQ2ftOiHa7vUkp1eAqVHSM', 'Utilisateur', 'user', TRUE)
+  RETURNING
+    id, default_role)
+  INSERT INTO auth.user_roles (ROLE, user_id)
+  SELECT
+    default_role,
+    id
+  FROM
+    admin_row;