From 63ed71bc84688e23de2c83b5f10f0996650826a3 Mon Sep 17 00:00:00 2001 From: LionelB Date: Thu, 26 Nov 2020 16:09:30 +0100 Subject: [PATCH 1/3] update --- targets/hasura/Dockerfile | 2 +- targets/hasura/config.yaml | 1 + .../migrations/1606401836569_version/down.sql | 1 + .../migrations/1606401836569_version/up.sql | 11 ++++++ targets/ingester/src/cli.js | 34 ++++++++++++++++++- 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 targets/hasura/migrations/1606401836569_version/down.sql create mode 100644 targets/hasura/migrations/1606401836569_version/up.sql diff --git a/targets/hasura/Dockerfile b/targets/hasura/Dockerfile index 05a143265..89359b848 100644 --- a/targets/hasura/Dockerfile +++ b/targets/hasura/Dockerfile @@ -1,4 +1,4 @@ -FROM hasura/graphql-engine:v1.3.2.cli-migrations-v2 +FROM hasura/graphql-engine:v1.3.3.cli-migrations-v2 ENV HASURA_GRAPHQL_ENABLE_TELEMETRY false COPY ./migrations /hasura-migrations COPY ./metadata /hasura-metadata diff --git a/targets/hasura/config.yaml b/targets/hasura/config.yaml index 6ddf586e5..c7dd00812 100644 --- a/targets/hasura/config.yaml +++ b/targets/hasura/config.yaml @@ -1,5 +1,6 @@ version: 2 endpoint: http://localhost:8080 +enable_telemetry: false metadata_directory: metadata actions: kind: synchronous diff --git a/targets/hasura/migrations/1606401836569_version/down.sql b/targets/hasura/migrations/1606401836569_version/down.sql new file mode 100644 index 000000000..4d589d08e --- /dev/null +++ b/targets/hasura/migrations/1606401836569_version/down.sql @@ -0,0 +1 @@ +DROP TABLE "public"."document_version"; diff --git a/targets/hasura/migrations/1606401836569_version/up.sql b/targets/hasura/migrations/1606401836569_version/up.sql new file mode 100644 index 000000000..4974096f1 --- /dev/null +++ b/targets/hasura/migrations/1606401836569_version/up.sql @@ -0,0 +1,11 @@ +CREATE TABLE "public"."document_version" ( "repository" text NOT NULL, "version" text NOT NULL, "created_at" timestamptz NOT NULL DEFAULT now(), "updated_at" timestamptz NOT NULL DEFAULT now(), PRIMARY KEY ("repository") +); + +COMMENT ON TABLE "public"."document_version" IS E'tracks documents git version by repository'; + +CREATE TRIGGER "set_public_document_version_updated_at" + BEFORE UPDATE ON "public"."document_version" + FOR EACH ROW + EXECUTE PROCEDURE trigger_set_timestamp (); + +COMMENT ON TRIGGER "set_public_document_version_updated_at" ON "public"."document_version" IS 'trigger to set value of column "updated_at" to current timestamp on row update'; diff --git a/targets/ingester/src/cli.js b/targets/ingester/src/cli.js index 0592c93eb..398b78761 100644 --- a/targets/ingester/src/cli.js +++ b/targets/ingester/src/cli.js @@ -42,6 +42,17 @@ mutation insert_documents($documents: [documents_insert_input!]!) { } `; +const insertDocumentsVersionMutation = ` +mutation insert_document_version($object:document_version_insert_input!) { + version: insert_document_version_one(object: $object, on_conflict: { + constraint: document_version_pkey, + update_columns: version + }) { + repository, version + } +} +`; + /** * * @param {string} pkgName @@ -119,6 +130,7 @@ async function getPackage(pkgName, pkgVersion = "latest") { console.debug(`download package ${pkgName}@${latest}`); await download(pkgName, url); } + return latest; } /** @@ -163,10 +175,29 @@ async function initDocAvailabity(source) { } return result.data.documents.affected_rows; } +/** + * + * @param {string} repository + * @param {string} version + */ +async function updateVersion(repository, version) { + const result = await client + .mutation(insertDocumentsVersionMutation, { + object: { repository, version }, + }) + .toPromise(); + if (result.error) { + console.error(result.error); + throw new Error(`error updating document_version ${repository}@${version}`); + } + return result.data.version; +} async function main() { + /** @type {{[key:string]:string}} */ + const pkgVersions = {}; for (const [pkgName] of dataPackages) { - await getPackage(pkgName); + pkgVersions[pkgName] = await getPackage(pkgName); } if (args.dryRun) { console.log("dry-run mode"); @@ -186,6 +217,7 @@ async function main() { const chunks = chunk(documents, 80); const inserts = await batchPromises(chunks, insertDocuments, 15); ids = ids.concat(inserts); + await updateVersion(pkgName, pkgVersions[pkgName]); } return ids; } From 45a33da01adf36218a6cffe82b474c3fc2c00339 Mon Sep 17 00:00:00 2001 From: LionelB Date: Thu, 26 Nov 2020 17:15:21 +0100 Subject: [PATCH 2/3] update tables.yml --- targets/hasura/metadata/tables.yaml | 884 ++++++++++++++-------------- 1 file changed, 452 insertions(+), 432 deletions(-) diff --git a/targets/hasura/metadata/tables.yaml b/targets/hasura/metadata/tables.yaml index c27d7a3fe..060c57193 100644 --- a/targets/hasura/metadata/tables.yaml +++ b/targets/hasura/metadata/tables.yaml @@ -2,512 +2,532 @@ schema: auth name: refresh_tokens object_relationships: - - name: user - using: - foreign_key_constraint_on: user_id + - name: user + using: + foreign_key_constraint_on: user_id - table: schema: auth name: user_roles object_relationships: - - name: roleByRole - using: - foreign_key_constraint_on: role - - name: user - using: - foreign_key_constraint_on: user_id + - name: roleByRole + using: + foreign_key_constraint_on: role + - name: user + using: + foreign_key_constraint_on: user_id select_permissions: - - role: user - permission: - columns: - - id - - role - filter: - id: - _eq: X-Hasura-User-Id + - role: user + permission: + columns: + - id + - role + filter: + id: + _eq: X-Hasura-User-Id update_permissions: - - role: user - permission: - columns: - - role - filter: - id: - _eq: X-Hasura-User-Id - check: null + - role: user + permission: + columns: + - role + filter: + id: + _eq: X-Hasura-User-Id + check: null - table: schema: auth name: users object_relationships: - - name: role - using: - foreign_key_constraint_on: default_role + - name: role + using: + foreign_key_constraint_on: default_role array_relationships: - - name: refresh_tokens - using: - foreign_key_constraint_on: - column: user_id - table: - schema: auth - name: refresh_tokens - - name: user_roles - using: - foreign_key_constraint_on: - column: user_id - table: - schema: auth - name: user_roles + - name: refresh_tokens + using: + foreign_key_constraint_on: + column: user_id + table: + schema: auth + name: refresh_tokens + - name: user_roles + using: + foreign_key_constraint_on: + column: user_id + table: + schema: auth + name: user_roles select_permissions: - - role: user - permission: - columns: - - active - - created_at - - default_role - - email - - id - - name - - secret_token - - secret_token_expires_at - - updated_at - filter: - id: - _eq: X-Hasura-User-Id + - role: user + permission: + columns: + - active + - created_at + - default_role + - email + - id + - name + - secret_token + - secret_token_expires_at + - updated_at + filter: + id: + _eq: X-Hasura-User-Id update_permissions: - - role: user - permission: - columns: - - email - - name - filter: - id: - _eq: X-Hasura-User-Id - check: null - event_triggers: - - name: account_email - definition: - enable_manual: false - insert: - columns: '*' - update: + - role: user + permission: columns: - - secret_token - retry_conf: - num_retries: 1 - interval_sec: 10 - timeout_sec: 60 - webhook_from_env: ACCOUNT_EMAIL_WEBHOOK_URL - headers: - - name: email-secret - value_from_env: ACCOUNT_EMAIL_SECRET + - email + - name + filter: + id: + _eq: X-Hasura-User-Id + check: null + event_triggers: + - name: account_email + definition: + enable_manual: false + insert: + columns: "*" + update: + columns: + - secret_token + retry_conf: + num_retries: 1 + interval_sec: 10 + timeout_sec: 60 + webhook_from_env: ACCOUNT_EMAIL_WEBHOOK_URL + headers: + - name: email-secret + value_from_env: ACCOUNT_EMAIL_SECRET - table: schema: public name: alert_notes object_relationships: - - name: user - using: - foreign_key_constraint_on: user_id + - name: user + using: + foreign_key_constraint_on: user_id - table: schema: public name: alert_status array_relationships: - - name: alerts - using: - foreign_key_constraint_on: - column: status - table: - schema: public - name: alerts + - name: alerts + using: + foreign_key_constraint_on: + column: status + table: + schema: public + name: alerts select_permissions: - - role: user - permission: - columns: - - name - filter: {} - allow_aggregations: true + - role: user + permission: + columns: + - name + filter: {} + allow_aggregations: true update_permissions: - - role: user - permission: - columns: - - name - filter: {} - check: {} + - role: user + permission: + columns: + - name + filter: {} + check: {} - table: schema: public name: alerts object_relationships: - - name: alert_status - using: - foreign_key_constraint_on: status - - name: source - using: - foreign_key_constraint_on: repository + - name: alert_status + using: + foreign_key_constraint_on: status + - name: source + using: + foreign_key_constraint_on: repository array_relationships: - - name: alert_notes - using: - foreign_key_constraint_on: - column: alert_id - table: - schema: public - name: alert_notes + - name: alert_notes + using: + foreign_key_constraint_on: + column: alert_id + table: + schema: public + name: alert_notes select_permissions: - - role: user - permission: - columns: - - id - - info - - status - - repository - - ref - - changes - - created_at - - updated_at - filter: {} - allow_aggregations: true + - role: user + permission: + columns: + - id + - info + - status + - repository + - ref + - changes + - created_at + - updated_at + filter: {} + allow_aggregations: true update_permissions: - - role: user - permission: - columns: - - id - - info - - status - - repository - - ref - - changes - - created_at - - updated_at - filter: {} - check: {} + - role: user + permission: + columns: + - id + - info + - status + - repository + - ref + - changes + - created_at + - updated_at + filter: {} + check: {} - table: schema: public name: document_relations object_relationships: - - name: a - using: - foreign_key_constraint_on: document_a - - name: b - using: - foreign_key_constraint_on: document_b + - name: a + using: + foreign_key_constraint_on: document_a + - name: b + using: + foreign_key_constraint_on: document_b insert_permissions: - - role: user - permission: - check: {} - columns: - - id - - document_a - - document_b - - type - - data - backend_only: false + - role: user + permission: + check: {} + columns: + - id + - document_a + - document_b + - type + - data + backend_only: false select_permissions: - - role: public - permission: - columns: - - id - - document_a - - document_b - - type - - data - filter: {} - allow_aggregations: true - - role: user - permission: - columns: - - id - - document_a - - document_b - - type - - data - filter: {} - allow_aggregations: true + - role: public + permission: + columns: + - id + - document_a + - document_b + - type + - data + filter: {} + allow_aggregations: true + - role: user + permission: + columns: + - id + - document_a + - document_b + - type + - data + filter: {} + allow_aggregations: true update_permissions: - - role: user - permission: - columns: - - data - - document_a - - document_b - - type - - id - filter: {} - check: {} + - role: user + permission: + columns: + - data + - document_a + - document_b + - type + - id + filter: {} + check: {} delete_permissions: - - role: user - permission: - filter: {} + - role: user + permission: + filter: {} - table: schema: public name: documents array_relationships: - - name: relation_a - using: - foreign_key_constraint_on: - column: document_a - table: - schema: public - name: document_relations - - name: relation_b - using: - foreign_key_constraint_on: - column: document_b - table: - schema: public - name: document_relations + - name: relation_a + using: + foreign_key_constraint_on: + column: document_a + table: + schema: public + name: document_relations + - name: relation_b + using: + foreign_key_constraint_on: + column: document_b + table: + schema: public + name: document_relations insert_permissions: - - role: user - permission: - check: {} - columns: - - cdtn_id - - created_at - - document - - initial_id - - is_available - - is_published - - is_searchable - - meta_description - - slug - - source - - text - - title - - updated_at - backend_only: false + - role: user + permission: + check: {} + columns: + - cdtn_id + - created_at + - document + - initial_id + - is_available + - is_published + - is_searchable + - meta_description + - slug + - source + - text + - title + - updated_at + backend_only: false select_permissions: - - role: public - permission: - columns: - - cdtn_id - - initial_id - - title - - meta_description - - source - - slug - - text - - document - - is_published - - is_searchable - - created_at - - updated_at - - is_available - filter: {} - allow_aggregations: true - - role: user - permission: - columns: - - cdtn_id - - created_at - - document - - initial_id - - is_available - - is_published - - is_searchable - - meta_description - - slug - - source - - text - - title - - updated_at - filter: {} - allow_aggregations: true + - role: public + permission: + columns: + - cdtn_id + - initial_id + - title + - meta_description + - source + - slug + - text + - document + - is_published + - is_searchable + - created_at + - updated_at + - is_available + filter: {} + allow_aggregations: true + - role: user + permission: + columns: + - cdtn_id + - created_at + - document + - initial_id + - is_available + - is_published + - is_searchable + - meta_description + - slug + - source + - text + - title + - updated_at + filter: {} + allow_aggregations: true update_permissions: - - role: user - permission: - columns: - - cdtn_id - - created_at - - document - - initial_id - - is_available - - is_published - - is_searchable - - meta_description - - slug - - source - - text - - title - - updated_at - filter: {} - check: {} + - role: user + permission: + columns: + - cdtn_id + - created_at + - document + - initial_id + - is_available + - is_published + - is_searchable + - meta_description + - slug + - source + - text + - title + - updated_at + filter: {} + check: {} delete_permissions: - - role: user - permission: - filter: {} + - role: user + permission: + filter: {} event_triggers: - - name: publication - definition: - enable_manual: false - update: + - name: publication + definition: + enable_manual: false + update: + columns: + - is_published + retry_conf: + num_retries: 2 + interval_sec: 10 + timeout_sec: 60 + webhook_from_env: PUBLICATION_WEBHOOK_URL + headers: + - name: publication-secret + value_from_env: PUBLICATION_SECRET +- table: + schema: public + name: document_version + select_permissions: + - role: public + permission: columns: - - is_published - retry_conf: - num_retries: 2 - interval_sec: 10 - timeout_sec: 60 - webhook_from_env: PUBLICATION_WEBHOOK_URL - headers: - - name: publication-secret - value_from_env: PUBLICATION_SECRET + - repository + - version + - created_at + - updated_at + filter: {} + - role: user + permission: + columns: + - repository + - version + - created_at + - updated_at + filter: {} - table: schema: public name: glossary insert_permissions: - - role: user - permission: - check: {} - columns: - - abbreviations - - references - - variants - - definition - - slug - - term - - created_at - - updated_at - - id - backend_only: false + - role: user + permission: + check: {} + columns: + - abbreviations + - references + - variants + - definition + - slug + - term + - created_at + - updated_at + - id + backend_only: false select_permissions: - - role: public - permission: - columns: - - abbreviations - - references - - variants - - definition - - slug - - term - - created_at - - updated_at - - id - filter: {} - - role: user - permission: - columns: - - abbreviations - - references - - variants - - definition - - slug - - term - - created_at - - updated_at - - id - filter: {} + - role: public + permission: + columns: + - abbreviations + - references + - variants + - definition + - slug + - term + - created_at + - updated_at + - id + filter: {} + - role: user + permission: + columns: + - abbreviations + - references + - variants + - definition + - slug + - term + - created_at + - updated_at + - id + filter: {} update_permissions: - - role: user - permission: - columns: - - abbreviations - - references - - variants - - definition - - slug - - term - - created_at - - updated_at - - id - filter: {} - check: null + - role: user + permission: + columns: + - abbreviations + - references + - variants + - definition + - slug + - term + - created_at + - updated_at + - id + filter: {} + check: null delete_permissions: - - role: user - permission: - filter: {} + - role: user + permission: + filter: {} - table: schema: public name: kali_blocks insert_permissions: - - role: user - permission: - check: {} - columns: - - id - - title - - idcc - - blocks - backend_only: false + - role: user + permission: + check: {} + columns: + - id + - title + - idcc + - blocks + backend_only: false select_permissions: - - role: public - permission: - columns: - - id - - title - - idcc - - blocks - filter: {} - - role: user - permission: - columns: - - idcc - - blocks - - id - - title - filter: {} + - role: public + permission: + columns: + - id + - title + - idcc + - blocks + filter: {} + - role: user + permission: + columns: + - idcc + - blocks + - id + - title + filter: {} update_permissions: - - role: user - permission: - columns: - - idcc - - blocks - - id - - title - filter: {} - check: {} + - role: user + permission: + columns: + - idcc + - blocks + - id + - title + filter: {} + check: {} delete_permissions: - - role: user - permission: - filter: {} + - role: user + permission: + filter: {} - table: schema: public name: roles array_relationships: - - name: user_roles - using: - foreign_key_constraint_on: - column: role - table: - schema: auth - name: user_roles - - name: users - using: - foreign_key_constraint_on: - column: default_role - table: - schema: auth - name: users + - name: user_roles + using: + foreign_key_constraint_on: + column: role + table: + schema: auth + name: user_roles + - name: users + using: + foreign_key_constraint_on: + column: default_role + table: + schema: auth + name: users select_permissions: - - role: user - permission: - columns: - - role - filter: {} + - role: user + permission: + columns: + - role + filter: {} - table: schema: public name: sources array_relationships: - - name: alerts - using: - foreign_key_constraint_on: - column: repository - table: - schema: public - name: alerts + - name: alerts + using: + foreign_key_constraint_on: + column: repository + table: + schema: public + name: alerts select_permissions: - - role: user - permission: - columns: - - repository - - label - - tag - - created_at - filter: {} - allow_aggregations: true + - role: user + permission: + columns: + - repository + - label + - tag + - created_at + filter: {} + allow_aggregations: true update_permissions: - - role: user - permission: - columns: - - repository - - label - - tag - - created_at - filter: {} - check: {} + - role: user + permission: + columns: + - repository + - label + - tag + - created_at + filter: {} + check: {} - table: schema: v1 name: fiches_travail_data_alerts From a2ca533bb6f4939b4217f784704f90c88074f839 Mon Sep 17 00:00:00 2001 From: LionelB Date: Thu, 26 Nov 2020 17:39:31 +0100 Subject: [PATCH 3/3] fix: rename --- targets/hasura/metadata/tables.yaml | 2 +- .../hasura/migrations/1606401836569_version/down.sql | 2 +- .../hasura/migrations/1606401836569_version/up.sql | 10 +++++----- targets/ingester/src/cli.js | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/targets/hasura/metadata/tables.yaml b/targets/hasura/metadata/tables.yaml index 060c57193..d2b39aad2 100644 --- a/targets/hasura/metadata/tables.yaml +++ b/targets/hasura/metadata/tables.yaml @@ -347,7 +347,7 @@ value_from_env: PUBLICATION_SECRET - table: schema: public - name: document_version + name: package_version select_permissions: - role: public permission: diff --git a/targets/hasura/migrations/1606401836569_version/down.sql b/targets/hasura/migrations/1606401836569_version/down.sql index 4d589d08e..1ea5b44f3 100644 --- a/targets/hasura/migrations/1606401836569_version/down.sql +++ b/targets/hasura/migrations/1606401836569_version/down.sql @@ -1 +1 @@ -DROP TABLE "public"."document_version"; +DROP TABLE "public"."package_version"; diff --git a/targets/hasura/migrations/1606401836569_version/up.sql b/targets/hasura/migrations/1606401836569_version/up.sql index 4974096f1..88eaa1ab7 100644 --- a/targets/hasura/migrations/1606401836569_version/up.sql +++ b/targets/hasura/migrations/1606401836569_version/up.sql @@ -1,11 +1,11 @@ -CREATE TABLE "public"."document_version" ( "repository" text NOT NULL, "version" text NOT NULL, "created_at" timestamptz NOT NULL DEFAULT now(), "updated_at" timestamptz NOT NULL DEFAULT now(), PRIMARY KEY ("repository") +CREATE TABLE "public"."package_version" ( "repository" text NOT NULL, "version" text NOT NULL, "created_at" timestamptz NOT NULL DEFAULT now(), "updated_at" timestamptz NOT NULL DEFAULT now(), PRIMARY KEY ("repository") ); -COMMENT ON TABLE "public"."document_version" IS E'tracks documents git version by repository'; +COMMENT ON TABLE "public"."package_version" IS E'tracks git pacakge version for documents'; -CREATE TRIGGER "set_public_document_version_updated_at" - BEFORE UPDATE ON "public"."document_version" +CREATE TRIGGER "set_public_package_version_updated_at" + BEFORE UPDATE ON "public"."package_version" FOR EACH ROW EXECUTE PROCEDURE trigger_set_timestamp (); -COMMENT ON TRIGGER "set_public_document_version_updated_at" ON "public"."document_version" IS 'trigger to set value of column "updated_at" to current timestamp on row update'; +COMMENT ON TRIGGER "set_public_package_version_updated_at" ON "public"."package_version" IS 'trigger to set value of column "updated_at" to current timestamp on row update'; diff --git a/targets/ingester/src/cli.js b/targets/ingester/src/cli.js index 398b78761..60ebd1a96 100644 --- a/targets/ingester/src/cli.js +++ b/targets/ingester/src/cli.js @@ -42,10 +42,10 @@ mutation insert_documents($documents: [documents_insert_input!]!) { } `; -const insertDocumentsVersionMutation = ` -mutation insert_document_version($object:document_version_insert_input!) { - version: insert_document_version_one(object: $object, on_conflict: { - constraint: document_version_pkey, +const insertPackageVersionMutation = ` +mutation insert_package_version($object:package_version_insert_input!) { + version: insert_package_version_one(object: $object, on_conflict: { + constraint: package_version_pkey, update_columns: version }) { repository, version @@ -182,13 +182,13 @@ async function initDocAvailabity(source) { */ async function updateVersion(repository, version) { const result = await client - .mutation(insertDocumentsVersionMutation, { + .mutation(insertPackageVersionMutation, { object: { repository, version }, }) .toPromise(); if (result.error) { console.error(result.error); - throw new Error(`error updating document_version ${repository}@${version}`); + throw new Error(`error updating package_version ${repository}@${version}`); } return result.data.version; }