From 815cb652c3893648f3be825afc4509b308ff440e Mon Sep 17 00:00:00 2001 From: Rodolphe Quiedeville Date: Thu, 12 Dec 2024 21:42:07 +0100 Subject: [PATCH] Add data for table exclusion --- .gitignore | 3 +- Makefile | 9 ++- dist/pgtle.schedoc--0.0.2.sql | 102 ++++++++++++++++++++++++++++++ dist/schedoc--0.0.2.sql | 102 ++++++++++++++++++++++++++++++ dist/schedoc.sql | 102 ++++++++++++++++++++++++++++++ sql/management.sql | 22 +++++++ sql/table.sql | 20 ++++++ test/sql/management_test.sql.in | 12 ++++ test/sql/table_test.sql.in | 20 +++++- tools_exclusion/celery.sql | 16 +++++ tools_exclusion/debezium.sql | 9 +++ tools_exclusion/django.sql | 16 +++++ tools_exclusion/procrastinate.sql | 10 +++ tools_exclusion/tastypie.sql | 9 +++ 14 files changed, 447 insertions(+), 5 deletions(-) create mode 100644 sql/management.sql create mode 100644 test/sql/management_test.sql.in create mode 100644 tools_exclusion/celery.sql create mode 100644 tools_exclusion/debezium.sql create mode 100644 tools_exclusion/django.sql create mode 100644 tools_exclusion/procrastinate.sql create mode 100644 tools_exclusion/tastypie.sql diff --git a/.gitignore b/.gitignore index 182fcdc..eda4284 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ schedoc.control *.zip test/sql/*.sql -test/*.sql \ No newline at end of file +test/*.sql +exclude.sql diff --git a/Makefile b/Makefile index 2914cb8..e329db1 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ INTETESTS = $(shell find test/ -type f -name '*.sql.in' | sed -e 's/.in$$//') EXTENSION = schedoc +TOOLSEXC = $(wildcard tools_exclusion/*.sql) + EXTVERSION = $(shell grep -m 1 '[[:space:]]\{3\}"version":' META.json | \ sed -e 's/[[:space:]]*"version":[[:space:]]*"\([^"]*\)",\{0,1\}/\1/') @@ -31,14 +33,19 @@ all: $(DIST) $(PGTLEOUT) $(EXTENSION).control $(UNITTESTS) $(INTETESTS) clean: rm -f $(PGTLEOUT) $(DIST) $(UNITTESTS) -$(DIST): $(FILES) +$(DIST): $(FILES) exclude.sql cat sql/table.sql > $@ + cat sql/management.sql >> $@ cat sql/function.sql >> $@ cat sql/function-stop.sql >> $@ cat sql/function-status.sql >> $@ + cat exclude.sql >> $@ cat sql/start.sql >> $@ cat $@ > dist/$(EXTENSION).sql +exclude.sql: $(TOOLSEXC) + cat $(TOOLSEXC) > exclude.sql + test: pg_prove -f test/sql/*.sql diff --git a/dist/pgtle.schedoc--0.0.2.sql b/dist/pgtle.schedoc--0.0.2.sql index ae235ce..79a4efd 100644 --- a/dist/pgtle.schedoc--0.0.2.sql +++ b/dist/pgtle.schedoc--0.0.2.sql @@ -54,7 +54,27 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); + +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); +-- +-- +-- CREATE VIEW @extschema@.schedoc_column_comments AS SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status @@ -67,6 +87,28 @@ CREATE VIEW @extschema@.schedoc_column_comments AS -- -- -- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text + LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +-- CREATE OR REPLACE FUNCTION @extschema@.schedoc_start() RETURNS void LANGUAGE plpgsql AS @@ -238,6 +280,66 @@ BEGIN END; $EOF$; -- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permissions', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']); +-- -- Check the schema of installation for schedoc -- DO diff --git a/dist/schedoc--0.0.2.sql b/dist/schedoc--0.0.2.sql index ff3bf01..c5d8a55 100644 --- a/dist/schedoc--0.0.2.sql +++ b/dist/schedoc--0.0.2.sql @@ -48,7 +48,27 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); + +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); +-- +-- +-- CREATE VIEW @extschema@.schedoc_column_comments AS SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status @@ -61,6 +81,28 @@ CREATE VIEW @extschema@.schedoc_column_comments AS -- -- -- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text + LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +-- CREATE OR REPLACE FUNCTION @extschema@.schedoc_start() RETURNS void LANGUAGE plpgsql AS @@ -232,6 +274,66 @@ BEGIN END; $EOF$; -- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permissions', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']); +-- -- Check the schema of installation for schedoc -- DO diff --git a/dist/schedoc.sql b/dist/schedoc.sql index ff3bf01..c5d8a55 100644 --- a/dist/schedoc.sql +++ b/dist/schedoc.sql @@ -48,7 +48,27 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); + +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); +-- +-- +-- CREATE VIEW @extschema@.schedoc_column_comments AS SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status @@ -61,6 +81,28 @@ CREATE VIEW @extschema@.schedoc_column_comments AS -- -- -- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text + LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +-- CREATE OR REPLACE FUNCTION @extschema@.schedoc_start() RETURNS void LANGUAGE plpgsql AS @@ -232,6 +274,66 @@ BEGIN END; $EOF$; -- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permissions', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']); +-- -- Check the schema of installation for schedoc -- DO diff --git a/sql/management.sql b/sql/management.sql new file mode 100644 index 0000000..19ee2d9 --- /dev/null +++ b/sql/management.sql @@ -0,0 +1,22 @@ +-- +-- +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text + LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; diff --git a/sql/table.sql b/sql/table.sql index 3f67469..ab929e5 100644 --- a/sql/table.sql +++ b/sql/table.sql @@ -48,7 +48,27 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); + +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); +-- +-- +-- CREATE VIEW @extschema@.schedoc_column_comments AS SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status diff --git a/test/sql/management_test.sql.in b/test/sql/management_test.sql.in new file mode 100644 index 0000000..9bff2c0 --- /dev/null +++ b/test/sql/management_test.sql.in @@ -0,0 +1,12 @@ +-- +-- +-- +SET search_path=public,pgtap; + +BEGIN; + +SELECT plan(1); + +SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_exclude_tool'::name, ARRAY['text']); + +ROLLBACK; diff --git a/test/sql/table_test.sql.in b/test/sql/table_test.sql.in index d52ccdf..d0bb202 100644 --- a/test/sql/table_test.sql.in +++ b/test/sql/table_test.sql.in @@ -1,19 +1,33 @@ -- --- We assume that ddl_historization is installed in public schema +-- -- SET search_path=public,pgtap; BEGIN; -SELECT plan(5); +SELECT plan(9); + -SELECT has_extension('schedoc'); SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_column_raw'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_valid'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_valid_status'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_column_log'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_table_exclusion'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_table_exclusion_templates'::name); + + SELECT has_view('_TEST_SCHEMA_'::name, 'schedoc_column_comments'::name); SELECT has_enum('_TEST_SCHEMA_'::name, 'schedoc_status'::name); SELECT enum_has_labels('_TEST_SCHEMA_'::name, 'schedoc_status'::name, ARRAY['public', 'private', 'legacy', 'wip']); +SELECT finish(); + ROLLBACK; diff --git a/tools_exclusion/celery.sql b/tools_exclusion/celery.sql new file mode 100644 index 0000000..f59023f --- /dev/null +++ b/tools_exclusion/celery.sql @@ -0,0 +1,16 @@ +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); diff --git a/tools_exclusion/debezium.sql b/tools_exclusion/debezium.sql new file mode 100644 index 0000000..2356e8e --- /dev/null +++ b/tools_exclusion/debezium.sql @@ -0,0 +1,9 @@ +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); diff --git a/tools_exclusion/django.sql b/tools_exclusion/django.sql new file mode 100644 index 0000000..7e05e73 --- /dev/null +++ b/tools_exclusion/django.sql @@ -0,0 +1,16 @@ +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permissions', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); diff --git a/tools_exclusion/procrastinate.sql b/tools_exclusion/procrastinate.sql new file mode 100644 index 0000000..25ce607 --- /dev/null +++ b/tools_exclusion/procrastinate.sql @@ -0,0 +1,10 @@ +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); diff --git a/tools_exclusion/tastypie.sql b/tools_exclusion/tastypie.sql new file mode 100644 index 0000000..c3c56a1 --- /dev/null +++ b/tools_exclusion/tastypie.sql @@ -0,0 +1,9 @@ +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']);