diff --git a/.gitignore b/.gitignore index 182fcdc..eda4284 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ schedoc.control *.zip test/sql/*.sql -test/*.sql \ No newline at end of file +test/*.sql +exclude.sql diff --git a/Makefile b/Makefile index 2914cb8..16b307a 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ INTETESTS = $(shell find test/ -type f -name '*.sql.in' | sed -e 's/.in$$//') EXTENSION = schedoc +TOOLSEXC = $(wildcard tools_exclusion/*.sql) + EXTVERSION = $(shell grep -m 1 '[[:space:]]\{3\}"version":' META.json | \ sed -e 's/[[:space:]]*"version":[[:space:]]*"\([^"]*\)",\{0,1\}/\1/') @@ -29,24 +31,31 @@ include $(PGXS) all: $(DIST) $(PGTLEOUT) $(EXTENSION).control $(UNITTESTS) $(INTETESTS) clean: - rm -f $(PGTLEOUT) $(DIST) $(UNITTESTS) + rm -f $(PGTLEOUT) $(DIST) $(UNITTESTS) exclude.sql -$(DIST): $(FILES) +$(DIST): $(FILES) exclude.sql cat sql/table.sql > $@ + cat sql/management.sql >> $@ cat sql/function.sql >> $@ cat sql/function-stop.sql >> $@ cat sql/function-status.sql >> $@ + cat sql/view.sql >> $@ + cat exclude.sql >> $@ cat sql/start.sql >> $@ cat $@ > dist/$(EXTENSION).sql -test: +exclude.sql: $(TOOLSEXC) + cat $(TOOLSEXC) > exclude.sql + +test: $(UNITTESTS) $(INTETESTS) pg_prove -f test/sql/*.sql test/sql/%.sql: test/sql/%.sql.in sed 's,_TEST_SCHEMA_,$(TEST_SCHEMA),g; ' $< > $@ test/%.sql: test/%.sql.in - sed 's,_TEST_SCHEMA_,$(TEST_SCHEMA),g; ' $< > $@ + echo "--\n-- Auto generated file DO NOT EDIT !!" > $@ + sed 's,_TEST_SCHEMA_,$(TEST_SCHEMA),g; ' $< >> $@ $(PGTLEOUT): dist/$(EXTENSION)--$(EXTVERSION).sql pgtle_header.in pgtle_footer.in sed -e 's/_EXTVERSION_/$(EXTVERSION)/' pgtle_header.in > $(PGTLEOUT) diff --git a/dist/pgtle.schedoc--0.0.2.sql b/dist/pgtle.schedoc--0.0.2.sql index ae235ce..84b66a5 100644 --- a/dist/pgtle.schedoc--0.0.2.sql +++ b/dist/pgtle.schedoc--0.0.2.sql @@ -54,16 +54,129 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); -CREATE VIEW @extschema@.schedoc_column_comments AS +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); - SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status - FROM schedoc_column_raw ccr - JOIN pg_class c ON c.oid = ccr.objoid - JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +-- +-- schedoc_exclude_tool(tool text) +-- schedoc_exclude_tools_all() +-- schedoc_is_table_excluded(tableoid oid) +-- +-- Set up the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tools_all() +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tags[0] FROM @extschema@.schedoc_table_exclusion_templates; + SELECT count(1) FROM @extschema@.schedoc_table_exclusion INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- Check if a table is present in the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_is_table_excluded(tableoid oid) +RETURNS boolean +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + WITH + excluded AS ( + SELECT c.oid + FROM pg_class c + JOIN @extschema@.schedoc_table_exclusion ste ON ste.table_name = c.relname + JOIN pg_namespace n ON (n.oid = c.relnamespace AND n.nspname = ste.schema_name) + ), + internals AS ( + SELECT oid FROM excluded + UNION + SELECT c.oid FROM pg_class c + JOIN pg_namespace n ON (n.oid = c.relnamespace) + WHERE relname IN ( + 'ddl_history', + 'ddl_history_column', + 'ddl_history_schema', + 'schedoc_column_log', + 'schedoc_column_raw', + 'schedoc_table_exclusion', + 'schedoc_table_exclusion_templates', + 'schedoc_valid', + 'schedoc_valid_status') + AND n.nspname = '@extschema@' + ) + SELECT count(1) > 0 FROM internals WHERE oid = tableoid INTO status; + + RETURN status; +END; +$EOF$; +-- -- -- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_init_existing_comments() +RETURNS void +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + + PERFORM @extschema@.schedoc_fill_raw(objoid, objsubid) + FROM @extschema@.schedoc_column_existing_comments + WHERE description IS NOT NULL; + +END; +$EOF$; -- -- -- @@ -75,47 +188,13 @@ BEGIN -- -- Function to manage INSERT statements -- + -- This function is called by a trigger on ddl_history CREATE OR REPLACE FUNCTION @extschema@.schedoc_trg() RETURNS trigger LANGUAGE plpgsql AS $fsub$ BEGIN - -- keep a log of all values - INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid), - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ); - + PERFORM @extschema@.schedoc_fill_raw(NEW.objoid, NEW.objsubid); - -- if the json is valid - IF schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON THEN - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid)::jsonb, - @extschema@.schedoc_get_column_status(NEW.objoid, NEW.objsubid)::@extschema@.schedoc_status, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, - status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - ELSE - -- - -- This is not a valid json, we store it - -- - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - END IF; RETURN NEW; END; $fsub$; @@ -129,6 +208,7 @@ BEGIN BEGIN -- -- + IF NOT @extschema@.schedoc_is_table_excluded(NEW.attrelid) THEN INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) VALUES ( NEW.attrelid, @@ -137,7 +217,7 @@ BEGIN ) ON CONFLICT (objoid, objsubid) DO UPDATE SET is_valid = false; - + END IF; RETURN NEW; END; $fsub$; @@ -163,6 +243,58 @@ BEGIN END; $EOF$; +-- +-- schedoc_fill_raw +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_fill_raw(p_oid oid, p_subid oid) +RETURNS void + LANGUAGE plpgsql AS +$EOF$ +BEGIN + -- + -- + -- keep a log of all values + INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid), + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ); + + IF NOT @extschema@.schedoc_is_table_excluded(p_oid) THEN + -- if the json is valid + IF schedoc_get_column_description(p_oid, p_subid) IS JSON THEN + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid)::jsonb, + @extschema@.schedoc_get_column_status(p_oid, p_subid)::@extschema@.schedoc_status, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, + status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, + is_valid = @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON; + ELSE + -- + -- This is not a valid json, we store it + -- + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + is_valid = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid) IS JSON; + END IF; + END IF; +END; +$EOF$; + + -- -- @@ -238,6 +370,111 @@ BEGIN END; $EOF$; -- +-- +-- + +CREATE VIEW schedoc_column_comments AS + + SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status + FROM schedoc_column_raw ccr + JOIN pg_class c ON c.oid = ccr.objoid + JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +CREATE OR REPLACE VIEW schedoc_object_tables AS + + SELECT n.nspname, c.relname + FROM pg_depend d + JOIN pg_class c ON c.oid = d.objid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_extension e ON e.oid = d.refobjid + WHERE e.extname in ('schedoc', 'ddl_historization') + AND c.relkind = 'r'; +-- +-- +-- +DROP VIEW IF EXISTS schedoc_column_existing_comments; +CREATE OR REPLACE VIEW schedoc_column_existing_comments AS + + WITH descr AS ( + SELECT c.oid as objoid, d.objsubid, c.relname, d.description + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_description d ON d.objoid = c.oid + LEFT JOIN schedoc_table_exclusion e ON e.table_name = c.relname + WHERE c.relkind = 'r' AND n.nspname='public' + AND e.table_name IS NULL + AND c.relname NOT IN ( + SELECT relname FROM @extschema@.schedoc_object_tables) + ) + SELECT objoid, objsubid, relname, description, description IS NOT NULL AND description IS JSON as is_ok + FROM descr + +ORDER BY relname +; +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permission', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_groups', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']); +-- -- Check the schema of installation for schedoc -- DO @@ -254,5 +491,7 @@ END IF; END; $check_start$; +-- +-- $_pg_tle_$ ); diff --git a/dist/schedoc--0.0.2.sql b/dist/schedoc--0.0.2.sql index ff3bf01..40b8029 100644 --- a/dist/schedoc--0.0.2.sql +++ b/dist/schedoc--0.0.2.sql @@ -48,16 +48,129 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); -CREATE VIEW @extschema@.schedoc_column_comments AS +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); - SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status - FROM schedoc_column_raw ccr - JOIN pg_class c ON c.oid = ccr.objoid - JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +-- +-- schedoc_exclude_tool(tool text) +-- schedoc_exclude_tools_all() +-- schedoc_is_table_excluded(tableoid oid) +-- +-- Set up the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tools_all() +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tags[0] FROM @extschema@.schedoc_table_exclusion_templates; + SELECT count(1) FROM @extschema@.schedoc_table_exclusion INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- Check if a table is present in the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_is_table_excluded(tableoid oid) +RETURNS boolean +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + WITH + excluded AS ( + SELECT c.oid + FROM pg_class c + JOIN @extschema@.schedoc_table_exclusion ste ON ste.table_name = c.relname + JOIN pg_namespace n ON (n.oid = c.relnamespace AND n.nspname = ste.schema_name) + ), + internals AS ( + SELECT oid FROM excluded + UNION + SELECT c.oid FROM pg_class c + JOIN pg_namespace n ON (n.oid = c.relnamespace) + WHERE relname IN ( + 'ddl_history', + 'ddl_history_column', + 'ddl_history_schema', + 'schedoc_column_log', + 'schedoc_column_raw', + 'schedoc_table_exclusion', + 'schedoc_table_exclusion_templates', + 'schedoc_valid', + 'schedoc_valid_status') + AND n.nspname = '@extschema@' + ) + SELECT count(1) > 0 FROM internals WHERE oid = tableoid INTO status; + + RETURN status; +END; +$EOF$; +-- -- -- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_init_existing_comments() +RETURNS void +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + + PERFORM @extschema@.schedoc_fill_raw(objoid, objsubid) + FROM @extschema@.schedoc_column_existing_comments + WHERE description IS NOT NULL; + +END; +$EOF$; -- -- -- @@ -69,47 +182,13 @@ BEGIN -- -- Function to manage INSERT statements -- + -- This function is called by a trigger on ddl_history CREATE OR REPLACE FUNCTION @extschema@.schedoc_trg() RETURNS trigger LANGUAGE plpgsql AS $fsub$ BEGIN - -- keep a log of all values - INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid), - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ); - + PERFORM @extschema@.schedoc_fill_raw(NEW.objoid, NEW.objsubid); - -- if the json is valid - IF schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON THEN - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid)::jsonb, - @extschema@.schedoc_get_column_status(NEW.objoid, NEW.objsubid)::@extschema@.schedoc_status, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, - status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - ELSE - -- - -- This is not a valid json, we store it - -- - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - END IF; RETURN NEW; END; $fsub$; @@ -123,6 +202,7 @@ BEGIN BEGIN -- -- + IF NOT @extschema@.schedoc_is_table_excluded(NEW.attrelid) THEN INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) VALUES ( NEW.attrelid, @@ -131,7 +211,7 @@ BEGIN ) ON CONFLICT (objoid, objsubid) DO UPDATE SET is_valid = false; - + END IF; RETURN NEW; END; $fsub$; @@ -157,6 +237,58 @@ BEGIN END; $EOF$; +-- +-- schedoc_fill_raw +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_fill_raw(p_oid oid, p_subid oid) +RETURNS void + LANGUAGE plpgsql AS +$EOF$ +BEGIN + -- + -- + -- keep a log of all values + INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid), + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ); + + IF NOT @extschema@.schedoc_is_table_excluded(p_oid) THEN + -- if the json is valid + IF schedoc_get_column_description(p_oid, p_subid) IS JSON THEN + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid)::jsonb, + @extschema@.schedoc_get_column_status(p_oid, p_subid)::@extschema@.schedoc_status, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, + status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, + is_valid = @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON; + ELSE + -- + -- This is not a valid json, we store it + -- + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + is_valid = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid) IS JSON; + END IF; + END IF; +END; +$EOF$; + + -- -- @@ -232,6 +364,111 @@ BEGIN END; $EOF$; -- +-- +-- + +CREATE VIEW schedoc_column_comments AS + + SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status + FROM schedoc_column_raw ccr + JOIN pg_class c ON c.oid = ccr.objoid + JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +CREATE OR REPLACE VIEW schedoc_object_tables AS + + SELECT n.nspname, c.relname + FROM pg_depend d + JOIN pg_class c ON c.oid = d.objid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_extension e ON e.oid = d.refobjid + WHERE e.extname in ('schedoc', 'ddl_historization') + AND c.relkind = 'r'; +-- +-- +-- +DROP VIEW IF EXISTS schedoc_column_existing_comments; +CREATE OR REPLACE VIEW schedoc_column_existing_comments AS + + WITH descr AS ( + SELECT c.oid as objoid, d.objsubid, c.relname, d.description + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_description d ON d.objoid = c.oid + LEFT JOIN schedoc_table_exclusion e ON e.table_name = c.relname + WHERE c.relkind = 'r' AND n.nspname='public' + AND e.table_name IS NULL + AND c.relname NOT IN ( + SELECT relname FROM @extschema@.schedoc_object_tables) + ) + SELECT objoid, objsubid, relname, description, description IS NOT NULL AND description IS JSON as is_ok + FROM descr + +ORDER BY relname +; +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permission', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_groups', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']); +-- -- Check the schema of installation for schedoc -- DO @@ -248,3 +485,5 @@ END IF; END; $check_start$; +-- +-- diff --git a/dist/schedoc.sql b/dist/schedoc.sql index ff3bf01..40b8029 100644 --- a/dist/schedoc.sql +++ b/dist/schedoc.sql @@ -48,16 +48,129 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); -CREATE VIEW @extschema@.schedoc_column_comments AS +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); - SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status - FROM schedoc_column_raw ccr - JOIN pg_class c ON c.oid = ccr.objoid - JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +-- +-- schedoc_exclude_tool(tool text) +-- schedoc_exclude_tools_all() +-- schedoc_is_table_excluded(tableoid oid) +-- +-- Set up the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tools_all() +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tags[0] FROM @extschema@.schedoc_table_exclusion_templates; + SELECT count(1) FROM @extschema@.schedoc_table_exclusion INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- Check if a table is present in the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_is_table_excluded(tableoid oid) +RETURNS boolean +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + WITH + excluded AS ( + SELECT c.oid + FROM pg_class c + JOIN @extschema@.schedoc_table_exclusion ste ON ste.table_name = c.relname + JOIN pg_namespace n ON (n.oid = c.relnamespace AND n.nspname = ste.schema_name) + ), + internals AS ( + SELECT oid FROM excluded + UNION + SELECT c.oid FROM pg_class c + JOIN pg_namespace n ON (n.oid = c.relnamespace) + WHERE relname IN ( + 'ddl_history', + 'ddl_history_column', + 'ddl_history_schema', + 'schedoc_column_log', + 'schedoc_column_raw', + 'schedoc_table_exclusion', + 'schedoc_table_exclusion_templates', + 'schedoc_valid', + 'schedoc_valid_status') + AND n.nspname = '@extschema@' + ) + SELECT count(1) > 0 FROM internals WHERE oid = tableoid INTO status; + + RETURN status; +END; +$EOF$; +-- -- -- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_init_existing_comments() +RETURNS void +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + + PERFORM @extschema@.schedoc_fill_raw(objoid, objsubid) + FROM @extschema@.schedoc_column_existing_comments + WHERE description IS NOT NULL; + +END; +$EOF$; -- -- -- @@ -69,47 +182,13 @@ BEGIN -- -- Function to manage INSERT statements -- + -- This function is called by a trigger on ddl_history CREATE OR REPLACE FUNCTION @extschema@.schedoc_trg() RETURNS trigger LANGUAGE plpgsql AS $fsub$ BEGIN - -- keep a log of all values - INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid), - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ); - + PERFORM @extschema@.schedoc_fill_raw(NEW.objoid, NEW.objsubid); - -- if the json is valid - IF schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON THEN - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid)::jsonb, - @extschema@.schedoc_get_column_status(NEW.objoid, NEW.objsubid)::@extschema@.schedoc_status, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, - status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - ELSE - -- - -- This is not a valid json, we store it - -- - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - END IF; RETURN NEW; END; $fsub$; @@ -123,6 +202,7 @@ BEGIN BEGIN -- -- + IF NOT @extschema@.schedoc_is_table_excluded(NEW.attrelid) THEN INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) VALUES ( NEW.attrelid, @@ -131,7 +211,7 @@ BEGIN ) ON CONFLICT (objoid, objsubid) DO UPDATE SET is_valid = false; - + END IF; RETURN NEW; END; $fsub$; @@ -157,6 +237,58 @@ BEGIN END; $EOF$; +-- +-- schedoc_fill_raw +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_fill_raw(p_oid oid, p_subid oid) +RETURNS void + LANGUAGE plpgsql AS +$EOF$ +BEGIN + -- + -- + -- keep a log of all values + INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid), + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ); + + IF NOT @extschema@.schedoc_is_table_excluded(p_oid) THEN + -- if the json is valid + IF schedoc_get_column_description(p_oid, p_subid) IS JSON THEN + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid)::jsonb, + @extschema@.schedoc_get_column_status(p_oid, p_subid)::@extschema@.schedoc_status, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, + status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, + is_valid = @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON; + ELSE + -- + -- This is not a valid json, we store it + -- + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + is_valid = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid) IS JSON; + END IF; + END IF; +END; +$EOF$; + + -- -- @@ -232,6 +364,111 @@ BEGIN END; $EOF$; -- +-- +-- + +CREATE VIEW schedoc_column_comments AS + + SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status + FROM schedoc_column_raw ccr + JOIN pg_class c ON c.oid = ccr.objoid + JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +CREATE OR REPLACE VIEW schedoc_object_tables AS + + SELECT n.nspname, c.relname + FROM pg_depend d + JOIN pg_class c ON c.oid = d.objid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_extension e ON e.oid = d.refobjid + WHERE e.extname in ('schedoc', 'ddl_historization') + AND c.relkind = 'r'; +-- +-- +-- +DROP VIEW IF EXISTS schedoc_column_existing_comments; +CREATE OR REPLACE VIEW schedoc_column_existing_comments AS + + WITH descr AS ( + SELECT c.oid as objoid, d.objsubid, c.relname, d.description + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_description d ON d.objoid = c.oid + LEFT JOIN schedoc_table_exclusion e ON e.table_name = c.relname + WHERE c.relkind = 'r' AND n.nspname='public' + AND e.table_name IS NULL + AND c.relname NOT IN ( + SELECT relname FROM @extschema@.schedoc_object_tables) + ) + SELECT objoid, objsubid, relname, description, description IS NOT NULL AND description IS JSON as is_ok + FROM descr + +ORDER BY relname +; +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permission', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_groups', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']); +-- -- Check the schema of installation for schedoc -- DO @@ -248,3 +485,5 @@ END IF; END; $check_start$; +-- +-- diff --git a/sql/function.sql b/sql/function.sql index c9a7628..ca389cd 100644 --- a/sql/function.sql +++ b/sql/function.sql @@ -9,47 +9,13 @@ BEGIN -- -- Function to manage INSERT statements -- + -- This function is called by a trigger on ddl_history CREATE OR REPLACE FUNCTION @extschema@.schedoc_trg() RETURNS trigger LANGUAGE plpgsql AS $fsub$ BEGIN - -- keep a log of all values - INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid), - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ); - + PERFORM @extschema@.schedoc_fill_raw(NEW.objoid, NEW.objsubid); - -- if the json is valid - IF schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON THEN - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid)::jsonb, - @extschema@.schedoc_get_column_status(NEW.objoid, NEW.objsubid)::@extschema@.schedoc_status, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, - status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - ELSE - -- - -- This is not a valid json, we store it - -- - INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) - VALUES ( - NEW.objoid, - NEW.objsubid, - @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON - ) ON CONFLICT (objoid, objsubid) - DO UPDATE SET - is_valid = @extschema@.schedoc_get_column_description(NEW.objoid, NEW.objsubid) IS JSON; - END IF; RETURN NEW; END; $fsub$; @@ -63,6 +29,7 @@ BEGIN BEGIN -- -- + IF NOT @extschema@.schedoc_is_table_excluded(NEW.attrelid) THEN INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) VALUES ( NEW.attrelid, @@ -71,7 +38,7 @@ BEGIN ) ON CONFLICT (objoid, objsubid) DO UPDATE SET is_valid = false; - + END IF; RETURN NEW; END; $fsub$; @@ -97,6 +64,58 @@ BEGIN END; $EOF$; +-- +-- schedoc_fill_raw +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_fill_raw(p_oid oid, p_subid oid) +RETURNS void + LANGUAGE plpgsql AS +$EOF$ +BEGIN + -- + -- + -- keep a log of all values + INSERT INTO @extschema@.schedoc_column_log (objoid, objsubid, comment, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid), + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ); + + IF NOT @extschema@.schedoc_is_table_excluded(p_oid) THEN + -- if the json is valid + IF schedoc_get_column_description(p_oid, p_subid) IS JSON THEN + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, comment, status, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid)::jsonb, + @extschema@.schedoc_get_column_status(p_oid, p_subid)::@extschema@.schedoc_status, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + comment = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid)::jsonb, + status = @extschema@.schedoc_get_column_status(EXCLUDED.objoid, EXCLUDED.objsubid)::@extschema@.schedoc_status, + is_valid = @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON; + ELSE + -- + -- This is not a valid json, we store it + -- + INSERT INTO @extschema@.schedoc_column_raw (objoid, objsubid, is_valid) + VALUES ( + p_oid, + p_subid, + @extschema@.schedoc_get_column_description(p_oid, p_subid) IS JSON + ) ON CONFLICT (objoid, objsubid) + DO UPDATE SET + is_valid = @extschema@.schedoc_get_column_description(EXCLUDED.objoid, EXCLUDED.objsubid) IS JSON; + END IF; + END IF; +END; +$EOF$; + + -- -- diff --git a/sql/management.sql b/sql/management.sql new file mode 100644 index 0000000..617f682 --- /dev/null +++ b/sql/management.sql @@ -0,0 +1,102 @@ +-- +-- schedoc_exclude_tool(tool text) +-- schedoc_exclude_tools_all() +-- schedoc_is_table_excluded(tableoid oid) +-- +-- Set up the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tool(tool text) +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tool FROM @extschema@.schedoc_table_exclusion_templates + WHERE tags @> ARRAY[tool]; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion WHERE tag = tool INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_exclude_tools_all() +RETURNS text +LANGUAGE plpgsql AS +$EOF$ +DECLARE + nbrows bigint; +BEGIN + -- + -- + -- + INSERT INTO @extschema@.schedoc_table_exclusion (schema_name, table_name, tag) + SELECT schema_name, table_name, tags[0] FROM @extschema@.schedoc_table_exclusion_templates; + + SELECT count(1) FROM @extschema@.schedoc_table_exclusion INTO nbrows; + + RETURN format ('Inserted %s row(s) in schedoc_table_exclusion', nbrows); +END; +$EOF$; +-- +-- Check if a table is present in the exclusion list +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_is_table_excluded(tableoid oid) +RETURNS boolean +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + WITH + excluded AS ( + SELECT c.oid + FROM pg_class c + JOIN @extschema@.schedoc_table_exclusion ste ON ste.table_name = c.relname + JOIN pg_namespace n ON (n.oid = c.relnamespace AND n.nspname = ste.schema_name) + ), + internals AS ( + SELECT oid FROM excluded + UNION + SELECT c.oid FROM pg_class c + JOIN pg_namespace n ON (n.oid = c.relnamespace) + WHERE relname IN ( + 'ddl_history', + 'ddl_history_column', + 'ddl_history_schema', + 'schedoc_column_log', + 'schedoc_column_raw', + 'schedoc_table_exclusion', + 'schedoc_table_exclusion_templates', + 'schedoc_valid', + 'schedoc_valid_status') + AND n.nspname = '@extschema@' + ) + SELECT count(1) > 0 FROM internals WHERE oid = tableoid INTO status; + + RETURN status; +END; +$EOF$; +-- +-- +-- +CREATE OR REPLACE FUNCTION @extschema@.schedoc_init_existing_comments() +RETURNS void +LANGUAGE plpgsql AS +$EOF$ +DECLARE + status boolean; +BEGIN + + PERFORM @extschema@.schedoc_fill_raw(objoid, objsubid) + FROM @extschema@.schedoc_column_existing_comments + WHERE description IS NOT NULL; + +END; +$EOF$; diff --git a/sql/start.sql b/sql/start.sql index 26fe3a1..8c20bf5 100644 --- a/sql/start.sql +++ b/sql/start.sql @@ -15,3 +15,5 @@ END IF; END; $check_start$; +-- +-- diff --git a/sql/table.sql b/sql/table.sql index 3f67469..c855678 100644 --- a/sql/table.sql +++ b/sql/table.sql @@ -48,13 +48,24 @@ CREATE TABLE @extschema@.schedoc_column_log ( -- -- -- +CREATE TABLE @extschema@.schedoc_table_exclusion ( + schema_name name, + table_name name, + tag text, + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); -CREATE VIEW @extschema@.schedoc_column_comments AS - - SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status - FROM schedoc_column_raw ccr - JOIN pg_class c ON c.oid = ccr.objoid - JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +CREATE TABLE @extschema@.schedoc_table_exclusion_templates ( + schema_name name, + table_name name, + tags text[], + created_at timestamp with time zone DEFAULT current_timestamp, + created_by text DEFAULT current_user, + PRIMARY KEY (schema_name, table_name) +); -- -- +-- diff --git a/sql/view.sql b/sql/view.sql new file mode 100644 index 0000000..7d18987 --- /dev/null +++ b/sql/view.sql @@ -0,0 +1,44 @@ +-- +-- +-- + +CREATE VIEW schedoc_column_comments AS + + SELECT current_database() as databasename, c.relname as tablename, a.attname as columnname, status + FROM schedoc_column_raw ccr + JOIN pg_class c ON c.oid = ccr.objoid + JOIN pg_attribute a ON (a.attnum = ccr.objsubid AND a.attrelid = ccr.objoid); +-- +-- +-- +CREATE OR REPLACE VIEW schedoc_object_tables AS + + SELECT n.nspname, c.relname + FROM pg_depend d + JOIN pg_class c ON c.oid = d.objid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_extension e ON e.oid = d.refobjid + WHERE e.extname in ('schedoc', 'ddl_historization') + AND c.relkind = 'r'; +-- +-- +-- +DROP VIEW IF EXISTS schedoc_column_existing_comments; +CREATE OR REPLACE VIEW schedoc_column_existing_comments AS + + WITH descr AS ( + SELECT c.oid as objoid, d.objsubid, c.relname, d.description + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_description d ON d.objoid = c.oid + LEFT JOIN schedoc_table_exclusion e ON e.table_name = c.relname + WHERE c.relkind = 'r' AND n.nspname='public' + AND e.table_name IS NULL + AND c.relname NOT IN ( + SELECT relname FROM @extschema@.schedoc_object_tables) + ) + SELECT objoid, objsubid, relname, description, description IS NOT NULL AND description IS JSON as is_ok + FROM descr + +ORDER BY relname +; diff --git a/test/create_table.sql.in b/test/create_table.sql.in index b03c07f..a1186eb 100644 --- a/test/create_table.sql.in +++ b/test/create_table.sql.in @@ -1,7 +1,5 @@ -- -- --- - SET search_path=public,pgtap,_TEST_SCHEMA_; BEGIN; diff --git a/test/sql/function_test.sql.in b/test/sql/function_test.sql.in index 7f927da..77dfa97 100644 --- a/test/sql/function_test.sql.in +++ b/test/sql/function_test.sql.in @@ -6,11 +6,12 @@ SET search_path=public,pgtap; BEGIN; -SELECT plan(4); +SELECT plan(5); SELECT has_extension('schedoc'); SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_start'::name); +SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_fill_raw'::name); SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_get_column_description'::name, ARRAY['oid', 'oid']); SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_get_column_status'::name, ARRAY['oid', 'oid']); diff --git a/test/sql/management_test.sql.in b/test/sql/management_test.sql.in new file mode 100644 index 0000000..a7618ca --- /dev/null +++ b/test/sql/management_test.sql.in @@ -0,0 +1,71 @@ +-- +-- +-- +SET search_path=public,pgtap,_TEST_SCHEMA_; + +BEGIN; + +SELECT plan(14); + +SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_exclude_tool'::name, ARRAY['text']); +SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_exclude_tools_all'::name); +SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_is_table_excluded'::name, ARRAY['oid']); +SELECT has_function('_TEST_SCHEMA_'::name, 'schedoc_init_existing_comments'::name); +-- +-- Return False on non existing id +-- +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(0)', + ARRAY['False'::boolean], + 'Return False on non existing id'); + + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''ddl_history''', + ARRAY['True'::boolean], + 'The table ddl_history is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''ddl_history_column''', + ARRAY['True'::boolean], + 'The table ddl_history_column is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''ddl_history_schema''', + ARRAY['True'::boolean], + 'The table ddl_history_schema is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''schedoc_column_log''', + ARRAY['True'::boolean], + 'The table schedoc_column_log is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''schedoc_column_raw''', + ARRAY['True'::boolean], + 'The table schedoc_column_raw is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''schedoc_table_exclusion''', + ARRAY['True'::boolean], + 'The table schedoc_table_exclusion is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''schedoc_table_exclusion_templates''', + ARRAY['True'::boolean], + 'The table schedoc_table_exclusion_templates is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''schedoc_valid''', + ARRAY['True'::boolean], + 'The table schedoc_valid is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''schedoc_valid_status''', + ARRAY['True'::boolean], + 'The table schedoc_valid_status is excluded'); + +SELECT finish(); + + +ROLLBACK; diff --git a/test/sql/table_test.sql.in b/test/sql/table_test.sql.in index d52ccdf..9125ff0 100644 --- a/test/sql/table_test.sql.in +++ b/test/sql/table_test.sql.in @@ -1,19 +1,28 @@ -- --- We assume that ddl_historization is installed in public schema -- - +-- SET search_path=public,pgtap; BEGIN; -SELECT plan(5); +SELECT plan(8); -SELECT has_extension('schedoc'); SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_column_raw'::name); -SELECT has_view('_TEST_SCHEMA_'::name, 'schedoc_column_comments'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_valid'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_valid_status'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_column_log'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_table_exclusion'::name); + +SELECT has_table('_TEST_SCHEMA_'::name, 'schedoc_table_exclusion_templates'::name); SELECT has_enum('_TEST_SCHEMA_'::name, 'schedoc_status'::name); SELECT enum_has_labels('_TEST_SCHEMA_'::name, 'schedoc_status'::name, ARRAY['public', 'private', 'legacy', 'wip']); +SELECT finish(); + ROLLBACK; diff --git a/test/sql/view_test.sql.in b/test/sql/view_test.sql.in new file mode 100644 index 0000000..e81d90a --- /dev/null +++ b/test/sql/view_test.sql.in @@ -0,0 +1,16 @@ +-- +-- 2 views +-- +SET search_path=public,_TEST_SCHEMA_; + +BEGIN; + +SELECT plan(2); + +SELECT has_view('_TEST_SCHEMA_'::name, 'schedoc_column_existing_comments'::name); + +SELECT has_view('_TEST_SCHEMA_'::name, 'schedoc_column_comments'::name); + +SELECT finish(); + +ROLLBACK; diff --git a/test/table_exclusion.sql.in b/test/table_exclusion.sql.in new file mode 100644 index 0000000..84c798b --- /dev/null +++ b/test/table_exclusion.sql.in @@ -0,0 +1,69 @@ +-- +-- We assume that ddl_historization is installed in public schema +-- + +SET search_path=public,pgtap,_TEST_SCHEMA_; + +BEGIN; + +SELECT plan(7); + +DROP EXTENSION IF EXISTS schedoc CASCADE; +CREATE EXTENSION schedoc WITH SCHEMA _TEST_SCHEMA_ CASCADE; +SELECT schedoc_start(); + +TRUNCATE schedoc_column_raw; +TRUNCATE schedoc_column_log; +TRUNCATE ddl_history; + +-- create the table before the exclusion list is loaded +CREATE TABLE IF NOT EXISTS dbz_signal (id int); + +SELECT results_eq( + 'SELECT count(*) FROM schedoc_column_raw', + 'SELECT CAST(1 as bigint)', + 'We have 1 row in schedoc_column_raw'); + +SELECT results_eq( + 'SELECT count(*) FROM schedoc_column_log', + 'SELECT CAST(0 as bigint)', + 'We have 0 row in schedoc_column_log'); + + +-- the table is not excluded by default +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''dbz_signal''', + ARRAY['False'::boolean], + 'The table dbz_signal is not excluded'); + +-- load the exclusion list +SELECT schedoc_exclude_tool('debezium'); + +SELECT results_eq( + 'SELECT count(*) FROM schedoc_table_exclusion', + 'SELECT CAST(2 as bigint)', + 'We have 2 rows in schedoc_table_exclusion'); + +-- create a table after the exclusion list is loaded +CREATE TABLE IF NOT EXISTS dbz_heartbeat (id int); + +-- Now the table is excluded +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''dbz_signal''', + ARRAY['True'::boolean], + 'The table dbz_signal is excluded'); + +SELECT results_eq( + 'SELECT schedoc_is_table_excluded(oid) FROM pg_class WHERE relname=''dbz_heartbeat''', + ARRAY['True'::boolean], + 'The table dbz_hearbeat is excluded'); + +-- There is only the column id of table dbz_signal +SELECT results_eq( + 'SELECT count(*) FROM schedoc_column_raw', + 'SELECT CAST(1 as bigint)', + 'We have 1 row in schedoc_column_raw'); + +SELECT finish(); + +ROLLBACK; diff --git a/tools_exclusion/celery.sql b/tools_exclusion/celery.sql new file mode 100644 index 0000000..f59023f --- /dev/null +++ b/tools_exclusion/celery.sql @@ -0,0 +1,16 @@ +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'celery_results_taskresult', ARRAY['celery']), +('public', 'celery_taskmeta', ARRAY['celery']), +('public', 'celery_tasksetmeta', ARRAY['celery']), + +('public', 'djcelery_crontabschedule', ARRAY['celery']), +('public', 'djcelery_intervalschedule', ARRAY['celery']), +('public', 'djcelery_periodictask', ARRAY['celery']), +('public', 'djcelery_taskstate', ARRAY['celery']), +('public', 'djcelery_workerstate', ARRAY['celery']); diff --git a/tools_exclusion/debezium.sql b/tools_exclusion/debezium.sql new file mode 100644 index 0000000..2356e8e --- /dev/null +++ b/tools_exclusion/debezium.sql @@ -0,0 +1,9 @@ +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'dbz_signal', ARRAY['debezium']), +('public', 'dbz_heartbeat', ARRAY['debezium']); diff --git a/tools_exclusion/django.sql b/tools_exclusion/django.sql new file mode 100644 index 0000000..ebc5191 --- /dev/null +++ b/tools_exclusion/django.sql @@ -0,0 +1,17 @@ +-- +-- Exclude tables created by Django Framework +-- +-- https://www.djangoproject.com/ +-- +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES +('public', 'auth_group', ARRAY['django']), +('public', 'auth_group_permissions', ARRAY['django']), +('public', 'auth_permission', ARRAY['django']), +('public', 'auth_user', ARRAY['django']), +('public', 'auth_user_groups', ARRAY['django']), +('public', 'auth_user_user_permissions', ARRAY['django']), +('public', 'django_admin_log', ARRAY['django']), +('public', 'django_content_type', ARRAY['django']), +('public', 'django_migrations', ARRAY['django']), +('public', 'django_session', ARRAY['django']); diff --git a/tools_exclusion/procrastinate.sql b/tools_exclusion/procrastinate.sql new file mode 100644 index 0000000..25ce607 --- /dev/null +++ b/tools_exclusion/procrastinate.sql @@ -0,0 +1,10 @@ +-- +-- Debezium +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'procrastinate_events', ARRAY['procrastinate']), +('public', 'procrastinate_jobs', ARRAY['procrastinate']), +('public', 'procrastinate_periodic_defers', ARRAY['procrastinate']); diff --git a/tools_exclusion/tastypie.sql b/tools_exclusion/tastypie.sql new file mode 100644 index 0000000..c3c56a1 --- /dev/null +++ b/tools_exclusion/tastypie.sql @@ -0,0 +1,9 @@ +-- +-- +-- + +INSERT INTO @extschema@.schedoc_table_exclusion_templates (schema_name, table_name, tags) +VALUES + +('public', 'tastypie_apiaccess', ARRAY['tastypie']), +('public', 'tastypie_apikey', ARRAY['tastypie']);