Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix removal of metadata function and update script #6996

Merged
merged 1 commit into from
Jun 5, 2024
Merged

Fix removal of metadata function and update script #6996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .unreleased/pr_6996
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixes: #6976 Fix removal of metadata function and update script
Thanks: @gugu for reporting the issue with catalog corruption due to update
5 changes: 5 additions & 0 deletions sql/maintenance_utils.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@ BEGIN
USING _timescaledb_catalog.chunk_constraint
WHERE dimension_slice.id = chunk_constraint.dimension_slice_id
AND chunk_constraint.chunk_id = _chunk_id
AND NOT EXISTS (
SELECT FROM _timescaledb_catalog.chunk_constraint cc
WHERE cc.chunk_id <> _chunk_id
AND cc.dimension_slice_id = dimension_slice.id
)
antekresic marked this conversation as resolved.
Show resolved Hide resolved
RETURNING _timescaledb_catalog.dimension_slice.id
)
DELETE FROM _timescaledb_catalog.chunk_constraint
Expand Down
5 changes: 5 additions & 0 deletions sql/updates/2.14.2--2.15.0.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ BEGIN
USING _timescaledb_catalog.chunk_constraint
WHERE dimension_slice.id = chunk_constraint.dimension_slice_id
AND chunk_constraint.chunk_id = _chunk_id
AND NOT EXISTS (
SELECT FROM _timescaledb_catalog.chunk_constraint cc
WHERE cc.chunk_id <> _chunk_id
AND cc.dimension_slice_id = dimension_slice.id
)
RETURNING _timescaledb_catalog.dimension_slice.id
)
DELETE FROM _timescaledb_catalog.chunk_constraint
Expand Down
113 changes: 113 additions & 0 deletions tsl/test/expected/cagg_migrate.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2955,3 +2955,116 @@ DROP MATERIALIZED VIEW conditions_summary_daily;
DROP MATERIALIZED VIEW conditions_summary_weekly;
DROP TABLE conditions CASCADE;
psql:include/cagg_migrate_custom_timezone.sql:23: NOTICE: drop cascades to 3 other objects
-- #########################################################
-- Issue 6976 - space partitioning should not cause catalog corruption
-- #########################################################
CREATE TABLE space_partitioning (
time timestamptz,
device_id integer,
value float
);
-- Updating sequence numbers so creating a hypertable doesn't mess with
-- data imports used by migration tests
SELECT setval('_timescaledb_catalog.hypertable_id_seq', 999, true);
setval
--------
999
(1 row)

SELECT setval('_timescaledb_catalog.chunk_id_seq', 999, true);
setval
--------
999
(1 row)

SELECT setval('_timescaledb_catalog.dimension_id_seq', 999, true);
setval
--------
999
(1 row)

SELECT setval('_timescaledb_catalog.dimension_slice_id_seq', 999, true);
setval
--------
999
(1 row)

SELECT create_hypertable('space_partitioning', 'time', chunk_time_interval=>'1 hour'::interval);
NOTICE: adding not-null constraint to column "time"
create_hypertable
------------------------------------
(1000,public,space_partitioning,t)
(1 row)

SELECT add_dimension('space_partitioning', 'device_id', 3);
add_dimension
----------------------------------------------
(1001,public,space_partitioning,device_id,t)
(1 row)

INSERT INTO space_partitioning SELECT t, 1, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
INSERT INTO space_partitioning SELECT t, 1000, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
CREATE MATERIALIZED VIEW space_partitioning_summary
WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
SELECT
time_bucket(INTERVAL '1 week', "time") AS bucket,
device_id,
MIN(value),
MAX(value),
SUM(value)
FROM
space_partitioning
GROUP BY
1, 2
WITH NO DATA;
-- setting up the state so that remove_dropped_chunk_metadata
-- would run on the hypertable and trigger the catalog corruption
UPDATE _timescaledb_catalog.chunk
SET dropped = TRUE
FROM _timescaledb_catalog.hypertable
WHERE chunk.hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning'
AND chunk.id = 1000;
UPDATE _timescaledb_catalog.continuous_agg
SET finalized = true
FROM _timescaledb_catalog.hypertable
WHERE continuous_agg.raw_hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning';
SET timescaledb.restoring TO ON;
DROP TABLE _timescaledb_internal._hyper_1000_1000_chunk;
SET timescaledb.restoring TO OFF;
SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id)
FROM _timescaledb_catalog.hypertable
WHERE table_name = 'space_partitioning';
INFO: Removing metadata of chunk 1000 from hypertable 1000
remove_dropped_chunk_metadata
-------------------------------
1
(1 row)

-- check every chunk has as many chunk constraints as
-- there are dimensions, should return empty result
-- this ensures we have avoided catalog corruption
WITH dimension_count as (
SELECT ht.id, count(*)
FROM _timescaledb_catalog.hypertable ht
INNER JOIN _timescaledb_catalog.dimension d
ON d.hypertable_id = ht.id
WHERE table_name = 'space_partitioning'
GROUP BY 1),
chunk_constraint_count AS (
SELECT c.hypertable_id, cc.chunk_id, count(*)
FROM _timescaledb_catalog.chunk_constraint cc
INNER JOIN _timescaledb_catalog.chunk c
ON cc.chunk_id = c.id
GROUP BY 1, 2
)
SELECT *
FROM dimension_count dc
INNER JOIN chunk_constraint_count ccc
ON ccc.hypertable_id = dc.id
WHERE dc.count != ccc.count;
id | count | hypertable_id | chunk_id | count
----+-------+---------------+----------+-------
(0 rows)

83 changes: 83 additions & 0 deletions tsl/test/sql/cagg_migrate.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,86 @@ SET timezone = 'Europe/Budapest';
-- Test with timestamptz
\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ
\ir include/cagg_migrate_custom_timezone.sql

-- #########################################################
-- Issue 6976 - space partitioning should not cause catalog corruption
-- #########################################################

CREATE TABLE space_partitioning (
time timestamptz,
device_id integer,
value float
);

-- Updating sequence numbers so creating a hypertable doesn't mess with
-- data imports used by migration tests
SELECT setval('_timescaledb_catalog.hypertable_id_seq', 999, true);
SELECT setval('_timescaledb_catalog.chunk_id_seq', 999, true);
SELECT setval('_timescaledb_catalog.dimension_id_seq', 999, true);
SELECT setval('_timescaledb_catalog.dimension_slice_id_seq', 999, true);

SELECT create_hypertable('space_partitioning', 'time', chunk_time_interval=>'1 hour'::interval);
SELECT add_dimension('space_partitioning', 'device_id', 3);

INSERT INTO space_partitioning SELECT t, 1, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
INSERT INTO space_partitioning SELECT t, 1000, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;

CREATE MATERIALIZED VIEW space_partitioning_summary
WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
SELECT
time_bucket(INTERVAL '1 week', "time") AS bucket,
device_id,
MIN(value),
MAX(value),
SUM(value)
FROM
space_partitioning
GROUP BY
1, 2
WITH NO DATA;

-- setting up the state so that remove_dropped_chunk_metadata
-- would run on the hypertable and trigger the catalog corruption
UPDATE _timescaledb_catalog.chunk
SET dropped = TRUE
FROM _timescaledb_catalog.hypertable
WHERE chunk.hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning'
AND chunk.id = 1000;
UPDATE _timescaledb_catalog.continuous_agg
SET finalized = true
FROM _timescaledb_catalog.hypertable
WHERE continuous_agg.raw_hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning';
SET timescaledb.restoring TO ON;
DROP TABLE _timescaledb_internal._hyper_1000_1000_chunk;
SET timescaledb.restoring TO OFF;

SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id)
FROM _timescaledb_catalog.hypertable
WHERE table_name = 'space_partitioning';


-- check every chunk has as many chunk constraints as
-- there are dimensions, should return empty result
-- this ensures we have avoided catalog corruption
WITH dimension_count as (
SELECT ht.id, count(*)
FROM _timescaledb_catalog.hypertable ht
INNER JOIN _timescaledb_catalog.dimension d
ON d.hypertable_id = ht.id
WHERE table_name = 'space_partitioning'
GROUP BY 1),
chunk_constraint_count AS (
SELECT c.hypertable_id, cc.chunk_id, count(*)
FROM _timescaledb_catalog.chunk_constraint cc
INNER JOIN _timescaledb_catalog.chunk c
ON cc.chunk_id = c.id
GROUP BY 1, 2
)
SELECT *
FROM dimension_count dc
INNER JOIN chunk_constraint_count ccc
ON ccc.hypertable_id = dc.id
WHERE dc.count != ccc.count;

Loading