From befa7b52a03bdd47329bbef13efb03b3646af91a Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Thu, 21 Nov 2024 17:06:48 -0500 Subject: [PATCH 01/18] add callout for net new --- website/docs/reference/snapshot-configs.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 7b3c0f8e5b1..22c5ebf86cc 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -13,11 +13,6 @@ import ConfigGeneral from '/snippets/_config-description-general.md'; * [Snapshots](/docs/build/snapshots) * The `dbt snapshot` [command](/reference/commands/snapshot) - ## Available configurations ### Snapshot-specific configurations @@ -125,7 +120,7 @@ snapshots: -Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. +Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. @@ -155,6 +150,17 @@ Configurations can be applied to snapshots using the [YAML syntax](/docs/build/s +:::tip + +Configuring snapshots in YAML is recommended for new snapshot configurations. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: +1. Creating a backup copy of your snapshots (or using version control) +2. Converting the configurations one at a time, testing as you go +3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements + +The YAML configuration offers improved maintainability and consistency, but there's no pressure to migrate existing snapshots immediately — feel free to do so at your own pace while ensuring data quality is maintained. +::: + + + ```yaml snapshots: [](/reference/resource-configs/resource-path): @@ -256,7 +263,7 @@ snapshots: -Configurations can be applied to snapshots using [YAML syntax](/docs/build/snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. +Configurations can be applied to snapshots using [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. From 0347fd3d174d81b5b738976b82a97e26e722bf66 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Mon, 25 Nov 2024 16:20:27 +0000 Subject: [PATCH 02/18] turn to snippet --- website/docs/reference/snapshot-configs.md | 6 ++++-- website/snippets/_snapshots-yaml-config.md | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 website/snippets/_snapshots-yaml-config.md diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 22c5ebf86cc..4a3541c2d51 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -7,6 +7,7 @@ meta: import ConfigResource from '/snippets/_config-description-resource.md'; import ConfigGeneral from '/snippets/_config-description-general.md'; +import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; ## Related documentation @@ -120,7 +121,8 @@ snapshots: -Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. + + @@ -263,7 +265,7 @@ snapshots: -Configurations can be applied to snapshots using [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. + diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md new file mode 100644 index 00000000000..68ccd7c0bc2 --- /dev/null +++ b/website/snippets/_snapshots-yaml-config.md @@ -0,0 +1,2 @@ +Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. + From 576af99f2d7654c733a884e415a3629e628ff4ae Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:22:10 +0000 Subject: [PATCH 03/18] Update website/docs/reference/snapshot-configs.md Co-authored-by: Grace Goheen <53586774+graciegoheen@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 4a3541c2d51..5f5e88c9776 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -154,7 +154,7 @@ snapshots: :::tip -Configuring snapshots in YAML is recommended for new snapshot configurations. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: +Configuring snapshots in YAML is recommended for new snapshots. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: 1. Creating a backup copy of your snapshots (or using version control) 2. Converting the configurations one at a time, testing as you go 3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements From 2532cad5b35d9c6ebd0d3e8e964eb1d00ebdd34d Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Mon, 25 Nov 2024 16:24:41 +0000 Subject: [PATCH 04/18] update grace feedback --- website/docs/reference/snapshot-configs.md | 2 +- website/snippets/_snapshots-yaml-config.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 4a3541c2d51..e9ba021403d 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -155,7 +155,7 @@ snapshots: :::tip Configuring snapshots in YAML is recommended for new snapshot configurations. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: -1. Creating a backup copy of your snapshots (or using version control) +1. Creating a backup copy of your snapshots. 2. Converting the configurations one at a time, testing as you go 3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md index 68ccd7c0bc2..10eb8fed17c 100644 --- a/website/snippets/_snapshots-yaml-config.md +++ b/website/snippets/_snapshots-yaml-config.md @@ -1,2 +1 @@ -Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. - +Define snapshots using the latest [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. From 7e68e914ffcd179b0b04ff28250fec5d42c61aaa Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Wed, 27 Nov 2024 11:20:11 +0000 Subject: [PATCH 05/18] update to snapshot configs --- website/docs/reference/snapshot-configs.md | 30 ++++++++++++------- website/snippets/_snapshots-yaml-config.md | 2 +- .../components/expandable/styles.module.css | 3 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index a2e8365ae93..fbcdad93d56 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -18,6 +18,7 @@ import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; ## Available configurations ### Snapshot-specific configurations + @@ -74,8 +75,9 @@ snapshots: [+](/reference/resource-configs/plus-prefix)[strategy](/reference/resource-configs/strategy): timestamp | check [+](/reference/resource-configs/plus-prefix)[updated_at](/reference/resource-configs/updated_at): [+](/reference/resource-configs/plus-prefix)[check_cols](/reference/resource-configs/check_cols): [] | all + [+](/reference/resource-configs/plus-prefix)[invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) : true | false [+](/reference/resource-configs/plus-prefix)[snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} - [+](/reference/resource-configs/plus-prefix)[invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) : true | false + [+](/reference/resource-configs/plus-prefix)[dbt_valid_to_current](/reference/resource-configs/dbt_valid_to_current): ``` @@ -108,8 +110,9 @@ snapshots: [strategy](/reference/resource-configs/strategy): timestamp | check [updated_at](/reference/resource-configs/updated_at): [check_cols](/reference/resource-configs/check_cols): [] | all - [snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} [invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) : true | false + [snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} + [dbt_valid_to_current](/reference/resource-configs/dbt_valid_to_current): ``` @@ -147,20 +150,25 @@ snapshots: +### Snapshot configuration migration -### General configurations +The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies to your snapshots: - +#### For new snapshots +- Use the latest snapshot YAML configurations when creating new snapshots that didn't exist previously. + +#### For existing snapshots +- Migrate tables — Migrate the previous snapshot to the new table schema and values: + - Create a backup copy of your snapshots. + - Use `alter` statements as needed to ensure table consistency or use a script to apply the `alter` statements. +- New YAML config — Convert the YAML configurations one at a time, testing as you go. -:::tip +If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data may result in mixed old and new data, leading to incorrect downstream result. -Configuring snapshots in YAML is recommended for new snapshots. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: -1. Creating a backup copy of your snapshots. -2. Converting the configurations one at a time, testing as you go -3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements -The YAML configuration offers improved maintainability and consistency, but there's no pressure to migrate existing snapshots immediately — feel free to do so at your own pace while ensuring data quality is maintained. -::: +### General configurations + + Date: Wed, 27 Nov 2024 15:52:47 +0000 Subject: [PATCH 06/18] Update website/docs/reference/snapshot-configs.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index fbcdad93d56..ad74ae9d423 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -163,7 +163,7 @@ The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such - Use `alter` statements as needed to ensure table consistency or use a script to apply the `alter` statements. - New YAML config — Convert the YAML configurations one at a time, testing as you go. -If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data may result in mixed old and new data, leading to incorrect downstream result. +If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data, you may have mixed old and new data, leading to an incorrect downstream result. ### General configurations From bdf2850e5d4aae38d6d5c04a790952103803d6ce Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:03 +0000 Subject: [PATCH 07/18] Update website/docs/reference/snapshot-configs.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index ad74ae9d423..a0edf569bf3 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -160,7 +160,7 @@ The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such #### For existing snapshots - Migrate tables — Migrate the previous snapshot to the new table schema and values: - Create a backup copy of your snapshots. - - Use `alter` statements as needed to ensure table consistency or use a script to apply the `alter` statements. + - Use `alter` statements as needed (or a script to apply `alter` statements) to ensure table consistency. - New YAML config — Convert the YAML configurations one at a time, testing as you go. If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data, you may have mixed old and new data, leading to an incorrect downstream result. From 6a84a7f955744ae9b52169acf2b9ec6a2dce532a Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:11 +0000 Subject: [PATCH 08/18] Update website/docs/reference/snapshot-configs.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index a0edf569bf3..598a8a35aa4 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -152,7 +152,7 @@ snapshots: ### Snapshot configuration migration -The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies to your snapshots: +The latest snapshot YAML configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: #### For new snapshots - Use the latest snapshot YAML configurations when creating new snapshots that didn't exist previously. From 37efd7a83bf6ca39130ef66883495c5279dd2664 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:27 +0000 Subject: [PATCH 09/18] Update website/snippets/_snapshots-yaml-config.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/snippets/_snapshots-yaml-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md index cdfb7ceb07e..648b65752c0 100644 --- a/website/snippets/_snapshots-yaml-config.md +++ b/website/snippets/_snapshots-yaml-config.md @@ -1 +1 @@ -Define snapshots using the latest [YAML configurations](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. +Define snapshots using the latest [YAML configurations](/docs/build/snapshots#configuring-snapshots), available in Versionless dbt Cloud and Core v1.9 and higher, in the `snapshot` directory file. From 5d3b48f6b12e90306403ca94d95fb9d0a991f318 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Thu, 28 Nov 2024 11:01:40 +0000 Subject: [PATCH 10/18] update --- website/docs/reference/snapshot-configs.md | 17 ++++++----------- website/snippets/_snapshots-yaml-config.md | 1 - 2 files changed, 6 insertions(+), 12 deletions(-) delete mode 100644 website/snippets/_snapshots-yaml-config.md diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 598a8a35aa4..4d8d8e42d26 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -18,14 +18,11 @@ import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; ## Available configurations ### Snapshot-specific configurations - -import SnapshotYaml from '/snippets/_snapshot-yaml-spec.md'; - - +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. @@ -152,19 +149,17 @@ snapshots: ### Snapshot configuration migration -The latest snapshot YAML configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: - -#### For new snapshots -- Use the latest snapshot YAML configurations when creating new snapshots that didn't exist previously. +The latest snapshot configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names), [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current), and `hard_deletes`) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: #### For existing snapshots - Migrate tables — Migrate the previous snapshot to the new table schema and values: - Create a backup copy of your snapshots. - Use `alter` statements as needed (or a script to apply `alter` statements) to ensure table consistency. -- New YAML config — Convert the YAML configurations one at a time, testing as you go. +- New configurations — Convert the configs one at a time, testing as you go. +:::warning If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data, you may have mixed old and new data, leading to an incorrect downstream result. - +::: ### General configurations @@ -273,7 +268,7 @@ snapshots: - +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [configurations](/docs/build/snapshots#configuring-snapshots). You can also continue using the legacy method for existing snapshots, but we recommend migrating over to the latest YAML format for better readability and maintainability. diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md deleted file mode 100644 index 648b65752c0..00000000000 --- a/website/snippets/_snapshots-yaml-config.md +++ /dev/null @@ -1 +0,0 @@ -Define snapshots using the latest [YAML configurations](/docs/build/snapshots#configuring-snapshots), available in Versionless dbt Cloud and Core v1.9 and higher, in the `snapshot` directory file. From b84f4a5d97a09a428ce58bb94a5446d0fd6cbb28 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Thu, 28 Nov 2024 11:12:14 +0000 Subject: [PATCH 11/18] remove iimport --- website/docs/reference/snapshot-configs.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 4d8d8e42d26..1867c6f12f0 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -7,8 +7,6 @@ meta: import ConfigResource from '/snippets/_config-description-resource.md'; import ConfigGeneral from '/snippets/_config-description-general.md'; -import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; - ## Related documentation * [Snapshots](/docs/build/snapshots) From 915e396b920e0d89ebe66fe08834f381da17078a Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:41:55 +0000 Subject: [PATCH 12/18] Update snapshot-configs.md --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 1867c6f12f0..dfc2bd087f1 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -406,7 +406,7 @@ The following examples demonstrate how to configure snapshots using the `dbt_pro - You can also define some common configs in a snapshot's `config` block. We don't recommend this for a snapshot's required configuration, however. + You can also define some common configs in a snapshot's `config` block. However, we don't recommend this for a snapshot's required configuration. From 61d3792815ad46094c30293475811145a3483b21 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:03:50 +0000 Subject: [PATCH 13/18] Update snapshot-configs.md --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index dfc2bd087f1..4d9bcf8a049 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -119,7 +119,7 @@ snapshots: - +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. From c4918188decf94faa637bb578c275cc2099edb79 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Tue, 3 Dec 2024 16:48:29 +0000 Subject: [PATCH 14/18] add link --- .../docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md b/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md index 31153188978..dedb9c0175f 100644 --- a/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md +++ b/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md @@ -68,6 +68,8 @@ Beginning in dbt Core 1.9, we've streamlined snapshot configuration and added a Read more about [Snapshots meta fields](/docs/build/snapshots#snapshot-meta-fields). +To learn how to safely migrate existing snapshots, refer to [Snapshot configuration migration](/reference/snapshot-configs#snapshot-configuration-migration) for more information. + ### `state:modified` improvements We’ve made improvements to `state:modified` behaviors to help reduce the risk of false positives and negatives. Read more about [the `state:modified` behavior flag](#managing-changes-to-legacy-behaviors) that unlocks this improvement: From 8382836851c67f11d3dfbd2b313b5c1404a03735 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:04:57 +0000 Subject: [PATCH 15/18] Update website/docs/reference/snapshot-configs.md Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index c8709f8f157..2e1167f875e 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -20,7 +20,7 @@ import ConfigGeneral from '/snippets/_config-description-general.md'; -Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshot-configs#snapshot-specific-configurations). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. From a9d4cc535a523388c4eb5ca3f05282e1b122349e Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:38:39 +0000 Subject: [PATCH 16/18] Update website/docs/reference/snapshot-configs.md Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 2e1167f875e..2fa9a81be51 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -119,7 +119,7 @@ snapshots: -Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshot-configs#snapshot-specific-configurations). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. From 944fb50089537a1016a2ef78fa5d2836e7b68664 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Tue, 3 Dec 2024 18:26:39 +0000 Subject: [PATCH 17/18] doug's feedback --- website/docs/docs/build/snapshots.md | 2 ++ website/docs/reference/snapshot-configs.md | 24 +++++---------------- website/snippets/_legacy-snapshot-config.md | 4 ++++ 3 files changed, 11 insertions(+), 19 deletions(-) create mode 100644 website/snippets/_legacy-snapshot-config.md diff --git a/website/docs/docs/build/snapshots.md b/website/docs/docs/build/snapshots.md index 9a020c7c940..494255135d3 100644 --- a/website/docs/docs/build/snapshots.md +++ b/website/docs/docs/build/snapshots.md @@ -40,6 +40,7 @@ This order is now in the "shipped" state, but we've lost the information about w - To configure snapshots in versions 1.8 and earlier, refer to [Configure snapshots in versions 1.8 and earlier](#configure-snapshots-in-versions-18-and-earlier). These versions use an older syntax where snapshots are defined within a snapshot block in a `.sql` file, typically located in your `snapshots` directory. - Note that defining multiple resources in a single file can significantly slow down parsing and compilation. For faster and more efficient management, consider the updated snapshot YAML syntax, [available in Versionless](/docs/dbt-versions/versionless-cloud) or [dbt Core v1.9 and later](/docs/dbt-versions/core). + - For more information on how to migrate from the legacy snapshot configurations to the updated snapshot YAML syntax, refer to [Snapshot configuration migration](/reference/snapshot-configs#snapshot-configuration-migration). @@ -496,6 +497,7 @@ To configure snapshots in versions 1.9 and later, refer to [Configuring snapshot - In dbt versions 1.8 and earlier, snapshots are `select` statements, defined within a snapshot block in a `.sql` file (typically in your `snapshots` directory). You'll also need to configure your snapshot to tell dbt how to detect record changes. - The earlier dbt versions use an older syntax that allows for defining multiple resources in a single file. This syntax can significantly slow down parsing and compilation. - For faster and more efficient management, consider[ upgrading to Versionless](/docs/dbt-versions/versionless-cloud) or the [latest version of dbt Core](/docs/dbt-versions/core), which introduces an updated snapshot configuration syntax that optimizes performance. + - For more information on how to migrate from the legacy snapshot configurations to the updated snapshot YAML syntax, refer to [Snapshot configuration migration](/reference/snapshot-configs#snapshot-configuration-migration). The following example shows how to configure a snapshot: diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 2e1167f875e..1e52eb66290 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -18,12 +18,6 @@ import ConfigGeneral from '/snippets/_config-description-general.md'; - - -Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshot-configs#snapshot-specific-configurations). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. - - - - - -Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. +import LegacySnapshotConfig from '/snippets/_legacy-snapshot-config.md'; - - + @@ -264,11 +255,8 @@ snapshots: - + -Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [configurations](/docs/build/snapshots#configuring-snapshots). You can also continue using the legacy method for existing snapshots, but we recommend migrating over to the latest YAML format for better readability and maintainability. - - @@ -303,12 +291,10 @@ Snapshots can be configured in multiple ways: -1. Defined in YAML files using a `config` [resource property](/reference/model-properties), typically in your [snapshots directory](/reference/project-configs/snapshot-paths) (available in [Versionless](/docs/dbt-versions/versionless-cloud) or and dbt Core v1.9 and higher). -2. Using a `config` block within a snapshot defined in Jinja SQL +1. Defined in a YAML file using a `config` [resource property](/reference/model-properties), typically in your [snapshots directory](/reference/project-configs/snapshot-paths) (available in [Versionless](/docs/dbt-versions/versionless-cloud) or and dbt Core v1.9 and higher). The latest snapshot YAML syntax provides faster and more efficient management. +2. Using a `config` block within a snapshot defined in Jinja SQL. 3. From the `dbt_project.yml` file, under the `snapshots:` key. To apply a configuration to a snapshot, or directory of snapshots, define the resource path as nested dictionary keys. -Note that in Versionless and dbt v1.9 and later, snapshots are defined in an updated syntax using a YAML file within your `snapshots/` directory (as defined by the [`snapshot-paths` config](/reference/project-configs/snapshot-paths)). For faster and more efficient management, consider the updated snapshot YAML syntax, [available in Versionless](/docs/dbt-versions/versionless-cloud) or [dbt Core v1.9 and later](/docs/dbt-versions/core). - Snapshot configurations are applied hierarchically in the order above with higher taking precedence. diff --git a/website/snippets/_legacy-snapshot-config.md b/website/snippets/_legacy-snapshot-config.md new file mode 100644 index 00000000000..b44dc1c0ef8 --- /dev/null +++ b/website/snippets/_legacy-snapshot-config.md @@ -0,0 +1,4 @@ + +:::info +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#configuring-snapshots). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to [migrate](#snapshot-configuration-migration) over. +::: From a475180c4fe9d3ac2ac51e1751fe2d3de6929205 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Wed, 4 Dec 2024 09:34:28 +0000 Subject: [PATCH 18/18] tweaks --- .../dbt-versions/core-upgrade/06-upgrading-to-v1.9.md | 2 ++ website/docs/reference/snapshot-configs.md | 11 +++++++++-- website/snippets/_legacy-snapshot-config.md | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md b/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md index dedb9c0175f..b8efd2661a9 100644 --- a/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md +++ b/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md @@ -51,9 +51,11 @@ Starting in Core 1.9, you can use the new [microbatch strategy](/docs/build/incr Currently microbatch is supported on these adapters with more to come: * postgres + * redshift * snowflake * bigquery * spark + * databricks ### Snapshots improvements diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 1e52eb66290..9efab769018 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -138,7 +138,7 @@ import LegacySnapshotConfig from '/snippets/_legacy-snapshot-config.md'; ### Snapshot configuration migration -The latest snapshot configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names), [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current), and `hard_deletes`) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: +The latest snapshot configurations introduced in dbt Core v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names), [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current), and `hard_deletes`) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: #### For existing snapshots - Migrate tables — Migrate the previous snapshot to the new table schema and values: @@ -300,7 +300,14 @@ Snapshots can be configured in multiple ways: Snapshot configurations are applied hierarchically in the order above with higher taking precedence. ### Examples -The following examples demonstrate how to configure snapshots using the `dbt_project.yml` file, a `config` block within a snapshot, and a `.yml` file. + + +The following examples demonstrate how to configure snapshots using the `dbt_project.yml` file and a `.yml` file. + + + +The following examples demonstrate how to configure snapshots using the `dbt_project.yml` file, a `config` block within a snapshot (legacy method), and a `.yml` file. + - #### Apply configurations to all snapshots To apply a configuration to all snapshots, including those in any installed [packages](/docs/build/packages), nest the configuration directly under the `snapshots` key: diff --git a/website/snippets/_legacy-snapshot-config.md b/website/snippets/_legacy-snapshot-config.md index b44dc1c0ef8..a0091215b46 100644 --- a/website/snippets/_legacy-snapshot-config.md +++ b/website/snippets/_legacy-snapshot-config.md @@ -1,4 +1,4 @@ :::info -Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#configuring-snapshots). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to [migrate](#snapshot-configuration-migration) over. +Starting from Versionless and dbt Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#configuring-snapshots). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to [migrate](#snapshot-configuration-migration) over. :::