From 64fa03001ed94e111337c9011be8293b68843549 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Tue, 10 Dec 2024 12:58:35 -0800 Subject: [PATCH 1/8] add Ephemeral-write-only subproject into teamcity --- .../FEATURE-BRANCH-ephemeral-write-only.kt | 129 ++++++++++++++++++ .../components/projects/root_project.kt | 1 + .../FEATURE-BRANCH-ephemera-write-only.kt | 79 +++++++++++ 3 files changed, 209 insertions(+) create mode 100644 mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt create mode 100644 mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt new file mode 100644 index 000000000000..f43d0323d429 --- /dev/null +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -0,0 +1,129 @@ +/* + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ + +// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. + +package projects.feature_branches + +import ProviderNameBeta +import ProviderNameGa +import SharedResourceNameBeta +import SharedResourceNameGa +import SharedResourceNameVcr +import builds.* +import generated.ServicesListBeta +import generated.ServicesListGa +import jetbrains.buildServer.configs.kotlin.Project +import replaceCharsId +import vcs_roots.HashiCorpVCSRootBeta +import vcs_roots.HashiCorpVCSRootGa +import vcs_roots.ModularMagicianVCSRootBeta +import vcs_roots.ModularMagicianVCSRootGa + +const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" +const val EphemeralWriteOnlyTfCoreVersion = "1.10.0" // will be changed to 1.11.0 when the new ephemeral values feature is released in release candidates + +// featureBranchEphemeralWriteOnlySubProject creates a project just for testing ephemeral write-only attributes. +// We know that all ephemeral write-only attributes we're adding are part of the Resource Manager service, so we only include those builds. +// We create builds for testing the resourcemanager service: +// - Against the GA hashicorp repo +// - Against the GA modular-magician repo +// - Against the Beta hashicorp repo +// - Against the Beta modular-magician repo +// These resemble existing projects present in TeamCity, but these all use a more recent version of Terraform including +// the new ephemeral values feature. +fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project { + + val projectId = replaceCharsId(featureBranchEphemeralWriteOnly) + + val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs + val trigger = NightlyTriggerConfiguration( + branch = "refs/heads/$featureBranchEphemeralWriteOnly" // triggered builds must test the feature branch + ) + + // All Ephemeral Write-Only attributes are in the following packages + var PackagesListWriteOnly = mapOf( + "compute" to mapOf( + "name" to "compute", + "displayName" to "Compute", + "path" to "./google/services/compute" + ), + "secretmanager" to mapOf( + "name" to "secretmanager", + "displayName" to "Secretmanager", + "path" to "./google/services/secretmanager" + ), + "bigquerydatatransfer" to mapOf( + "name" to "bigquerydatatransfer", + "displayName" to "Bigquerydatatransfer", + "path" to "./google/services/bigquerydatatransfer" + ), + "sql" to mapOf( + "name" to "sql", + "displayName" to "Sql", + "path" to "./google/services/sql" + ), + ) + + // GA + var parentId = "${projectId}_HC_GA" + val buildConfigHashiCorpGa = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameGa, parentId, vcsRoot, listOf(SharedResourceNameGa), config) + packageBuildConfigs.forEach { buildConfiguration -> + buildConfiguration.addTrigger(cron) + } + + var parentId = "${projectId}_MM_GA" + val buildConfigModularMagicianGa = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameGa, parentId, vcsRoot, listOf(SharedResourceNameGa), config) + // No trigger added here (MM upstream is manual only) + + // Beta + parentId = "${projectId}_HC_Beta" + val buildConfigHashiCorpBeta = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameBeta, parentId, vcsRoot, listOf(SharedResourceNameBeta), config) + buildConfigHashiCorpBeta.forEach { buildConfiguration -> + buildConfiguration.addTrigger(cron) + } + + parentId = "${projectId}_MM_Beta" + val buildConfigModularMagicianBeta = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameBeta, parentId, vcsRoot, listOf(SharedResourceNameBeta), config) + // No trigger added here (MM upstream is manual only) + + // Create build config for sweeping the ephemeral write-only project + var sweepersList: Map> + when(providerName) { + ProviderNameGa -> sweepersList = SweepersListGa + ProviderNameBeta -> sweepersList = SweepersListBeta + else -> throw Exception("Provider name not supplied when generating a nightly test subproject") + } + val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config) + val sweeperCron = cron.clone() + sweeperCron.startHour += 5 // Ensure triggered after the package test builds are triggered + serviceSweeperConfig.addTrigger(sweeperCron) + + // ------ + + // Make all builds use a 1.11.0 version of TF core + val allBuildConfigs = listOf(buildConfigHashiCorpGa, buildConfigModularMagicianGa, buildConfigHashiCorpBeta, buildConfigModularMagicianBeta) + allBuildConfigs.forEach{ b -> + b.overrideTerraformCoreVersion(EphemeralWriteOnlyTfCoreVersion) + } + + // ------ + + return Project{ + id(projectId) + name = featureBranchEphemeralWriteOnly + description = "Subproject for testing feature branch $featureBranchEphemeralWriteOnly" + + // Register build configs in the project + packageBuildConfigs.forEach { buildConfiguration -> + buildType(buildConfiguration) + } + buildType(serviceSweeperConfig) + + params { + readOnlySettings() + } + } +} diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt b/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt index c551f8f20fd2..4055c8644418 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt @@ -61,6 +61,7 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project { subProject(googleSubProjectGa(allConfig)) subProject(googleSubProjectBeta(allConfig)) subProject(projectSweeperSubProject(allConfig)) + subProject(featureBranchEphemeralWriteOnlySubProject(allConfig)) // Feature branch-testing projects - these will be added and removed as needed diff --git a/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt b/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt new file mode 100644 index 000000000000..3c4a5bc707ad --- /dev/null +++ b/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ + +// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. + +package tests + +import jetbrains.buildServer.configs.kotlin.triggers.ScheduleTrigger +import org.junit.Assert +import org.junit.Test +import projects.feature_branches.featureBranchEphemeralResources +import projects.googleCloudRootProject + +class FeatureBranchEphemeralWriteOnlySubProject { + @Test + fun buildsUsingHashiCorpReposAreOnSchedule() { + val root = googleCloudRootProject(testContextParameters()) + + // Find feature branch project + val project = getSubProject(root, featureBranchEphemeralResources) + + // All builds using the HashiCorp owned GitHub repos + val hashiBuilds = project.buildTypes.filter { bt -> + bt.name.contains("HashiCorp downstream") + } + + hashiBuilds.forEach{bt -> + Assert.assertTrue( + "Build configuration `${bt.name}` should contain at least one trigger", + bt.triggers.items.isNotEmpty() + ) + // Look for at least one CRON trigger + var found = false + lateinit var schedulingTrigger: ScheduleTrigger + for (item in bt.triggers.items){ + if (item.type == "schedulingTrigger") { + schedulingTrigger = item as ScheduleTrigger + found = true + break + } + } + + Assert.assertTrue( + "Build configuration `${bt.name}` should contain a CRON/'schedulingTrigger' trigger", + found + ) + + // Check that triggered builds are being run on the feature branch + val isCorrectBranch: Boolean = schedulingTrigger.branchFilter == "+:refs/heads/$featureBranchEphemeralResources" + + Assert.assertTrue( + "Build configuration `${bt.name}` is using the $featureBranchEphemeralResources branch filter", + isCorrectBranch + ) + } + } + + @Test + fun buildsUsingModularMagicianReposAreNotTriggered() { + val root = googleCloudRootProject(testContextParameters()) + + // Find feature branch project + val project = getSubProject(root, featureBranchEphemeralResources) + + // All builds using the HashiCorp owned GitHub repos + val magicianBuilds = project.buildTypes.filter { bt -> + bt.name.contains("MM upstream") + } + + magicianBuilds.forEach{bt -> + Assert.assertTrue( + "Build configuration `${bt.name}` should not have any triggers", + bt.triggers.items.isEmpty() + ) + } + } +} From ea46802e291a6cdbb9a0cf242629346553016b00 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Tue, 10 Dec 2024 22:41:08 -0800 Subject: [PATCH 2/8] fix teamcity config --- .../FEATURE-BRANCH-ephemeral-write-only.kt | 156 +++++++++++------- .../FEATURE-BRANCH-ephemera-write-only.kt | 10 +- 2 files changed, 101 insertions(+), 65 deletions(-) diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt index f43d0323d429..5bf234ebdd85 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -23,28 +23,19 @@ import vcs_roots.ModularMagicianVCSRootBeta import vcs_roots.ModularMagicianVCSRootGa const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" -const val EphemeralWriteOnlyTfCoreVersion = "1.10.0" // will be changed to 1.11.0 when the new ephemeral values feature is released in release candidates - -// featureBranchEphemeralWriteOnlySubProject creates a project just for testing ephemeral write-only attributes. -// We know that all ephemeral write-only attributes we're adding are part of the Resource Manager service, so we only include those builds. -// We create builds for testing the resourcemanager service: -// - Against the GA hashicorp repo -// - Against the GA modular-magician repo -// - Against the Beta hashicorp repo -// - Against the Beta modular-magician repo -// These resemble existing projects present in TeamCity, but these all use a more recent version of Terraform including -// the new ephemeral values feature. -fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project { +const val EphemeralWriteOnlyTfCoreVersion = "1.10.0" - val projectId = replaceCharsId(featureBranchEphemeralWriteOnly) +fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project { - val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs val trigger = NightlyTriggerConfiguration( branch = "refs/heads/$featureBranchEphemeralWriteOnly" // triggered builds must test the feature branch ) + val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs - // All Ephemeral Write-Only attributes are in the following packages - var PackagesListWriteOnly = mapOf( + // GA + val gaConfig = getGaAcceptanceTestConfig(allConfig) + // These are the packages that have resources that will use write-only attributes + var ServicesListWriteOnlyGa = mapOf( "compute" to mapOf( "name" to "compute", "displayName" to "Compute", @@ -55,75 +46,120 @@ fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): "displayName" to "Secretmanager", "path" to "./google/services/secretmanager" ), - "bigquerydatatransfer" to mapOf( - "name" to "bigquerydatatransfer", - "displayName" to "Bigquerydatatransfer", - "path" to "./google/services/bigquerydatatransfer" - ), "sql" to mapOf( "name" to "sql", "displayName" to "Sql", "path" to "./google/services/sql" ), + "bigquery_datatransfer" to mapOf( + "name" to "bigquery_datatransfer", + "displayName" to "Bigquery Datatransfer", + "path" to "./google/services/bigquery_datatransfer" + ) ) - // GA - var parentId = "${projectId}_HC_GA" - val buildConfigHashiCorpGa = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameGa, parentId, vcsRoot, listOf(SharedResourceNameGa), config) - packageBuildConfigs.forEach { buildConfiguration -> - buildConfiguration.addTrigger(cron) + val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGa, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig) + buildConfigsGa.forEach{ builds -> + builds.addTrigger(trigger) } - var parentId = "${projectId}_MM_GA" - val buildConfigModularMagicianGa = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameGa, parentId, vcsRoot, listOf(SharedResourceNameGa), config) - // No trigger added here (MM upstream is manual only) + var ServicesListWriteOnlyGaMM = mapOf( + "compute" to mapOf( + "name" to "compute", + "displayName" to "Compute - MM", + "path" to "./google/services/compute" + ), + "secretmanager" to mapOf( + "name" to "secretmanager", + "displayName" to "Secretmanager - MM", + "path" to "./google/services/secretmanager" + ), + "sql" to mapOf( + "name" to "sql", + "displayName" to "Sql - MM", + "path" to "./google/services/sql" + ), + "bigquery_datatransfer" to mapOf( + "name" to "bigquery_datatransfer", + "displayName" to "Bigquery Datatransfer - MM", + "path" to "./google/services/bigquery_datatransfer" + ) + ) + val buildConfigsMMGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGaMM, ProviderNameGa, "EphemeralWriteOnlyGa - MM", ModularMagicianVCSRootGa, listOf(SharedResourceNameGa), vcrConfig) // Beta - parentId = "${projectId}_HC_Beta" - val buildConfigHashiCorpBeta = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameBeta, parentId, vcsRoot, listOf(SharedResourceNameBeta), config) - buildConfigHashiCorpBeta.forEach { buildConfiguration -> - buildConfiguration.addTrigger(cron) - } - - parentId = "${projectId}_MM_Beta" - val buildConfigModularMagicianBeta = BuildConfigurationsForPackages(PackagesListWriteOnly, ProviderNameBeta, parentId, vcsRoot, listOf(SharedResourceNameBeta), config) - // No trigger added here (MM upstream is manual only) - - // Create build config for sweeping the ephemeral write-only project - var sweepersList: Map> - when(providerName) { - ProviderNameGa -> sweepersList = SweepersListGa - ProviderNameBeta -> sweepersList = SweepersListBeta - else -> throw Exception("Provider name not supplied when generating a nightly test subproject") + val betaConfig = getBetaAcceptanceTestConfig(allConfig) + var ServicesListWriteOnlyBeta = mapOf( + "compute" to mapOf( + "name" to "compute", + "displayName" to "Compute - Beta", + "path" to "./google-beta/services/compute" + ), + "secretmanager" to mapOf( + "name" to "secretmanager", + "displayName" to "Secretmanager - Beta", + "path" to "./google-beta/services/secretmanager" + ), + "sql" to mapOf( + "name" to "sql", + "displayName" to "Sql - Beta", + "path" to "./google-beta/services/sql" + ), + "bigquery_datatransfer" to mapOf( + "name" to "bigquery_datatransfer", + "displayName" to "Bigquery Datatransfer - Beta", + "path" to "./google-beta/services/bigquery_datatransfer" + ) + ) + val buildConfigsBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBeta, ProviderNameBeta, "EphemeralWriteOnlyBeta - HC", HashiCorpVCSRootBeta, listOf(SharedResourceNameBeta), betaConfig) + buildConfigsBeta.forEach{ builds -> + builds.addTrigger(trigger) } - val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config) - val sweeperCron = cron.clone() - sweeperCron.startHour += 5 // Ensure triggered after the package test builds are triggered - serviceSweeperConfig.addTrigger(sweeperCron) - // ------ + var ServicesListWriteOnlyBetaMM = mapOf( + "compute" to mapOf( + "name" to "compute", + "displayName" to "Compute - Beta - MM", + "path" to "./google-beta/services/compute" + ), + "secretmanager" to mapOf( + "name" to "secretmanager", + "displayName" to "Secretmanager - Beta - MM", + "path" to "./google-beta/services/secretmanager" + ), + "sql" to mapOf( + "name" to "sql", + "displayName" to "Sql - Beta - MM", + "path" to "./google-beta/services/sql" + ), + "bigquery_datatransfer" to mapOf( + "name" to "bigquery_datatransfer", + "displayName" to "Bigquery Datatransfer - Beta - MM", + "path" to "./google-beta/services/bigquery_datatransfer" + ) + ) + val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig) - // Make all builds use a 1.11.0 version of TF core - val allBuildConfigs = listOf(buildConfigHashiCorpGa, buildConfigModularMagicianGa, buildConfigHashiCorpBeta, buildConfigModularMagicianBeta) - allBuildConfigs.forEach{ b -> - b.overrideTerraformCoreVersion(EphemeralWriteOnlyTfCoreVersion) + // Make all builds use a 1.10.0-ish version of TF core + val allBuildConfigs = buildConfigsGa + buildConfigsBeta + buildConfigsMMGa + buildConfigsMMBeta + allBuildConfigs.forEach{ builds -> + builds.overrideTerraformCoreVersion(EphemeralWriteOnlyTfCoreVersion) } // ------ return Project{ - id(projectId) + id("FEATURE_BRANCH_ephemeral_write_only") name = featureBranchEphemeralWriteOnly description = "Subproject for testing feature branch $featureBranchEphemeralWriteOnly" - // Register build configs in the project - packageBuildConfigs.forEach { buildConfiguration -> - buildType(buildConfiguration) + // Register all build configs in the project + allBuildConfigs.forEach{ builds -> + buildType(builds) } - buildType(serviceSweeperConfig) params { readOnlySettings() } } -} +} \ No newline at end of file diff --git a/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt b/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt index 3c4a5bc707ad..643050e0e0d6 100644 --- a/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt +++ b/mmv1/third_party/terraform/.teamcity/tests/FEATURE-BRANCH-ephemera-write-only.kt @@ -10,7 +10,7 @@ package tests import jetbrains.buildServer.configs.kotlin.triggers.ScheduleTrigger import org.junit.Assert import org.junit.Test -import projects.feature_branches.featureBranchEphemeralResources +import projects.feature_branches.featureBranchEphemeralWriteOnly import projects.googleCloudRootProject class FeatureBranchEphemeralWriteOnlySubProject { @@ -19,7 +19,7 @@ class FeatureBranchEphemeralWriteOnlySubProject { val root = googleCloudRootProject(testContextParameters()) // Find feature branch project - val project = getSubProject(root, featureBranchEphemeralResources) + val project = getSubProject(root, featureBranchEphemeralWriteOnly ) // All builds using the HashiCorp owned GitHub repos val hashiBuilds = project.buildTypes.filter { bt -> @@ -48,10 +48,10 @@ class FeatureBranchEphemeralWriteOnlySubProject { ) // Check that triggered builds are being run on the feature branch - val isCorrectBranch: Boolean = schedulingTrigger.branchFilter == "+:refs/heads/$featureBranchEphemeralResources" + val isCorrectBranch: Boolean = schedulingTrigger.branchFilter == "+:refs/heads/$featureBranchEphemeralWriteOnly" Assert.assertTrue( - "Build configuration `${bt.name}` is using the $featureBranchEphemeralResources branch filter", + "Build configuration `${bt.name}` is using the $featureBranchEphemeralWriteOnly branch filter", isCorrectBranch ) } @@ -62,7 +62,7 @@ class FeatureBranchEphemeralWriteOnlySubProject { val root = googleCloudRootProject(testContextParameters()) // Find feature branch project - val project = getSubProject(root, featureBranchEphemeralResources) + val project = getSubProject(root, featureBranchEphemeralWriteOnly) // All builds using the HashiCorp owned GitHub repos val magicianBuilds = project.buildTypes.filter { bt -> From d909d01e8d1bb2cd3f8342c02e096c7885551afd Mon Sep 17 00:00:00 2001 From: BBBmau Date: Mon, 16 Dec 2024 18:39:26 -0800 Subject: [PATCH 3/8] use 1.11 alpha --- .../feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt index 5bf234ebdd85..503baf8e6c8e 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -23,7 +23,7 @@ import vcs_roots.ModularMagicianVCSRootBeta import vcs_roots.ModularMagicianVCSRootGa const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" -const val EphemeralWriteOnlyTfCoreVersion = "1.10.0" +const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211" fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project { @@ -140,7 +140,7 @@ fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): ) val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig) - // Make all builds use a 1.10.0-ish version of TF core + // Make all builds use a 1.11.0-ish version of TF core val allBuildConfigs = buildConfigsGa + buildConfigsBeta + buildConfigsMMGa + buildConfigsMMBeta allBuildConfigs.forEach{ builds -> builds.overrideTerraformCoreVersion(EphemeralWriteOnlyTfCoreVersion) From dadf0d364650ce087db4b13f36daf91cdb88b776 Mon Sep 17 00:00:00 2001 From: BBBmau Date: Mon, 16 Dec 2024 20:33:47 -0800 Subject: [PATCH 4/8] rewrite to use getServices --- .../FEATURE-BRANCH-ephemeral-write-only.kt | 95 ++----------------- 1 file changed, 6 insertions(+), 89 deletions(-) diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt index 503baf8e6c8e..e5768d653467 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -21,6 +21,7 @@ import vcs_roots.HashiCorpVCSRootBeta import vcs_roots.HashiCorpVCSRootGa import vcs_roots.ModularMagicianVCSRootBeta import vcs_roots.ModularMagicianVCSRootGa +import getServicesList const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211" @@ -35,109 +36,25 @@ fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): // GA val gaConfig = getGaAcceptanceTestConfig(allConfig) // These are the packages that have resources that will use write-only attributes - var ServicesListWriteOnlyGa = mapOf( - "compute" to mapOf( - "name" to "compute", - "displayName" to "Compute", - "path" to "./google/services/compute" - ), - "secretmanager" to mapOf( - "name" to "secretmanager", - "displayName" to "Secretmanager", - "path" to "./google/services/secretmanager" - ), - "sql" to mapOf( - "name" to "sql", - "displayName" to "Sql", - "path" to "./google/services/sql" - ), - "bigquery_datatransfer" to mapOf( - "name" to "bigquery_datatransfer", - "displayName" to "Bigquery Datatransfer", - "path" to "./google/services/bigquery_datatransfer" - ) - ) + var ServicesListWriteOnlyGA = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA") - val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGa, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig) + val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGA, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig) buildConfigsGa.forEach{ builds -> builds.addTrigger(trigger) } - var ServicesListWriteOnlyGaMM = mapOf( - "compute" to mapOf( - "name" to "compute", - "displayName" to "Compute - MM", - "path" to "./google/services/compute" - ), - "secretmanager" to mapOf( - "name" to "secretmanager", - "displayName" to "Secretmanager - MM", - "path" to "./google/services/secretmanager" - ), - "sql" to mapOf( - "name" to "sql", - "displayName" to "Sql - MM", - "path" to "./google/services/sql" - ), - "bigquery_datatransfer" to mapOf( - "name" to "bigquery_datatransfer", - "displayName" to "Bigquery Datatransfer - MM", - "path" to "./google/services/bigquery_datatransfer" - ) - ) + var ServicesListWriteOnlyGaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA-MM") val buildConfigsMMGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGaMM, ProviderNameGa, "EphemeralWriteOnlyGa - MM", ModularMagicianVCSRootGa, listOf(SharedResourceNameGa), vcrConfig) // Beta val betaConfig = getBetaAcceptanceTestConfig(allConfig) - var ServicesListWriteOnlyBeta = mapOf( - "compute" to mapOf( - "name" to "compute", - "displayName" to "Compute - Beta", - "path" to "./google-beta/services/compute" - ), - "secretmanager" to mapOf( - "name" to "secretmanager", - "displayName" to "Secretmanager - Beta", - "path" to "./google-beta/services/secretmanager" - ), - "sql" to mapOf( - "name" to "sql", - "displayName" to "Sql - Beta", - "path" to "./google-beta/services/sql" - ), - "bigquery_datatransfer" to mapOf( - "name" to "bigquery_datatransfer", - "displayName" to "Bigquery Datatransfer - Beta", - "path" to "./google-beta/services/bigquery_datatransfer" - ) - ) + var ServicesListWriteOnlyBeta = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta") val buildConfigsBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBeta, ProviderNameBeta, "EphemeralWriteOnlyBeta - HC", HashiCorpVCSRootBeta, listOf(SharedResourceNameBeta), betaConfig) buildConfigsBeta.forEach{ builds -> builds.addTrigger(trigger) } - var ServicesListWriteOnlyBetaMM = mapOf( - "compute" to mapOf( - "name" to "compute", - "displayName" to "Compute - Beta - MM", - "path" to "./google-beta/services/compute" - ), - "secretmanager" to mapOf( - "name" to "secretmanager", - "displayName" to "Secretmanager - Beta - MM", - "path" to "./google-beta/services/secretmanager" - ), - "sql" to mapOf( - "name" to "sql", - "displayName" to "Sql - Beta - MM", - "path" to "./google-beta/services/sql" - ), - "bigquery_datatransfer" to mapOf( - "name" to "bigquery_datatransfer", - "displayName" to "Bigquery Datatransfer - Beta - MM", - "path" to "./google-beta/services/bigquery_datatransfer" - ) - ) + var ServicesListWriteOnlyBetaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta-MM") val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig) // Make all builds use a 1.11.0-ish version of TF core From e8004751ce297cbc9d26cbdd196fa5fc27088e0f Mon Sep 17 00:00:00 2001 From: BBBmau Date: Mon, 16 Dec 2024 20:42:59 -0800 Subject: [PATCH 5/8] add getServices.kt --- .../.teamcity/components/get_services.kt | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 mmv1/third_party/terraform/.teamcity/components/get_services.kt diff --git a/mmv1/third_party/terraform/.teamcity/components/get_services.kt b/mmv1/third_party/terraform/.teamcity/components/get_services.kt new file mode 100644 index 000000000000..98ee3c3f7f92 --- /dev/null +++ b/mmv1/third_party/terraform/.teamcity/components/get_services.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ +import generated.ServicesListGa +import generated.ServicesListBeta + +// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. + +fun getServicesList(Services: Array, version: String): Map> { + if (Services.isEmpty()) { + throw Exception("No services found for version $version") + } + + var servicesList = mutableMapOf>() + for (service in Services) { + if (version == "GA" || version == "GA-MM") { + servicesList[service] = ServicesListGa.getOrElse(service) { throw Exception("Service $service not found") } + } else if (version == "Beta" || version == "Beta-MM") { + servicesList[service] = ServicesListBeta.getOrElse(service) { throw Exception("Service $service not found") } + } else { + throw Exception("Invalid version $version") + } + } + + when (version) { + "GA" -> return servicesList + "Beta" -> { + servicesList = servicesList.mapValues { (_, value) -> + value + mapOf( + "displayName" to "${value["displayName"]} - Beta", + "path" to (value["path"]?.replace("./google/", "./google-beta/") ?: "") + ) + }.toMutableMap() + } + "GA-MM" -> { + servicesList = servicesList.mapValues { (_, value) -> + value + mapOf( + "displayName" to "${value["displayName"]} - MM" + ) + }.toMutableMap() + } + "Beta-MM" -> { + servicesList = servicesList.mapValues { (_, value) -> + value + mapOf( + "displayName" to "${value["displayName"]} - Beta - MM", + "path" to (value["path"]?.replace("./google-beta/", "./google-beta/services/") ?: "") + ) + }.toMutableMap() + } + else -> throw Exception("Invalid version $version") + } + + return servicesList +} \ No newline at end of file From cc295705795301154a31920424d26801d7673d5c Mon Sep 17 00:00:00 2001 From: BBBmau Date: Mon, 16 Dec 2024 20:55:07 -0800 Subject: [PATCH 6/8] root_project add --- .../terraform/.teamcity/components/projects/root_project.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt b/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt index 4055c8644418..1daac200f2f0 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt @@ -18,6 +18,7 @@ import generated.ServicesListBeta import generated.ServicesListGa import jetbrains.buildServer.configs.kotlin.Project import jetbrains.buildServer.configs.kotlin.sharedResource +import projects.feature_branches.featureBranchEphemeralWriteOnlySubProject // googleCloudRootProject returns a root project that contains a subprojects for the GA and Beta version of the // Google provider. There are also resources to help manage the test projects used for acceptance tests. From 18aa211ebc699c2de37d151db82bdb089599a52a Mon Sep 17 00:00:00 2001 From: BBBmau Date: Tue, 17 Dec 2024 09:24:55 -0800 Subject: [PATCH 7/8] remove unneccessary transformation of paths in get_services.kt --- .../FEATURE-BRANCH-ephemeral-write-only.kt | 2 +- .../feature_branches}/get_services.kt | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) rename mmv1/third_party/terraform/.teamcity/components/{ => projects/feature_branches}/get_services.kt (68%) diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt index e5768d653467..44af9a7dccb3 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -21,7 +21,7 @@ import vcs_roots.HashiCorpVCSRootBeta import vcs_roots.HashiCorpVCSRootGa import vcs_roots.ModularMagicianVCSRootBeta import vcs_roots.ModularMagicianVCSRootGa -import getServicesList +import components.projects.feature_branches.getServicesList const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211" diff --git a/mmv1/third_party/terraform/.teamcity/components/get_services.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/get_services.kt similarity index 68% rename from mmv1/third_party/terraform/.teamcity/components/get_services.kt rename to mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/get_services.kt index 98ee3c3f7f92..14b98a1848f9 100644 --- a/mmv1/third_party/terraform/.teamcity/components/get_services.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/get_services.kt @@ -2,11 +2,14 @@ * Copyright (c) HashiCorp, Inc. * SPDX-License-Identifier: MPL-2.0 */ +package components.projects.feature_branches + import generated.ServicesListGa import generated.ServicesListBeta // This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten. +// This function is used to get the services list for a given version. Typically used in feature branch builds for testing very specific services only. fun getServicesList(Services: Array, version: String): Map> { if (Services.isEmpty()) { throw Exception("No services found for version $version") @@ -24,32 +27,30 @@ fun getServicesList(Services: Array, version: String): Map return servicesList + "GA" -> servicesList "Beta" -> { - servicesList = servicesList.mapValues { (_, value) -> + servicesList.mapValues { (_, value) -> value + mapOf( - "displayName" to "${value["displayName"]} - Beta", - "path" to (value["path"]?.replace("./google/", "./google-beta/") ?: "") + "displayName" to "${value["displayName"]} - Beta" ) }.toMutableMap() } "GA-MM" -> { - servicesList = servicesList.mapValues { (_, value) -> + servicesList.mapValues { (_, value) -> value + mapOf( - "displayName" to "${value["displayName"]} - MM" + "displayName" to "${value["displayName"]} - MM" ) }.toMutableMap() } "Beta-MM" -> { - servicesList = servicesList.mapValues { (_, value) -> + servicesList.mapValues { (_, value) -> value + mapOf( - "displayName" to "${value["displayName"]} - Beta - MM", - "path" to (value["path"]?.replace("./google-beta/", "./google-beta/services/") ?: "") + "displayName" to "${value["displayName"]} - Beta - MM" ) }.toMutableMap() } else -> throw Exception("Invalid version $version") - } - + }.also { servicesList = it as MutableMap> } + return servicesList } \ No newline at end of file From aba9b898b4bd491c86ea2da181ec467a5d50605c Mon Sep 17 00:00:00 2001 From: BBBmau Date: Tue, 17 Dec 2024 10:48:14 -0800 Subject: [PATCH 8/8] add startHour and support for custom testprefix in buildConfigurations --- .../builds/build_configuration_per_package.kt | 11 +++++------ .../FEATURE-BRANCH-ephemeral-write-only.kt | 12 +++++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt index 0e9dcbfc44c3..8971a41a0372 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt @@ -20,7 +20,7 @@ import replaceCharsId // BuildConfigurationsForPackages accepts a map containing details of multiple packages in a provider and returns a list of build configurations for them all. // Intended to be used in projects where we're testing all packages, e.g. the nightly test projects -fun BuildConfigurationsForPackages(packages: Map>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration): List { +fun BuildConfigurationsForPackages(packages: Map>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): List { val list = ArrayList() // Create build configurations for all packages, except sweeper @@ -29,7 +29,7 @@ fun BuildConfigurationsForPackages(packages: Map>, p val displayName: String = info.getValue("displayName").toString() val pkg = PackageDetails(packageName, displayName, providerName, parentProjectName) - val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables) + val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix) list.add(buildConfig) } @@ -38,18 +38,17 @@ fun BuildConfigurationsForPackages(packages: Map>, p // BuildConfigurationForSinglePackage accepts details of a single package in a provider and returns a build configuration for it // Intended to be used in short-lived projects where we're testing specific packages, e.g. feature branch testing -fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration): BuildType{ +fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): BuildType{ val pkg = PackageDetails(packageName, packageDisplayName, providerName, parentProjectName) - return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables) + return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix) } class PackageDetails(private val packageName: String, private val displayName: String, private val providerName: String, private val parentProjectName: String) { // buildConfiguration returns a BuildType for a service package // For BuildType docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/build-type/index.html - fun buildConfiguration(path: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, buildTimeout: Int = DefaultBuildTimeoutDuration): BuildType { + fun buildConfiguration(path: String, vcsRoot: GitVcsRoot, sharedResources: List, environmentVariables: AccTestConfiguration, buildTimeout: Int = DefaultBuildTimeoutDuration, testPrefix: String): BuildType { - val testPrefix = "TestAcc" val testTimeout = "12" var parallelism = DefaultParallelism diff --git a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt index 44af9a7dccb3..249e8eb517a9 100644 --- a/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt +++ b/mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-ephemeral-write-only.kt @@ -22,6 +22,7 @@ import vcs_roots.HashiCorpVCSRootGa import vcs_roots.ModularMagicianVCSRootBeta import vcs_roots.ModularMagicianVCSRootGa import components.projects.feature_branches.getServicesList +import DefaultStartHour const val featureBranchEphemeralWriteOnly = "FEATURE-BRANCH-ephemeral-write-only" const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211" @@ -29,7 +30,8 @@ const val EphemeralWriteOnlyTfCoreVersion = "1.11.0-alpha20241211" fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): Project { val trigger = NightlyTriggerConfiguration( - branch = "refs/heads/$featureBranchEphemeralWriteOnly" // triggered builds must test the feature branch + branch = "refs/heads/$featureBranchEphemeralWriteOnly", // triggered builds must test the feature branch + startHour = DefaultStartHour + 6, ) val vcrConfig = getVcrAcceptanceTestConfig(allConfig) // Reused below for both MM testing build configs @@ -38,24 +40,24 @@ fun featureBranchEphemeralWriteOnlySubProject(allConfig: AllContextParameters): // These are the packages that have resources that will use write-only attributes var ServicesListWriteOnlyGA = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA") - val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGA, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig) + val buildConfigsGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGA, ProviderNameGa, "EphemeralWriteOnlyGa - HC", HashiCorpVCSRootGa, listOf(SharedResourceNameGa), gaConfig, "TestAcc.*Ephemeral") buildConfigsGa.forEach{ builds -> builds.addTrigger(trigger) } var ServicesListWriteOnlyGaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "GA-MM") - val buildConfigsMMGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGaMM, ProviderNameGa, "EphemeralWriteOnlyGa - MM", ModularMagicianVCSRootGa, listOf(SharedResourceNameGa), vcrConfig) + val buildConfigsMMGa = BuildConfigurationsForPackages(ServicesListWriteOnlyGaMM, ProviderNameGa, "EphemeralWriteOnlyGa - MM", ModularMagicianVCSRootGa, listOf(SharedResourceNameGa), vcrConfig, "TestAcc.*Ephemeral") // Beta val betaConfig = getBetaAcceptanceTestConfig(allConfig) var ServicesListWriteOnlyBeta = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta") - val buildConfigsBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBeta, ProviderNameBeta, "EphemeralWriteOnlyBeta - HC", HashiCorpVCSRootBeta, listOf(SharedResourceNameBeta), betaConfig) + val buildConfigsBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBeta, ProviderNameBeta, "EphemeralWriteOnlyBeta - HC", HashiCorpVCSRootBeta, listOf(SharedResourceNameBeta), betaConfig, "TestAcc.*Ephemeral") buildConfigsBeta.forEach{ builds -> builds.addTrigger(trigger) } var ServicesListWriteOnlyBetaMM = getServicesList(arrayOf("compute", "secretmanager", "sql", "bigquerydatatransfer"), "Beta-MM") - val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig) + val buildConfigsMMBeta = BuildConfigurationsForPackages(ServicesListWriteOnlyBetaMM, ProviderNameBeta, "EphemeralWriteOnlyBeta - MM", ModularMagicianVCSRootBeta, listOf(SharedResourceNameBeta), vcrConfig, "TestAcc.*Ephemeral") // Make all builds use a 1.11.0-ish version of TF core val allBuildConfigs = buildConfigsGa + buildConfigsBeta + buildConfigsMMGa + buildConfigsMMBeta