TeamCity v6.0.0 feature branch testing (GoogleCloudPlatform#11104)

mmv1/third_party/terraform/.teamcity/components/builds/build_triggers.kt

@@ -18,10 +18,20 @@ import jetbrains.buildServer.configs.kotlin.triggers.schedule
 class NightlyTriggerConfiguration(
     val branch: String = DefaultBranchName,
     val nightlyTestsEnabled: Boolean = true,
-    val startHour: Int = DefaultStartHour,
-    val daysOfWeek: String = DefaultDaysOfWeek,
+    var startHour: Int = DefaultStartHour,
+    var daysOfWeek: String = DefaultDaysOfWeek,
     val daysOfMonth: String = DefaultDaysOfMonth
-)
+){
+    fun clone(): NightlyTriggerConfiguration{
+        return NightlyTriggerConfiguration(
+            this.branch,
+            this.nightlyTestsEnabled,
+            this.startHour,
+            this.daysOfWeek,
+            this.daysOfMonth
+        )
+    }
+}
 
 fun Triggers.runNightly(config: NightlyTriggerConfiguration) {
 

mmv1/third_party/terraform/.teamcity/components/projects/feature_branches/FEATURE-BRANCH-major-release-6.0.0.kt

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+// This file is controlled by MMv1, any changes made here will be overwritten
+
+package projects.feature_branches
+
+import ProviderNameBeta
+import ProviderNameGa
+import builds.*
+import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
+import projects.reused.nightlyTests
+import replaceCharsId
+
+const val branchName = "FEATURE-BRANCH-major-release-6.0.0"
+
+// VCS Roots specifically for pulling code from the feature branches in the downstream repos
+
+object HashicorpVCSRootGa_featureBranchMajorRelease600: GitVcsRoot({
+    name = "VCS root for the hashicorp/terraform-provider-${ProviderNameGa} repo @ refs/heads/${branchName}"
+    url = "https://github.com/hashicorp/terraform-provider-${ProviderNameGa}"
+    branch = "refs/heads/${branchName}"
+    branchSpec = """
+        +:FEATURE-BRANCH-major-release-6*
+    """.trimIndent()
+})
+
+object HashicorpVCSRootBeta_featureBranchMajorRelease600: GitVcsRoot({
+    name = "VCS root for the hashicorp/terraform-provider-${ProviderNameBeta} repo @ refs/heads/${branchName}"
+    url = "https://github.com/hashicorp/terraform-provider-${ProviderNameBeta}"
+    branch = "refs/heads/${branchName}"
+    branchSpec = """
+        +:FEATURE-BRANCH-major-release-6*
+    """.trimIndent()
+})
+
+fun featureBranchMajorRelease600_Project(allConfig: AllContextParameters): Project {
+
+    val projectId = replaceCharsId(branchName)
+    val gaProjectId = replaceCharsId(projectId + "_GA")
+    val betaProjectId= replaceCharsId(projectId + "_BETA")
+
+    // Get config for using the GA and Beta identities
+    val gaConfig = getGaAcceptanceTestConfig(allConfig)
+    val betaConfig = getBetaAcceptanceTestConfig(allConfig)
+
+    return Project{
+        id(projectId)
+        name = "6.0.0 Major Release Testing"
+        description = "Subproject for testing feature branch $branchName"
+
+        // Register feature branch-specific VCS roots in the project
+        vcsRoot(HashicorpVCSRootGa_featureBranchMajorRelease600)
+        vcsRoot(HashicorpVCSRootBeta_featureBranchMajorRelease600)
+
+        // Nested Nightly Test project that uses hashicorp/terraform-provider-google
+        subProject(
+            Project{
+                id(gaProjectId)
+                name = "Google"
+                subProject(
+                    nightlyTests(
+                        gaProjectId,
+                        ProviderNameGa,
+                        HashicorpVCSRootGa_featureBranchMajorRelease600,
+                        gaConfig,
+                        NightlyTriggerConfiguration(
+                            branch = branchName, // Make triggered builds use the feature branch
+                            daysOfWeek = "5"     // Thursday for GA, TeamCity numbers days Sun=1...Sat=7
+                        ), 
+                    )
+                )
+            }
+        )
+
+        // Nested Nightly Test project that uses hashicorp/terraform-provider-google-beta
+        subProject(
+            Project {
+                id(betaProjectId)
+                name = "Google Beta"
+                subProject(
+                    nightlyTests(
+                        betaProjectId,
+                        ProviderNameBeta,
+                        HashicorpVCSRootBeta_featureBranchMajorRelease600,
+                        betaConfig,
+                        NightlyTriggerConfiguration(
+                            branch = branchName, // Make triggered builds use the feature branch
+                            daysOfWeek="6"       // Friday for Beta, TeamCity numbers days Sun=1...Sat=7
+                        ),
+                    )
+                )
+            }
+        )
+
+        params {
+            readOnlySettings()
+        }
+    }
+}

mmv1/third_party/terraform/.teamcity/components/projects/google_beta_subproject.kt

@@ -8,10 +8,7 @@
 package projects
 
 import ProviderNameBeta
-import builds.AllContextParameters
-import builds.getBetaAcceptanceTestConfig
-import builds.getVcrAcceptanceTestConfig
-import builds.readOnlySettings
+import builds.*
 import jetbrains.buildServer.configs.kotlin.Project
 import projects.reused.mmUpstream
 import projects.reused.nightlyTests
@@ -23,7 +20,7 @@ import vcs_roots.ModularMagicianVCSRootBeta
 // googleSubProjectBeta returns a subproject that is used for testing terraform-provider-google-beta (Beta)
 fun googleSubProjectBeta(allConfig: AllContextParameters): Project {
 
-    var betaId = replaceCharsId("GOOGLE_BETA")
+    val betaId = replaceCharsId("GOOGLE_BETA")
 
     // Get config for using the Beta and VCR identities
     val betaConfig = getBetaAcceptanceTestConfig(allConfig)
@@ -35,10 +32,10 @@ fun googleSubProjectBeta(allConfig: AllContextParameters): Project {
         description = "Subproject containing builds for testing the Beta version of the Google provider"
 
         // Nightly Test project that uses hashicorp/terraform-provider-google-beta
-        subProject(nightlyTests(betaId, ProviderNameBeta, HashiCorpVCSRootBeta, betaConfig))
+        subProject(nightlyTests(betaId, ProviderNameBeta, HashiCorpVCSRootBeta, betaConfig, NightlyTriggerConfiguration(daysOfWeek="1-5,7"))) // All nights except Friday (6) for Beta; feature branch testing happens on Fridays and TeamCity numbers days Sun=1...Sat=7
 
         // MM Upstream project that uses modular-magician/terraform-provider-google-beta
-        subProject(mmUpstream(betaId, ProviderNameBeta, ModularMagicianVCSRootBeta, HashiCorpVCSRootBeta, vcrConfig))
+        subProject(mmUpstream(betaId, ProviderNameBeta, ModularMagicianVCSRootBeta, HashiCorpVCSRootBeta, vcrConfig, NightlyTriggerConfiguration()))
 
         // VCR recording project that allows VCR recordings to be made using hashicorp/terraform-provider-google-beta OR modular-magician/terraform-provider-google-beta
         // This is only present for the Beta provider, as only TPGB VCR recordings are used.

mmv1/third_party/terraform/.teamcity/components/projects/google_ga_subproject.kt

@@ -8,10 +8,7 @@
 package projects
 
 import ProviderNameGa
-import builds.AllContextParameters
-import builds.getGaAcceptanceTestConfig
-import builds.getVcrAcceptanceTestConfig
-import builds.readOnlySettings
+import builds.*
 import jetbrains.buildServer.configs.kotlin.Project
 import projects.reused.mmUpstream
 import projects.reused.nightlyTests
@@ -22,7 +19,7 @@ import vcs_roots.ModularMagicianVCSRootGa
 // googleSubProjectGa returns a subproject that is used for testing terraform-provider-google (GA)
 fun googleSubProjectGa(allConfig: AllContextParameters): Project {
 
-    var gaId = replaceCharsId("GOOGLE")
+    val gaId = replaceCharsId("GOOGLE")
 
     // Get config for using the GA and VCR identities
     val gaConfig = getGaAcceptanceTestConfig(allConfig)
@@ -34,10 +31,10 @@ fun googleSubProjectGa(allConfig: AllContextParameters): Project {
         description = "Subproject containing builds for testing the GA version of the Google provider"
 
         // Nightly Test project that uses hashicorp/terraform-provider-google
-        subProject(nightlyTests(gaId, ProviderNameGa, HashiCorpVCSRootGa, gaConfig))
+        subProject(nightlyTests(gaId, ProviderNameGa, HashiCorpVCSRootGa, gaConfig, NightlyTriggerConfiguration(daysOfWeek="1-4,6-7"))) // All nights except Thursday (5) for GA; feature branch testing happens on Thursdays and TeamCity numbers days Sun=1...Sat=7
 
         // MM Upstream project that uses modular-magician/terraform-provider-google
-        subProject(mmUpstream(gaId, ProviderNameGa, ModularMagicianVCSRootGa, HashiCorpVCSRootGa, vcrConfig))
+        subProject(mmUpstream(gaId, ProviderNameGa, ModularMagicianVCSRootGa, HashiCorpVCSRootGa, vcrConfig, NightlyTriggerConfiguration()))
 
         params {
             readOnlySettings()

mmv1/third_party/terraform/.teamcity/components/projects/project_sweeper_project.kt

@@ -23,7 +23,7 @@ fun projectSweeperSubProject(allConfig: AllContextParameters): Project {
 
     val projectId = replaceCharsId("PROJECT_SWEEPER")
 
-    // Get config for using the GA identity (arbitrary choice as sweeper isn't confined by GA/Beta etc)
+    // Get config for using the GA identity (arbitrary choice as sweeper isn't confined by GA/Beta etc.)
     val gaConfig = getGaAcceptanceTestConfig(allConfig)
 
     // List of ALL shared resources; avoid clashing with any other running build

mmv1/third_party/terraform/.teamcity/components/projects/reused/mm_upstream.kt

@@ -25,7 +25,7 @@ import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
 import replaceCharsId
 
-fun mmUpstream(parentProject: String, providerName: String, vcsRoot: GitVcsRoot, cronSweeperVcsRoot: GitVcsRoot, config: AccTestConfiguration): Project {
+fun mmUpstream(parentProject: String, providerName: String, vcsRoot: GitVcsRoot, cronSweeperVcsRoot: GitVcsRoot, config: AccTestConfiguration, cron: NightlyTriggerConfiguration): Project {
 
     // Create unique ID for the dynamically-created project
     var projectId = "${parentProject}_${MMUpstreamProjectId}"
@@ -45,11 +45,13 @@ fun mmUpstream(parentProject: String, providerName: String, vcsRoot: GitVcsRoot,
         ProviderNameBeta -> sweepersList = SweepersListBeta
         else -> throw Exception("Provider name not supplied when generating a nightly test subproject")
     }
+
+    // This build is for manually-initiated runs of sweepers, to test changes to sweepers from the upstream repo
     val serviceSweeperManualConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperManualName, sweepersList, projectId, vcsRoot, sharedResources, config)
 
+    // This build runs on a schedule to do actual sweeping of the VCR project, using the downstream repo's code
     val serviceSweeperCronConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperCronName, sweepersList, projectId, cronSweeperVcsRoot, sharedResources, config)
-    val trigger  = NightlyTriggerConfiguration(startHour=12)
-    serviceSweeperCronConfig.addTrigger(trigger) // Only the sweeper is on a schedule in this project
+    serviceSweeperCronConfig.addTrigger(cron)
 
     return Project {
         id(projectId)

mmv1/third_party/terraform/.teamcity/components/projects/reused/nightly_tests.kt

@@ -20,7 +20,7 @@ import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
 import replaceCharsId
 
-fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot, config: AccTestConfiguration): Project {
+fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot, config: AccTestConfiguration, cron: NightlyTriggerConfiguration): Project {
 
     // Create unique ID for the dynamically-created project
     var projectId = "${parentProject}_${NightlyTestsProjectId}"
@@ -36,11 +36,11 @@ fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot
     }
 
     // Create build configs to run acceptance tests for each package defined in packages.kt and services.kt files
+    // and add cron trigger to them all
     val allPackages = getAllPackageInProviderVersion(providerName)
     val packageBuildConfigs = BuildConfigurationsForPackages(allPackages, providerName, projectId, vcsRoot, sharedResources, config)
-    val accTestTrigger  = NightlyTriggerConfiguration()
     packageBuildConfigs.forEach { buildConfiguration ->
-        buildConfiguration.addTrigger(accTestTrigger)
+        buildConfiguration.addTrigger(cron)
     }
 
     // Create build config for sweeping the nightly test project
@@ -51,8 +51,9 @@ fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot
         else -> throw Exception("Provider name not supplied when generating a nightly test subproject")
     }
     val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config)
-    val sweeperTrigger  = NightlyTriggerConfiguration(startHour=11)  // Override hour
-    serviceSweeperConfig.addTrigger(sweeperTrigger)
+    val sweeperCron = cron.clone()
+    sweeperCron.startHour += 5  // Ensure triggered after the package test builds are triggered
+    serviceSweeperConfig.addTrigger(sweeperCron)
 
     return Project {
         id(projectId)

mmv1/third_party/terraform/.teamcity/components/projects/root_project.kt

@@ -18,6 +18,8 @@ import generated.ServicesListBeta
 import generated.ServicesListGa
 import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.sharedResource
+import projects.feature_branches.featureBranchMajorRelease600_Project
+
 
 // googleCloudRootProject returns a root project that contains a subprojects for the GA and Beta version of the
 // Google provider. There are also resources to help manage the test projects used for acceptance tests.
@@ -62,6 +64,9 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project {
         subProject(googleSubProjectBeta(allConfig))
         subProject(projectSweeperSubProject(allConfig))
 
+        // Feature branch testing
+        subProject(featureBranchMajorRelease600_Project(allConfig)) // FEATURE-BRANCH-major-release-6.0.0
+
         params {
             readOnlySettings()
         }

mmv1/third_party/terraform/.teamcity/components/unique_id.kt

@@ -6,8 +6,8 @@
 // This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten.
 
 fun replaceCharsId(id: String): String{
-    var newId = id.replace("-", "")
-    newId = newId.replace(" ", "_")
+    // ID should start with a latin letter and contain only latin letters, digits and underscores
+    var newId = id.replace("-", "").replace(" ", "_").replace(".", "_")
     newId = newId.uppercase()
 
     return newId

mmv1/third_party/terraform/.teamcity/tests/sweepers.kt

@@ -151,7 +151,7 @@ class SweeperTests {
         val cronBeta = stBeta.schedulingPolicy as ScheduleTrigger.SchedulingPolicy.Cron
         val stProject = projectSweeper.triggers.items[0] as ScheduleTrigger
         val cronProject = stProject.schedulingPolicy as ScheduleTrigger.SchedulingPolicy.Cron
-        assertTrue("Service sweeper for the GA Nightly Test project should be triggered at an earlier hour than the project sweeper", cronGa.hours.toString() < cronProject.hours.toString()) // Values are strings like "11", "12"
-        assertTrue("Service sweeper for the Beta Nightly Test project should be triggered at an earlier hour than the project sweeper", cronBeta.hours.toString() < cronProject.hours.toString() )
+        assertTrue("Service sweeper for the GA Nightly Test project should be triggered at an earlier hour than the project sweeper", cronGa.hours.toString().toInt() < cronProject.hours.toString().toInt()) // Converting nullable strings to ints
+        assertTrue("Service sweeper for the Beta Nightly Test project should be triggered at an earlier hour than the project sweeper", cronBeta.hours.toString().toInt() < cronProject.hours.toString().toInt() )
     }
 }