Skip to content

Commit

Permalink
OCM-5627 | ci: upgrade scenario test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
amalykhi committed Feb 5, 2024
1 parent 66471f4 commit f31b309
Show file tree
Hide file tree
Showing 16 changed files with 443 additions and 25 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ require (
github.com/fatih/color v1.13.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang/glog v1.1.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,8 @@ github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3a
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9 h1:wqckanyE9qc/XnvnybC6SHOb8Nyd62QXAZOzA8twFig=
github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9/go.mod h1:64ikIrMv84B+raz7akXOqbF7cK3/OQQ/6cClY10oy7A=
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -514,6 +516,7 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC
github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down
2 changes: 2 additions & 0 deletions tests/ci/labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ var Day1Prepare = Label("day1-prepare")
var Day1Negative = Label("day1-negative")
var Day1Post = Label("day1-post")
var Day2 = Label("day2")
var Upgrade = Label("upgrade")
var Update = Label("update")

// day3 : the test cases will destroy default resource
var Day3 = Label("day3")
Expand Down
10 changes: 9 additions & 1 deletion tests/ci/profile_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,13 @@ func PrepareVersion(connection *client.Connection, versionTag string, channelGro
}
return vResult
}

func GetMajorVersion(rawVersion string) string {
versionRegex := regexp.MustCompile(`^[0-9]+\.[0-9]+`)
vResult := versionRegex.FindAllStringSubmatch(rawVersion, 1)[0][0]
return vResult
}

func PrepareProxy() {}

func PrepareKMSKey(profile *Profile, kmsName string, accountRolePrefix string, accountRolePath string) (string, error) {
Expand Down Expand Up @@ -303,7 +310,8 @@ func GenerateClusterCreationArgsByProfile(token string, profile *Profile) (clust
}

if profile.STS {
accountRolesOutput, err := PrepareAccountRoles(token, clusterArgs.ClusterName, profile.UnifiedAccRolesPath, clusterArgs.AWSRegion, profile.MajorVersion, profile.ChannelGroup, CON.AccountRolesDir)
majorVersion := GetMajorVersion(profile.Version)
accountRolesOutput, err := PrepareAccountRoles(token, clusterArgs.ClusterName, profile.UnifiedAccRolesPath, clusterArgs.AWSRegion, majorVersion, profile.ChannelGroup, CON.AccountRolesDir)
Expect(err).ToNot(HaveOccurred())
clusterArgs.AccountRolePrefix = accountRolesOutput.AccountRolePrefix
clusterArgs.UnifiedAccRolesPath = profile.UnifiedAccRolesPath
Expand Down
42 changes: 37 additions & 5 deletions tests/ci/profiles/tf_cluster_profile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ profiles:
additional_sg_number: 4
worker_disk_size: 200
unified_acc_role_path: "/unified/"
# rosa-sts-up :: creating a managed cluster for upgrade purpose
- as: rosa-sts-up
# rosa-up-y :: creating a cluster for y-stream upgrade purpose
- as: rosa-up-y
cluster:
multi_az: false
product_id: "rosa"
Expand All @@ -79,14 +79,46 @@ profiles:
fips: false
autoscale: false
byok: true
version: "z-1"
version: "y-1"
major_version: "4.14"
compute_machine_type: "m5.xlarge"
proxy: false
labeling: false
tagging: true
channel_group: stable
zones: ""
imdsv2: ""
oidc_config: "managed"
admin_enabled: true
unified_acc_role_path: "/uni-fied/"
# rosa-up-z :: creating a cluster for z-stream upgrade purpose
- as: rosa-up-z
cluster:
multi_az: true
product_id: "rosa"
hypershift: false
cloud_provider: "aws"
region: "ap-northeast-1"
ccs: true
sts: true
byovpc: false
private_link: false
private: false
etcd_encryption: true
kms_key_arn: false
fips: false
autoscale: false
byok: true
version: "z-1"
major_version: "4.14"
compute_machine_type: "m5.xlarge"
proxy: false
labeling: true
tagging: false
channel_group: candidate
channel_group: stable
zones: ""
imdsv2: ""
oidc_config: "managed"
admin_enabled: true
admin_enabled: true
unified_acc_role_path: ""

14 changes: 2 additions & 12 deletions tests/e2e/cluster_creation_test.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
package e2e

import (
"fmt"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
CI "github.com/terraform-redhat/terraform-provider-rhcs/tests/ci"
CON "github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/constants"
. "github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/log"
"github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/openshift"
)

Expand All @@ -23,16 +19,10 @@ var _ = Describe("RHCS Provider Test", func() {
Expect(err).ToNot(HaveOccurred())
Expect(clusterID).ToNot(BeEmpty())
//TODO: implement waiter for the private cluster once bastion is implemented
if CON.GetEnvWithDefault(CON.WaitOperators, "false") == "true" {
if CON.GetEnvWithDefault(CON.WaitOperators, "false") == "true" && !profile.Private {
// WaitClusterOperatorsToReadyStatus will wait for cluster operators ready
timeout := 60
timeoutMin := time.Duration(timeout)
console, err := openshift.NewConsole(clusterID, CI.RHCSConnection)
if err != nil {
Logger.Warnf("Got error %s when config the openshift console. Return without waiting for operators ready", err.Error())
return
}
_, err = openshift.RetryCMDRun(fmt.Sprintf("oc wait clusteroperators --all --for=condition=Progressing=false --kubeconfig %s --timeout %dm", console.KubePath, timeout), timeoutMin)
err = openshift.WaitForOperatorsToBeReady(CI.RHCSConnection, clusterID, timeout)
Expect(err).ToNot(HaveOccurred())
}
})
Expand Down
157 changes: 157 additions & 0 deletions tests/e2e/cluster_upgrade_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package e2e

import (
"fmt"
"strconv"
"strings"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
CI "github.com/terraform-redhat/terraform-provider-rhcs/tests/ci"
CMS "github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/cms"
CON "github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/constants"
EXE "github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/exec"
"github.com/terraform-redhat/terraform-provider-rhcs/tests/utils/openshift"
)

var _ = Describe("RHCS Provider Test", func() {
Describe("Upgrade cluster tests", func() {

var targetV string
var clusterID string
BeforeEach(OncePerOrdered, func() {
clusterID, err = CI.PrepareRHCSClusterByProfileENV()
Expect(err).ToNot(HaveOccurred())

})
Context("Author:amalykhi-Critical-OCP-63153 @OCP-63153 @amalykhi", func() {
It("Z-stream upgrade a ROSA STS cluster with RHCS provider", CI.Upgrade,
func() {
if profile.Version != "z-1" {
Skip("The test is configured only for Z-stream upgrade")
}
clusterResp, err := CMS.RetrieveClusterDetail(CI.RHCSConnection, clusterID)
targetV, err = CMS.GetVersionUpgradeTarget(clusterResp.Body().Version().RawID(),
CON.Z, clusterResp.Body().Version().AvailableUpgrades())
Expect(err).ToNot(HaveOccurred())

clusterService, err := EXE.NewClusterService(profile.ManifestsDIR)
Expect(err).ToNot(HaveOccurred())

By("Validate invalid OCP version - downgrade")
currentVersion := string(clusterResp.Body().Version().RawID())
splittedVersion := strings.Split(currentVersion, ".")
zStreamV, err := strconv.Atoi(splittedVersion[2])
Expect(err).ToNot(HaveOccurred())

downgradedVersion := fmt.Sprintf("%s.%s.%s", splittedVersion[0], splittedVersion[1], fmt.Sprint(zStreamV-1))

clusterArgs := &EXE.ClusterCreationArgs{
OpenshiftVersion: downgradedVersion,
}
err = clusterService.Apply(clusterArgs, false, false)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("cluster version is already above the\nrequested version"))

By("Run the cluster update")
clusterArgs = &EXE.ClusterCreationArgs{
OpenshiftVersion: targetV,
}
err = clusterService.Apply(clusterArgs, false, false)
Expect(err).ToNot(HaveOccurred())

By("Wait the upgrade finished")
err = openshift.WaitClusterUpgradeFinished(CI.RHCSConnection, clusterID)
Expect(err).ToNot(HaveOccurred(), "Cluster upgrade %s failed with the error %v", clusterID, err)

By("Check the cluster status and OCP version")
clusterResp, err = CMS.RetrieveClusterDetail(CI.RHCSConnection, clusterID)
Expect(err).ToNot(HaveOccurred())
Expect(string(clusterResp.Body().State())).To(Equal(CON.Ready))
Expect(string(clusterResp.Body().Version().RawID())).To(Equal(targetV))

if CON.GetEnvWithDefault(CON.WaitOperators, "false") == "true" && !profile.Private {
// WaitClusterOperatorsToReadyStatus will wait for cluster operators ready
timeout := 60
err = openshift.WaitForOperatorsToBeReady(CI.RHCSConnection, clusterID, timeout)
Expect(err).ToNot(HaveOccurred())
}
})
})
Context("Author:amalykhi-Critical-OCP-63152 @OCP-63152 @amalykhi", func() {
It("Y-stream Upgrade ROSA STS cluster with RHCS provider", CI.Upgrade,
func() {
if profile.Version != "y-1" {
Skip("The test is configured only for Y-stream upgrade")
}

clusterResp, err := CMS.RetrieveClusterDetail(CI.RHCSConnection, clusterID)

targetV, err = CMS.GetVersionUpgradeTarget(clusterResp.Body().Version().RawID(),
CON.Y, clusterResp.Body().Version().AvailableUpgrades())
Expect(err).ToNot(HaveOccurred())

By("Upgrade account-roles")
majorVersion := CI.GetMajorVersion(targetV)
_, err = CI.PrepareAccountRoles(token, clusterResp.Body().Name(), profile.UnifiedAccRolesPath, profile.Region, majorVersion, profile.ChannelGroup, CON.AccountRolesDir)
Expect(err).ToNot(HaveOccurred())

clusterService, err := EXE.NewClusterService(profile.ManifestsDIR)
Expect(err).ToNot(HaveOccurred())

By("Validate invalid OCP version field - downgrade")
currentVersion := string(clusterResp.Body().Version().RawID())
splittedVersion := strings.Split(currentVersion, ".")
yStreamV, err := strconv.Atoi(splittedVersion[1])
Expect(err).ToNot(HaveOccurred())

downgradedVersion := fmt.Sprintf("%s.%s.%s", splittedVersion[0], fmt.Sprint(yStreamV-1), splittedVersion[2])

clusterArgs := &EXE.ClusterCreationArgs{
OpenshiftVersion: downgradedVersion,
}
err = clusterService.Apply(clusterArgs, false, false)
Expect(err).To(HaveOccurred())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("cluster version is already above the\nrequested version"))

By("Validate the cluster Upgrade upgrade_acknowledge field")

clusterArgs = &EXE.ClusterCreationArgs{
OpenshiftVersion: targetV,
}

err = clusterService.Apply(clusterArgs, false, false)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Missing required acknowledgements to schedule upgrade"))

By("Apply the cluster Upgrade")

clusterArgs = &EXE.ClusterCreationArgs{
OpenshiftVersion: targetV,
UpgradeAcknowledgementsFor: majorVersion,
}

err = clusterService.Apply(clusterArgs, false, false)
Expect(err).ToNot(HaveOccurred())

By("Wait the upgrade finished")
err = openshift.WaitClusterUpgradeFinished(CI.RHCSConnection, clusterID)
Expect(err).ToNot(HaveOccurred(), "Cluster %s failed with the error %v", clusterID, err)

By("Check the cluster status and OCP version")
clusterResp, err = CMS.RetrieveClusterDetail(CI.RHCSConnection, clusterID)
Expect(err).ToNot(HaveOccurred())
Expect(string(clusterResp.Body().State())).To(Equal(CON.Ready))
Expect(string(clusterResp.Body().Version().RawID())).To(Equal(targetV))

if CON.GetEnvWithDefault(CON.WaitOperators, "false") == "true" && !profile.Private {
// WaitClusterOperatorsToReadyStatus will wait for cluster operators ready
timeout := 60
err = openshift.WaitForOperatorsToBeReady(CI.RHCSConnection, clusterID, timeout)
Expect(err).ToNot(HaveOccurred())
}
})
})
})
})
5 changes: 1 addition & 4 deletions tests/e2e/general_day_two_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package e2e

import (
"path/filepath"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
ci "github.com/terraform-redhat/terraform-provider-rhcs/tests/ci"
Expand Down Expand Up @@ -88,8 +86,7 @@ var _ = Describe("TF day2 scenrios", func() {
Expect(err).ShouldNot(HaveOccurred())

// Read terraform.tfvars file and get its content as a map
terraformTFVarsPath := filepath.Join(profile.ManifestsDIR, "terraform.tfvars")
terraformTFVarsContent := exe.ReadTerraformTFVars(terraformTFVarsPath)
terraformTFVarsContent := exe.ReadTerraformTFVars(profile.ManifestsDIR)
Expect(err).ShouldNot(HaveOccurred())

// Capture the original custom properties
Expand Down
2 changes: 2 additions & 0 deletions tests/tf-manifests/rhcs/clusters/rosa-classic/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,11 @@ resource "rhcs_cluster_rosa_classic" "rosa_sts_cluster" {
aws_additional_infra_security_group_ids = var.additional_infra_security_groups
aws_additional_control_plane_security_group_ids = var.additional_control_plane_security_groups
destroy_timeout = 120
upgrade_acknowledgements_for = var.upgrade_acknowledgements_for
lifecycle {
ignore_changes = [availability_zones]
}
wait_for_create_complete = true
}

resource "rhcs_cluster_wait" "rosa_cluster" {
Expand Down
5 changes: 5 additions & 0 deletions tests/tf-manifests/rhcs/clusters/rosa-classic/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -213,4 +213,9 @@ variable "path" {
description = "(Optional) The arn path for the account/operator roles as well as their policies."
type = string
default = null
}

variable "upgrade_acknowledgements_for" {
type = string
default = null
}
5 changes: 5 additions & 0 deletions tests/utils/cms/cms.go
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,11 @@ func ListUpgradePolicies(connection *client.Connection, clusterID string, params
return request.Send()
}

func GetUpgradePolicyState(connection *client.Connection, clusterID string, upgradepolicyID string) (*cmv1.UpgradePolicyStateGetResponse, error) {
resp, err := connection.ClustersMgmt().V1().Clusters().Cluster(clusterID).UpgradePolicies().UpgradePolicy(upgradepolicyID).State().Get().Send()
return resp, err
}

func RetrieveUpgradePolicies(connection *client.Connection, clusterID string, upgradepolicyID string) (*cmv1.UpgradePolicyGetResponse, error) {
resp, err := connection.ClustersMgmt().V1().Clusters().Cluster(clusterID).UpgradePolicies().UpgradePolicy(upgradepolicyID).Get().Send()
return resp, err
Expand Down
Loading

0 comments on commit f31b309

Please sign in to comment.