diff --git a/Makefile b/Makefile index c8df0eca16954..3437a4403c2cf 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,10 @@ DOCKER_REGISTRY?=gcr.io/must-override S3_BUCKET?=s3://must-override/ UPLOAD_DEST?=$(S3_BUCKET) GCS_LOCATION?=gs://must-override +S3_REGION?=us-east-1 GCS_URL=$(GCS_LOCATION:gs://%=https://storage.googleapis.com/%) +S3_HOSTNAME=$(subst %,$(S3_REGION),https://s3.%.amazonaws.com) +S3_URL=$(S3_BUCKET:s3://%=${S3_HOSTNAME}/%) LATEST_FILE?=latest-ci.txt GOPATH_1ST:=$(shell go env GOPATH) UNIQUE:=$(shell date +%s) @@ -260,7 +263,7 @@ gcs-upload-and-tag: gsutil gcs-upload echo "${GCS_URL}${VERSION}" > ${UPLOAD}/latest.txt gsutil -h "Cache-Control:private, max-age=0, no-transform" cp ${UPLOAD}/latest.txt ${GCS_LOCATION}${LATEST_FILE} -# gcs-publish-ci is the entry point for CI testing +# gcs-publish-ci and s3-publish-ci are the entry points for CI testing # In CI testing, always upload the CI version. .PHONY: gcs-publish-ci gcs-publish-ci: VERSION := ${KOPS_CI_VERSION}+${GITSHA} @@ -271,6 +274,15 @@ gcs-publish-ci: gsutil version-dist-ci echo "${GCS_URL}/${VERSION}" > ${UPLOAD}/${LATEST_FILE} gsutil -h "Cache-Control:private, max-age=0, no-transform" cp ${UPLOAD}/${LATEST_FILE} ${GCS_LOCATION} +.PHONY: s3-publish-ci +s3-publish-ci: VERSION := ${KOPS_CI_VERSION}+${GITSHA} +s3-publish-ci: version-dist-ci + @echo "== Uploading kops ==" + aws s3 sync -r --cache-control "private, max-age=0, no-transform" ${UPLOAD}/kops/* ${S3_LOCATION} + echo "VERSION: ${VERSION}" + echo "${S3_URL}/${VERSION}" > ${UPLOAD}/${LATEST_FILE} + aws s3 sync --cache-control "private, max-age=0, no-transform" ${UPLOAD}/${LATEST_FILE} ${S3_LOCATION} + .PHONY: gen-cli-docs gen-cli-docs: kops # Regenerate CLI docs KOPS_STATE_STORE= \ diff --git a/tests/e2e/kubetest2-kops/aws/bucket_region.go b/tests/e2e/kubetest2-kops/aws/bucket_region.go new file mode 100644 index 0000000000000..2ade861de5fde --- /dev/null +++ b/tests/e2e/kubetest2-kops/aws/bucket_region.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func GetBucketRegion(ctx context.Context, bucket string) (string, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return "", fmt.Errorf("failed to load AWS configuration: %w", err) + } + client := s3.NewFromConfig(cfg) + + bucket = strings.TrimPrefix(bucket, "s3://") + bucket = strings.Split(bucket, "/")[0] + + resp, err := client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + return "", fmt.Errorf("failed to get bucket region: %w", err) + } + + return aws.ToString(resp.BucketRegion), nil +} diff --git a/tests/e2e/kubetest2-kops/builder/build.go b/tests/e2e/kubetest2-kops/builder/build.go index dba1395a52078..33a001d4749e8 100644 --- a/tests/e2e/kubetest2-kops/builder/build.go +++ b/tests/e2e/kubetest2-kops/builder/build.go @@ -17,6 +17,7 @@ limitations under the License. package builder import ( + "errors" "fmt" "net/url" "os" @@ -33,6 +34,7 @@ type BuildOptions struct { KopsRoot string `flag:"-"` KubeRoot string `flag:"-"` StageLocation string `flag:"-"` + S3BucketRegion string `flag:"-"` TargetBuildArch string `flag:"~target-build-arch" desc:"CPU architecture to test against"` BuildKubernetes bool `flag:"~build-kubernetes" desc:"Set this flag to true to build kubernetes"` } @@ -83,13 +85,29 @@ func (b *BuildOptions) Build() (*BuildResults, error) { return results, nil } - cmd := exec.Command("make", "gcs-publish-ci") + var cmd exec.Cmd + switch { + case strings.HasPrefix(b.StageLocation, "gs://"): + cmd = exec.Command("make", "gcs-publish-ci") + cmd.SetEnv( + fmt.Sprintf("GCS_LOCATION=%v", gcsLocation), + ) + case strings.HasPrefix(b.StageLocation, "s3://"): + if b.S3BucketRegion == "" { + return nil, errors.New("missing required S3 bucket region") + } + cmd = exec.Command("make", "s3-publish-ci") + cmd.SetEnv( + fmt.Sprintf("S3_BUCKET=%v", gcsLocation), + fmt.Sprintf("S3_REGION=%v", b.S3BucketRegion), + ) + } cmd.SetEnv( fmt.Sprintf("HOME=%v", os.Getenv("HOME")), fmt.Sprintf("PATH=%v", os.Getenv("PATH")), - fmt.Sprintf("GCS_LOCATION=%v", gcsLocation), fmt.Sprintf("GOPATH=%v", os.Getenv("GOPATH")), ) + cmd.SetDir(b.KopsRoot) exec.InheritOutput(cmd) if err := cmd.Run(); err != nil { @@ -97,7 +115,7 @@ func (b *BuildOptions) Build() (*BuildResults, error) { } // Get the full path (including subdirectory) that we uploaded to - // It is written by gcs-publish-ci to .build/upload/latest-ci.txt + // It is written by the *-publish-ci make tasks to .build/upload/latest-ci.txt latestPath := filepath.Join(b.KopsRoot, ".build", "upload", "latest-ci.txt") kopsBaseURL, err := os.ReadFile(latestPath) if err != nil { diff --git a/tests/e2e/kubetest2-kops/deployer/build.go b/tests/e2e/kubetest2-kops/deployer/build.go index df77fc3f4a945..4f3e9498fb71a 100644 --- a/tests/e2e/kubetest2-kops/deployer/build.go +++ b/tests/e2e/kubetest2-kops/deployer/build.go @@ -17,6 +17,7 @@ limitations under the License. package deployer import ( + "context" "errors" "fmt" "os" @@ -24,6 +25,7 @@ import ( "strings" "k8s.io/klog/v2" + "k8s.io/kops/tests/e2e/kubetest2-kops/aws" "k8s.io/kops/tests/e2e/kubetest2-kops/gce" "k8s.io/kops/tests/e2e/pkg/util" "k8s.io/kops/tests/e2e/pkg/version" @@ -34,6 +36,7 @@ import ( const ( defaultJobName = "pull-kops-e2e-kubernetes-aws" defaultGCSPath = "gs://k8s-staging-kops/pulls/%v/pull-%v" + defaultS3Path = "s3://k8s-infra-kops-ci-results/pulls/%v/pull-%v" ) func (d *deployer) Build() error { @@ -109,8 +112,8 @@ func (d *deployer) verifyBuildFlags() error { d.BuildOptions.KubeRoot = KubeRoot } if d.StageLocation != "" { - if !strings.HasPrefix(d.StageLocation, "gs://") { - return errors.New("stage-location must be a gs:// path") + if !strings.HasPrefix(d.StageLocation, "gs://") && !strings.HasPrefix(d.StageLocation, "s3://") { + return errors.New("stage-location must be a gs:// or s3:// path") } } else if d.boskos != nil { d.StageLocation = d.stagingStore() @@ -119,14 +122,26 @@ func (d *deployer) verifyBuildFlags() error { return err } } else { - stageLocation, err := defaultStageLocation(d.KopsRoot) + stageLocation, err := defaultStageLocation(d.CloudProvider, d.KopsRoot) if err != nil { return err } d.StageLocation = stageLocation } + if strings.HasPrefix(d.StageLocation, "s3://") { + region, err := aws.GetBucketRegion(context.TODO(), d.StageLocation) + if err != nil { + return fmt.Errorf("failed to get bucket region: %w", err) + } + d.BuildOptions.S3BucketRegion = region + } if d.KopsBaseURL == "" && os.Getenv("KOPS_BASE_URL") == "" { - d.KopsBaseURL = strings.Replace(d.StageLocation, "gs://", "https://storage.googleapis.com/", 1) + switch { + case strings.HasPrefix(d.StageLocation, "gs://"): + d.KopsBaseURL = strings.Replace(d.StageLocation, "gs://", "https://storage.googleapis.com/", 1) + case strings.HasPrefix(d.StageLocation, "s3://"): + d.KopsBaseURL = fmt.Sprintf("https://s3.%s.amazonaws.com/%s", d.BuildOptions.S3BucketRegion, strings.TrimPrefix(d.StageLocation, "s3://")) + } } if d.KopsVersionMarker != "" && !d.BuildOptions.BuildKubernetes { @@ -138,7 +153,7 @@ func (d *deployer) verifyBuildFlags() error { return nil } -func defaultStageLocation(kopsRoot string) (string, error) { +func defaultStageLocation(cloudProvider, kopsRoot string) (string, error) { jobName := os.Getenv("JOB_NAME") if jobName == "" { jobName = defaultJobName @@ -156,5 +171,9 @@ func defaultStageLocation(kopsRoot string) (string, error) { } sha = strings.TrimSpace(output[0]) } + switch cloudProvider { + case "aws": + return fmt.Sprintf(defaultS3Path, jobName, sha), nil + } return fmt.Sprintf(defaultGCSPath, jobName, sha), nil } diff --git a/tests/e2e/kubetest2-kops/deployer/publish.go b/tests/e2e/kubetest2-kops/deployer/publish.go index 07aac0cf3b32b..5739eee712e54 100644 --- a/tests/e2e/kubetest2-kops/deployer/publish.go +++ b/tests/e2e/kubetest2-kops/deployer/publish.go @@ -32,7 +32,7 @@ func (d *deployer) PostTest(testErr error) error { if testErr != nil || d.PublishVersionMarker == "" { return nil } - if !strings.HasPrefix(d.PublishVersionMarker, "gs://") { + if !strings.HasPrefix(d.PublishVersionMarker, "gs://") && !strings.HasPrefix(d.PublishVersionMarker, "s3://") { return fmt.Errorf("unsupported --publish-version-marker protocol: %v", d.PublishVersionMarker) } if d.KopsVersionMarker == "" { @@ -51,13 +51,26 @@ func (d *deployer) PostTest(testErr error) error { if err != nil { return err } - - args := []string{ - "gsutil", - "-h", "Cache-Control:private, max-age=0, no-transform", - "cp", - tempSrc.Name(), - d.PublishVersionMarker, + var args []string + switch { + case strings.HasPrefix(d.PublishVersionMarker, "gs://"): + args = []string{ + "gsutil", + "-h", "Cache-Control:private, max-age=0, no-transform", + "cp", + tempSrc.Name(), + d.PublishVersionMarker, + } + case strings.HasPrefix(d.PublishVersionMarker, "s3://"): + args = []string{ + "aws", + "s3", + "sync", + "--cache-control", + "private, max-age=0, no-transform", + tempSrc.Name(), + d.PublishVersionMarker, + } } klog.Info(strings.Join(args, " "))