Support publishing CI artifacts to S3

Makefile

@@ -18,7 +18,10 @@ DOCKER_REGISTRY?=gcr.io/must-override
 S3_BUCKET?=s3://must-override/
 UPLOAD_DEST?=$(S3_BUCKET)
 GCS_LOCATION?=gs://must-override
+S3_REGION?=us-east-1
 GCS_URL=$(GCS_LOCATION:gs://%=https://storage.googleapis.com/%)
+S3_HOSTNAME=$(subst %,$(S3_REGION),https://s3.%.amazonaws.com)
+S3_URL=$(S3_BUCKET:s3://%=${S3_HOSTNAME}/%)
 LATEST_FILE?=latest-ci.txt
 GOPATH_1ST:=$(shell go env GOPATH)
 UNIQUE:=$(shell date +%s)
@@ -260,7 +263,7 @@ gcs-upload-and-tag: gsutil gcs-upload
 	echo "${GCS_URL}${VERSION}" > ${UPLOAD}/latest.txt
 	gsutil -h "Cache-Control:private, max-age=0, no-transform" cp ${UPLOAD}/latest.txt ${GCS_LOCATION}${LATEST_FILE}
 
-# gcs-publish-ci is the entry point for CI testing
+# gcs-publish-ci and s3-publish-ci are the entry points for CI testing
 # In CI testing, always upload the CI version.
 .PHONY: gcs-publish-ci
 gcs-publish-ci: VERSION := ${KOPS_CI_VERSION}+${GITSHA}
@@ -271,6 +274,15 @@ gcs-publish-ci: gsutil version-dist-ci
 	echo "${GCS_URL}/${VERSION}" > ${UPLOAD}/${LATEST_FILE}
 	gsutil -h "Cache-Control:private, max-age=0, no-transform" cp ${UPLOAD}/${LATEST_FILE} ${GCS_LOCATION}
 
+.PHONY: s3-publish-ci
+s3-publish-ci: VERSION := ${KOPS_CI_VERSION}+${GITSHA}
+s3-publish-ci: version-dist-ci
+	@echo "== Uploading kops =="
+	aws s3 sync -r --cache-control "private, max-age=0, no-transform" ${UPLOAD}/kops/* ${S3_LOCATION}
+	echo "VERSION: ${VERSION}"
+	echo "${S3_URL}/${VERSION}" > ${UPLOAD}/${LATEST_FILE}
+	aws s3 sync --cache-control "private, max-age=0, no-transform" ${UPLOAD}/${LATEST_FILE} ${S3_LOCATION}
+
 .PHONY: gen-cli-docs
 gen-cli-docs: kops # Regenerate CLI docs
 	KOPS_STATE_STORE= \

tests/e2e/kubetest2-kops/aws/bucket_region.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+func GetBucketRegion(ctx context.Context, bucket string) (string, error) {
+	cfg, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to load AWS configuration: %w", err)
+	}
+	client := s3.NewFromConfig(cfg)
+
+	bucket = strings.TrimPrefix(bucket, "s3://")
+	bucket = strings.Split(bucket, "/")[0]
+
+	resp, err := client.HeadBucket(ctx, &s3.HeadBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to get bucket region: %w", err)
+	}
+
+	return aws.ToString(resp.BucketRegion), nil
+}

tests/e2e/kubetest2-kops/builder/build.go

@@ -17,6 +17,7 @@ limitations under the License.
 package builder
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -33,6 +34,7 @@ type BuildOptions struct {
 	KopsRoot        string `flag:"-"`
 	KubeRoot        string `flag:"-"`
 	StageLocation   string `flag:"-"`
+	S3BucketRegion  string `flag:"-"`
 	TargetBuildArch string `flag:"~target-build-arch" desc:"CPU architecture to test against"`
 	BuildKubernetes bool   `flag:"~build-kubernetes" desc:"Set this flag to true to build kubernetes"`
 }
@@ -83,21 +85,37 @@ func (b *BuildOptions) Build() (*BuildResults, error) {
 		return results, nil
 	}
 
-	cmd := exec.Command("make", "gcs-publish-ci")
+	var cmd exec.Cmd
+	switch {
+	case strings.HasPrefix(b.StageLocation, "gs://"):
+		cmd = exec.Command("make", "gcs-publish-ci")
+		cmd.SetEnv(
+			fmt.Sprintf("GCS_LOCATION=%v", gcsLocation),
+		)
+	case strings.HasPrefix(b.StageLocation, "s3://"):
+		if b.S3BucketRegion == "" {
+			return nil, errors.New("missing required S3 bucket region")
+		}
+		cmd = exec.Command("make", "s3-publish-ci")
+		cmd.SetEnv(
+			fmt.Sprintf("S3_BUCKET=%v", gcsLocation),
+			fmt.Sprintf("S3_REGION=%v", b.S3BucketRegion),
+		)
+	}
 	cmd.SetEnv(
 		fmt.Sprintf("HOME=%v", os.Getenv("HOME")),
 		fmt.Sprintf("PATH=%v", os.Getenv("PATH")),
-		fmt.Sprintf("GCS_LOCATION=%v", gcsLocation),
 		fmt.Sprintf("GOPATH=%v", os.Getenv("GOPATH")),
 	)
+
 	cmd.SetDir(b.KopsRoot)
 	exec.InheritOutput(cmd)
 	if err := cmd.Run(); err != nil {
 		return nil, err
 	}
 
 	// Get the full path (including subdirectory) that we uploaded to
-	// It is written by gcs-publish-ci to .build/upload/latest-ci.txt
+	// It is written by the *-publish-ci make tasks to .build/upload/latest-ci.txt
 	latestPath := filepath.Join(b.KopsRoot, ".build", "upload", "latest-ci.txt")
 	kopsBaseURL, err := os.ReadFile(latestPath)
 	if err != nil {

tests/e2e/kubetest2-kops/deployer/build.go

@@ -17,13 +17,15 @@ limitations under the License.
 package deployer
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
 	"path"
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"k8s.io/kops/tests/e2e/pkg/util"
 	"k8s.io/kops/tests/e2e/pkg/version"
@@ -34,6 +36,7 @@ import (
 const (
 	defaultJobName = "pull-kops-e2e-kubernetes-aws"
 	defaultGCSPath = "gs://k8s-staging-kops/pulls/%v/pull-%v"
+	defaultS3Path  = "s3://k8s-infra-kops-ci-results/pulls/%v/pull-%v"
 )
 
 func (d *deployer) Build() error {
@@ -109,8 +112,8 @@ func (d *deployer) verifyBuildFlags() error {
 		d.BuildOptions.KubeRoot = KubeRoot
 	}
 	if d.StageLocation != "" {
-		if !strings.HasPrefix(d.StageLocation, "gs://") {
-			return errors.New("stage-location must be a gs:// path")
+		if !strings.HasPrefix(d.StageLocation, "gs://") && !strings.HasPrefix(d.StageLocation, "s3://") {
+			return errors.New("stage-location must be a gs:// or s3:// path")
 		}
 	} else if d.boskos != nil {
 		d.StageLocation = d.stagingStore()
@@ -119,14 +122,26 @@ func (d *deployer) verifyBuildFlags() error {
 			return err
 		}
 	} else {
-		stageLocation, err := defaultStageLocation(d.KopsRoot)
+		stageLocation, err := defaultStageLocation(d.CloudProvider, d.KopsRoot)
 		if err != nil {
 			return err
 		}
 		d.StageLocation = stageLocation
 	}
+	if strings.HasPrefix(d.StageLocation, "s3://") {
+		region, err := aws.GetBucketRegion(context.TODO(), d.StageLocation)
+		if err != nil {
+			return fmt.Errorf("failed to get bucket region: %w", err)
+		}
+		d.BuildOptions.S3BucketRegion = region
+	}
 	if d.KopsBaseURL == "" && os.Getenv("KOPS_BASE_URL") == "" {
-		d.KopsBaseURL = strings.Replace(d.StageLocation, "gs://", "https://storage.googleapis.com/", 1)
+		switch {
+		case strings.HasPrefix(d.StageLocation, "gs://"):
+			d.KopsBaseURL = strings.Replace(d.StageLocation, "gs://", "https://storage.googleapis.com/", 1)
+		case strings.HasPrefix(d.StageLocation, "s3://"):
+			d.KopsBaseURL = fmt.Sprintf("https://s3.%s.amazonaws.com/%s", d.BuildOptions.S3BucketRegion, strings.TrimPrefix(d.StageLocation, "s3://"))
+		}
 	}
 
 	if d.KopsVersionMarker != "" && !d.BuildOptions.BuildKubernetes {
@@ -138,7 +153,7 @@ func (d *deployer) verifyBuildFlags() error {
 	return nil
 }
 
-func defaultStageLocation(kopsRoot string) (string, error) {
+func defaultStageLocation(cloudProvider, kopsRoot string) (string, error) {
 	jobName := os.Getenv("JOB_NAME")
 	if jobName == "" {
 		jobName = defaultJobName
@@ -156,5 +171,9 @@ func defaultStageLocation(kopsRoot string) (string, error) {
 		}
 		sha = strings.TrimSpace(output[0])
 	}
+	switch cloudProvider {
+	case "aws":
+		return fmt.Sprintf(defaultS3Path, jobName, sha), nil
+	}
 	return fmt.Sprintf(defaultGCSPath, jobName, sha), nil
 }

tests/e2e/kubetest2-kops/deployer/publish.go

@@ -32,7 +32,7 @@ func (d *deployer) PostTest(testErr error) error {
 	if testErr != nil || d.PublishVersionMarker == "" {
 		return nil
 	}
-	if !strings.HasPrefix(d.PublishVersionMarker, "gs://") {
+	if !strings.HasPrefix(d.PublishVersionMarker, "gs://") && !strings.HasPrefix(d.PublishVersionMarker, "s3://") {
 		return fmt.Errorf("unsupported --publish-version-marker protocol: %v", d.PublishVersionMarker)
 	}
 	if d.KopsVersionMarker == "" {
@@ -51,13 +51,26 @@ func (d *deployer) PostTest(testErr error) error {
 	if err != nil {
 		return err
 	}
-
-	args := []string{
-		"gsutil",
-		"-h", "Cache-Control:private, max-age=0, no-transform",
-		"cp",
-		tempSrc.Name(),
-		d.PublishVersionMarker,
+	var args []string
+	switch {
+	case strings.HasPrefix(d.PublishVersionMarker, "gs://"):
+		args = []string{
+			"gsutil",
+			"-h", "Cache-Control:private, max-age=0, no-transform",
+			"cp",
+			tempSrc.Name(),
+			d.PublishVersionMarker,
+		}
+	case strings.HasPrefix(d.PublishVersionMarker, "s3://"):
+		args = []string{
+			"aws",
+			"s3",
+			"sync",
+			"--cache-control",
+			"private, max-age=0, no-transform",
+			tempSrc.Name(),
+			d.PublishVersionMarker,
+		}
 	}
 	klog.Info(strings.Join(args, " "))