From 2175d3808b67585f64102f83ecbf0a75340f6c39 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Wed, 2 Oct 2024 08:52:39 +0200 Subject: [PATCH 01/43] Update sdk-go/v2 from 2.10.1 to 2.15.2 to fix CVE-2024-28110 (#5050) --- go.mod | 2 +- go.sum | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index fb4bc48b80..72a0c02072 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/bmatcuk/doublestar v1.3.4 github.com/bndr/gojenkins v1.1.1-0.20240109173050-c316119c46d5 github.com/buildpacks/lifecycle v0.18.5 - github.com/cloudevents/sdk-go/v2 v2.10.1 + github.com/cloudevents/sdk-go/v2 v2.15.2 github.com/docker/cli v27.1.0+incompatible github.com/docker/docker v27.1.1+incompatible github.com/evanphx/json-patch v5.7.0+incompatible diff --git a/go.sum b/go.sum index a03a5990bd..23fb9a724f 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go/v2 v2.10.1 h1:qNFovJ18fWOd8Q9ydWJPk1oiFudXyv1GxJIP7MwPjuM= -github.com/cloudevents/sdk-go/v2 v2.10.1/go.mod h1:GpCBmUj7DIRiDhVvsK5d6WCbgTWs8DxAWTRtAwQmIXs= +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -619,7 +619,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -986,13 +985,10 @@ go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lI go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1227,7 +1223,6 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 5230c3d45447fa91468277fb4a0d3b30aac9d39a Mon Sep 17 00:00:00 2001 From: Manjunath Date: Wed, 2 Oct 2024 09:34:34 +0200 Subject: [PATCH 02/43] Include purl info in the event (#5092) --- cmd/mavenBuild.go | 40 ++++++++- cmd/mavenBuild_test.go | 105 +++++++++++++++++++++++ pkg/npm/publish.go | 16 ++++ pkg/npm/publish_test.go | 44 ++++++++++ pkg/piperutils/cyclonedxBom.go | 48 +++++++++++ pkg/piperutils/cyclonedxbom_test.go | 126 ++++++++++++++++++++++++++++ pkg/versioning/versioning.go | 1 + 7 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 pkg/piperutils/cyclonedxBom.go create mode 100644 pkg/piperutils/cyclonedxbom_test.go diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index af6461a8af..552d7b4e65 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -27,7 +27,6 @@ const ( func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) { utils := maven.NewUtilsBundle() - // enables url-log.json creation cmd := reflect.ValueOf(utils).Elem().FieldByName("Command") if cmd.IsValid() { @@ -62,7 +61,7 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat } if config.CreateBOM { - goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeBom", "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") createBOMConfig := []string{ "-DschemaVersion=1.4", "-DincludeBomSerialNumber=true", @@ -183,6 +182,7 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat } else { coordinate.BuildPath = filepath.Dir(match) coordinate.URL = config.AltDeploymentRepositoryURL + coordinate.PURL = getPurlForThePomAndDeleteIndividualBom(match) buildCoordinates = append(buildCoordinates, coordinate) } } @@ -209,6 +209,42 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat return err } +func getPurlForThePomAndDeleteIndividualBom(pomFilePath string) string { + bomPath := filepath.Join(filepath.Dir(pomFilePath) + "/target/" + mvnBomFilename + ".xml") + exists, _ := piperutils.FileExists(bomPath) + if !exists { + log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", bomPath) + return "" + } + bom, err := piperutils.GetBom(bomPath) + if err != nil { + log.Entry().Warnf("failed to get bom file %s: %v", bomPath, err) + return "" + } + + log.Entry().Debugf("Found purl: %s for the bomPath: %s", bom.Metadata.Component.Purl, bomPath) + purl := bom.Metadata.Component.Purl + + // Check if the BOM is an aggregated BOM + if !isAggregatedBOM(bom) { + // Delete the individual BOM file + err = os.Remove(bomPath) + if err != nil { + log.Entry().Warnf("failed to delete bom file %s: %v", bomPath, err) + } + } + return purl +} + +func isAggregatedBOM(bom piperutils.Bom) bool { + for _, property := range bom.Metadata.Properties { + if property.Name == "maven.goal" && property.Value == "makeAggregateBom" { + return true + } + } + return false +} + func createOrUpdateProjectSettingsXML(projectSettingsFile string, altDeploymentRepositoryID string, altDeploymentRepositoryUser string, altDeploymentRepositoryPassword string, utils maven.Utils) (string, error) { if len(projectSettingsFile) > 0 { projectSettingsFilePath, err := maven.UpdateProjectSettingsXML(projectSettingsFile, altDeploymentRepositoryID, altDeploymentRepositoryUser, altDeploymentRepositoryPassword, utils) diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index c7f6bb3d59..ca48d89f95 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -4,8 +4,11 @@ package cmd import ( + "os" + "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) @@ -157,3 +160,105 @@ func TestMavenBuild(t *testing.T) { }) } + +func TestIsAggregatedBOM(t *testing.T) { + t.Run("is aggregated BOM", func(t *testing.T) { + bom := piperutils.Bom{ + Metadata: piperutils.Metadata{ + Properties: []piperutils.BomProperty{ + {Name: "maven.goal", Value: "makeAggregateBom"}, + }, + }, + } + assert.True(t, isAggregatedBOM(bom)) + }) + + t.Run("is not aggregated BOM", func(t *testing.T) { + bom := piperutils.Bom{ + Metadata: piperutils.Metadata{ + Properties: []piperutils.BomProperty{ + {Name: "some.property", Value: "someValue"}, + }, + }, + } + assert.False(t, isAggregatedBOM(bom)) + }) +} + +func createTempFile(t *testing.T, dir string, filename string, content string) string { + filePath := filepath.Join(dir, filename) + err := os.WriteFile(filePath, []byte(content), 0666) + if err != nil { + t.Fatalf("Failed to create temp file: %s", err) + } + return filePath +} + +func TestGetPurlForThePomAndDeleteIndividualBom(t *testing.T) { + t.Run("valid BOM file, non-aggregated", func(t *testing.T) { + tempDir, err := piperutils.Files{}.TempDir("", "test") + if err != nil { + t.Fatalf("Failed to create temp directory: %s", err) + } + + bomContent := ` + + + pkg:maven/com.example/mycomponent@1.0.0 + + + + + + ` + pomFilePath := createTempFile(t, tempDir, "pom.xml", "") + bomDir := filepath.Join(tempDir, "target") + if err := os.MkdirAll(bomDir, 0777); err != nil { + t.Fatalf("Failed to create temp directory: %s", err) + } + bomFilePath := createTempFile(t, bomDir, mvnBomFilename+".xml", bomContent) + defer os.Remove(bomFilePath) + + purl := getPurlForThePomAndDeleteIndividualBom(pomFilePath) + assert.Equal(t, "pkg:maven/com.example/mycomponent@1.0.0", purl) + _, err = os.Stat(bomFilePath) + assert.True(t, os.IsNotExist(err)) + }) + + t.Run("valid BOM file, aggregated BOM", func(t *testing.T) { + tempDir, err := piperutils.Files{}.TempDir("", "test") + if err != nil { + t.Fatalf("Failed to create temp directory: %s", err) + } + + bomContent := ` + + + pkg:maven/com.example/aggregatecomponent@1.0.0 + + + + + + ` + pomFilePath := createTempFile(t, tempDir, "pom.xml", "") + bomDir := filepath.Join(tempDir, "target") + if err := os.MkdirAll(bomDir, 0777); err != nil { + t.Fatalf("Failed to create temp directory: %s", err) + } + bomFilePath := createTempFile(t, bomDir, mvnBomFilename+".xml", bomContent) + + purl := getPurlForThePomAndDeleteIndividualBom(pomFilePath) + assert.Equal(t, "pkg:maven/com.example/aggregatecomponent@1.0.0", purl) + _, err = os.Stat(bomFilePath) + assert.False(t, os.IsNotExist(err)) // File should not be deleted + }) + + t.Run("BOM file does not exist", func(t *testing.T) { + tempDir := t.TempDir() + pomFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file + + purl := getPurlForThePomAndDeleteIndividualBom(pomFilePath) + assert.Equal(t, "", purl) + }) +} diff --git a/pkg/npm/publish.go b/pkg/npm/publish.go index 660e1ae114..3466e89aea 100644 --- a/pkg/npm/publish.go +++ b/pkg/npm/publish.go @@ -217,6 +217,7 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p coordinate.BuildPath = filepath.Dir(packageJSON) coordinate.URL = registry coordinate.Packaging = "tgz" + coordinate.PURL = getPurl(packageJSON) *buildCoordinates = append(*buildCoordinates, coordinate) } @@ -225,6 +226,21 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p return nil } +func getPurl(packageJSON string) string { + expectedBomFilePath := filepath.Join(filepath.Dir(packageJSON) + "/" + npmBomFilename) + exists, _ := CredentialUtils.FileExists(expectedBomFilePath) + if !exists { + log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", expectedBomFilePath) + return "" + } + bom, err := CredentialUtils.GetBom(expectedBomFilePath) + if err != nil { + log.Entry().Warnf("unable to get bom metdata : %v", err) + return "" + } + return bom.Metadata.Component.Purl +} + func (exec *Execute) readPackageScope(packageJSON string) (string, error) { b, err := exec.Utils.FileRead(packageJSON) if err != nil { diff --git a/pkg/npm/publish_test.go b/pkg/npm/publish_test.go index 782e1b7c5c..4c7b24f95d 100644 --- a/pkg/npm/publish_test.go +++ b/pkg/npm/publish_test.go @@ -12,6 +12,7 @@ import ( "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/versioning" "github.com/stretchr/testify/assert" + "os" ) type npmMockUtilsBundleRelativeGlob struct { @@ -573,3 +574,46 @@ func TestNpmPublish(t *testing.T) { }) } } + +func createTempFile(t *testing.T, dir string, filename string, content string) string { + filePath := filepath.Join(dir, filename) + err := os.WriteFile(filePath, []byte(content), 0666) + if err != nil { + t.Fatalf("Failed to create temp file: %s", err) + } + return filePath +} + +func TestGetPurl(t *testing.T) { + t.Run("valid BOM file", func(t *testing.T) { + tempDir, err := piperutils.Files{}.TempDir("", "test") + if err != nil { + t.Fatalf("Failed to create temp directory: %s", err) + } + + bomContent := ` + + + pkg:npm/com.example/mycomponent@1.0.0 + + + + + + ` + packageJsonFilePath := createTempFile(t, tempDir, "package.json", "") + bomFilePath := createTempFile(t, tempDir, npmBomFilename, bomContent) + defer os.Remove(bomFilePath) + + purl := getPurl(packageJsonFilePath) + assert.Equal(t, "pkg:npm/com.example/mycomponent@1.0.0", purl) + }) + + t.Run("BOM file does not exist", func(t *testing.T) { + tempDir := t.TempDir() + packageJsonFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file + + purl := getPurl(packageJsonFilePath) + assert.Equal(t, "", purl) + }) +} diff --git a/pkg/piperutils/cyclonedxBom.go b/pkg/piperutils/cyclonedxBom.go new file mode 100644 index 0000000000..fddebae8a6 --- /dev/null +++ b/pkg/piperutils/cyclonedxBom.go @@ -0,0 +1,48 @@ +package piperutils + +import ( + "encoding/xml" + "github.com/SAP/jenkins-library/pkg/log" + "io" + "os" +) + +// To serialize the cyclonedx BOM file +type Bom struct { + Metadata Metadata `xml:"metadata"` +} + +type Metadata struct { + Component BomComponent `xml:"component"` + Properties []BomProperty `xml:"properties>property"` +} + +type BomProperty struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +type BomComponent struct { + Purl string `xml:"purl"` +} + +func GetBom(absoluteBomPath string) (Bom, error) { + xmlFile, err := os.Open(absoluteBomPath) + if err != nil { + log.Entry().Debugf("failed to open bom file %s", absoluteBomPath) + return Bom{}, err + } + defer xmlFile.Close() + byteValue, err := io.ReadAll(xmlFile) + if err != nil { + log.Entry().Debugf("failed to read bom file %s", absoluteBomPath) + return Bom{}, err + } + var bom Bom + err = xml.Unmarshal(byteValue, &bom) + if err != nil { + log.Entry().Debugf("failed to unmarshal bom file %s", absoluteBomPath) + return Bom{}, err + } + return bom, nil +} diff --git a/pkg/piperutils/cyclonedxbom_test.go b/pkg/piperutils/cyclonedxbom_test.go new file mode 100644 index 0000000000..d9d25e6256 --- /dev/null +++ b/pkg/piperutils/cyclonedxbom_test.go @@ -0,0 +1,126 @@ +package piperutils + +import ( + "os" + "path/filepath" + "testing" +) + +func createTempFile(t *testing.T, content string) (string, func()) { + dir := t.TempDir() + fileName := filepath.Join(dir, "test.xml") + err := os.WriteFile(fileName, []byte(content), 0666) + if err != nil { + t.Fatalf("Failed to create temp file: %s", err) + } + return fileName, func() { + os.Remove(fileName) + } +} + +func TestGetBom(t *testing.T) { + tests := []struct { + name string + xmlContent string + expectedBom Bom + expectError bool + expectedError string + }{ + { + name: "valid file", + xmlContent: ` + + + pkg:maven/com.example/mycomponent@1.0.0 + + + + + + + `, + expectedBom: Bom{ + Metadata: Metadata{ + Component: BomComponent{ + Purl: "pkg:maven/com.example/mycomponent@1.0.0", + }, + Properties: []BomProperty{ + {Name: "name1", Value: "value1"}, + {Name: "name2", Value: "value2"}, + }, + }, + }, + expectError: false, + }, + { + name: "file not found", + xmlContent: "", + expectedBom: Bom{}, + expectError: true, + expectedError: "no such file or directory", + }, + { + name: "invalid XML file", + xmlContent: "invalid xml", + expectedBom: Bom{}, + expectError: true, + expectedError: "XML syntax error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var fileName string + var cleanup func() + if tt.xmlContent != "" { + var err error + fileName, cleanup = createTempFile(t, tt.xmlContent) + defer cleanup() + if err != nil { + t.Fatalf("Failed to create temp file: %s", err) + } + } else { + // Use a non-existent file path + fileName = "nonexistent.xml" + } + + bom, err := GetBom(fileName) + if (err != nil) != tt.expectError { + t.Errorf("Expected error: %v, got: %v", tt.expectError, err) + } + + if err != nil && !tt.expectError { + if !tt.expectError && !containsSubstring(err.Error(), tt.expectedError) { + t.Errorf("Expected error message: %v, got: %v", tt.expectedError, err.Error()) + } + } + + if !tt.expectError && !bomEquals(bom, tt.expectedBom) { + t.Errorf("Expected BOM: %+v, got: %+v", tt.expectedBom, bom) + } + }) + } +} + +func bomEquals(a, b Bom) bool { + // compare a and b manually since reflect.DeepEqual can be problematic with slices and nil values + return a.Metadata.Component.Purl == b.Metadata.Component.Purl && + len(a.Metadata.Properties) == len(b.Metadata.Properties) && + propertiesMatch(a.Metadata.Properties, b.Metadata.Properties) +} + +func propertiesMatch(a, b []BomProperty) bool { + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func containsSubstring(str, substr string) bool { + if len(substr) == 0 { + return true + } + return len(str) >= len(substr) && str[:len(substr)] == substr +} diff --git a/pkg/versioning/versioning.go b/pkg/versioning/versioning.go index 1ea42b48f8..a716cb192f 100644 --- a/pkg/versioning/versioning.go +++ b/pkg/versioning/versioning.go @@ -19,6 +19,7 @@ type Coordinates struct { Packaging string BuildPath string URL string + PURL string } // Artifact defines the versioning operations for various build tools From ef25e31acba5cf40ac9b1fce635144a75edcb485 Mon Sep 17 00:00:00 2001 From: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:33:53 +0200 Subject: [PATCH 03/43] fix(vault): properly consider vault parameters & flags (#5118) --- cmd/getConfig.go | 33 +++++++++++++++++++-------------- pkg/config/config.go | 22 ++++++++++++++++++---- pkg/config/vault.go | 2 -- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/cmd/getConfig.go b/cmd/getConfig.go index aa4734a75c..0712f22e84 100644 --- a/cmd/getConfig.go +++ b/cmd/getConfig.go @@ -65,7 +65,7 @@ func ConfigCommand() *cobra.Command { OpenFile: config.OpenPiperFile, }) - var createConfigCmd = &cobra.Command{ + createConfigCmd := &cobra.Command{ Use: "getConfig", Short: "Loads the project 'Piper' configuration respecting defaults and parameters.", PreRun: func(cmd *cobra.Command, args []string) { @@ -76,7 +76,7 @@ func ConfigCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) }, Run: func(cmd *cobra.Command, _ []string) { - if err := generateConfigWrapper(); err != nil { + if err := generateConfigWrapper(cmd); err != nil { log.SetErrorCategory(log.ErrorConfiguration) log.Entry().WithError(err).Fatal("failed to retrieve configuration") } @@ -152,6 +152,10 @@ func GetStageConfig() (config.StepConfig, error) { } func getConfig() (config.StepConfig, error) { + return getConfigWithFlagValues(nil) +} + +func getConfigWithFlagValues(cmd *cobra.Command) (config.StepConfig, error) { var myConfig config.Config var stepConfig config.StepConfig var err error @@ -208,13 +212,16 @@ func getConfig() (config.StepConfig, error) { } } - var flags map[string]interface{} - if configOptions.ContextConfig { metadata.Spec.Inputs.Parameters = []config.StepParameters{} } - stepConfig, err = myConfig.GetStepConfig(flags, GeneralConfig.ParametersJSON, customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, paramFilter, metadata, resourceParams, GeneralConfig.StageName, metadata.Metadata.Name) + var flagValues map[string]interface{} + if cmd != nil { + flagValues = config.AvailableFlagValues(cmd, ¶mFilter) + } + + stepConfig, err = myConfig.GetStepConfig(flagValues, GeneralConfig.ParametersJSON, customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, paramFilter, metadata, resourceParams, GeneralConfig.StageName, metadata.Metadata.Name) if err != nil { return stepConfig, errors.Wrap(err, "getting step config failed") } @@ -227,7 +234,7 @@ func getConfig() (config.StepConfig, error) { return stepConfig, nil } -func generateConfigWrapper() error { +func generateConfigWrapper(cmd *cobra.Command) error { var formatter func(interface{}) (string, error) switch strings.ToLower(configOptions.Output) { case "yaml", "yml": @@ -237,13 +244,13 @@ func generateConfigWrapper() error { default: formatter = config.GetJSON } - return GenerateConfig(formatter) + return GenerateConfig(cmd, formatter) } -func GenerateConfig(formatter func(interface{}) (string, error)) error { +func GenerateConfig(cmd *cobra.Command, formatter func(interface{}) (string, error)) error { utils := newGetConfigUtilsUtils() - stepConfig, err := getConfig() + stepConfig, err := getConfigWithFlagValues(cmd) if err != nil { return err } @@ -254,7 +261,7 @@ func GenerateConfig(formatter func(interface{}) (string, error)) error { } if len(configOptions.OutputFile) > 0 { - if err := utils.FileWrite(configOptions.OutputFile, []byte(myConfig), 0666); err != nil { + if err := utils.FileWrite(configOptions.OutputFile, []byte(myConfig), 0o666); err != nil { return fmt.Errorf("failed to write output file %v: %w", configOptions.OutputFile, err) } return nil @@ -265,7 +272,6 @@ func GenerateConfig(formatter func(interface{}) (string, error)) error { } func addConfigFlags(cmd *cobra.Command) { - // ToDo: support more output options, like https://kubernetes.io/docs/reference/kubectl/overview/#formatting-output cmd.Flags().StringVar(&configOptions.Output, "output", "json", "Defines the output format") cmd.Flags().StringVar(&configOptions.OutputFile, "outputFile", "", "Defines a file path. f set, the output will be written to the defines file") @@ -276,7 +282,6 @@ func addConfigFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&configOptions.StepMetadata, "stepMetadata", "", "Step metadata, passed as path to yaml") cmd.Flags().StringVar(&configOptions.StepName, "stepName", "", "Step name, used to get step metadata if yaml path is not set") cmd.Flags().BoolVar(&configOptions.ContextConfig, "contextConfig", false, "Defines if step context configuration should be loaded instead of step config") - } func defaultsAndFilters(metadata *config.StepData, stepName string) ([]io.ReadCloser, config.StepFilters, error) { @@ -316,7 +321,7 @@ func prepareOutputEnvironment(outputResources []config.StepResources, envRootPat } if _, err := os.Stat(filepath.Dir(paramPath)); errors.Is(err, os.ErrNotExist) { log.Entry().Debugf("Creating directory: %v", filepath.Dir(paramPath)) - _ = os.MkdirAll(filepath.Dir(paramPath), 0777) + _ = os.MkdirAll(filepath.Dir(paramPath), 0o777) } } } @@ -332,7 +337,7 @@ func prepareOutputEnvironment(outputResources []config.StepResources, envRootPat for _, dir := range stepOutputDirectories { if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { log.Entry().Debugf("Creating directory: %v", dir) - _ = os.MkdirAll(dir, 0777) + _ = os.MkdirAll(dir, 0o777) } } } diff --git a/pkg/config/config.go b/pkg/config/config.go index e35aa589e7..13e91771f3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -218,6 +218,8 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri // merge parameters provided via env vars stepConfig.mixIn(envValues(filters.All), filters.All, metadata) + vaultParams := map[string]interface{}{} + // if parameters are provided in JSON format merge them if len(paramJSON) != 0 { var params map[string]interface{} @@ -228,10 +230,17 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri // apply aliases for _, p := range parameters { params = setParamValueFromAlias(stepName, params, filters.Parameters, p.Name, p.Aliases) + vaultParams = setParamValueFromAlias(stepName, vaultParams, vaultFilter, p.Name, p.Aliases) } for _, s := range secrets { params = setParamValueFromAlias(stepName, params, filters.Parameters, s.Name, s.Aliases) } + // retrieve Vault config if provided + for _, v := range vaultFilter { + if params[v] != nil { + vaultParams[v] = params[v] + } + } stepConfig.mixIn(params, filters.Parameters, metadata) } @@ -239,8 +248,13 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri // merge command line flags if flagValues != nil { - flagFilter := append(filters.Parameters, vaultFilter...) - stepConfig.mixIn(flagValues, flagFilter, metadata) + stepConfig.mixIn(flagValues, filters.Parameters, metadata) + // retrieve Vault config from flags if provided + for _, v := range vaultFilter { + if flagValues[v] != nil { + vaultParams[v] = flagValues[v] + } + } } if verbose, ok := stepConfig.Config["verbose"].(bool); ok && verbose { @@ -249,7 +263,7 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri log.Entry().Warnf("invalid value for parameter verbose: '%v'", stepConfig.Config["verbose"]) } - stepConfig.mixinVaultConfig(parameters, c.General, c.Steps[stepName], c.Stages[stageName]) + stepConfig.mixinVaultConfig(parameters, c.General, c.Steps[stepName], c.Stages[stageName], vaultParams) reportingConfig, err := cloneConfig(c) if err != nil { @@ -517,7 +531,7 @@ func merge(base, overlay map[string]interface{}, metadata StepData) map[string]i tVal := reflect.TypeOf(value).String() if v.Name == key && tVal != v.Type { if tVal == "[]interface {}" && v.Type == "[]string" { - //json Unmarshal genertes arrays of interface{} for string arrays + // json Unmarshal genertes arrays of interface{} for string arrays for _, interfaceValue := range value.([]interface{}) { arrayValueType := reflect.TypeOf(interfaceValue).String() if arrayValueType != "string" { diff --git a/pkg/config/vault.go b/pkg/config/vault.go index 8803268de6..d9cd73c85d 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -306,7 +306,6 @@ func resolveVaultCredentials(config *StepConfig, client VaultClient) { } func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) { - vaultTestCredentialEnvPrefix, ok := config.Config["vaultTestCredentialEnvPrefix"].(string) if !ok || len(vaultTestCredentialEnvPrefix) == 0 { vaultTestCredentialEnvPrefix = vaultTestCredentialEnvPrefixDefault @@ -326,7 +325,6 @@ func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string, } func populateCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) { - vaultCredentialEnvPrefix, ok := config.Config["vaultCredentialEnvPrefix"].(string) isCredentialEnvPrefixDefault := false From 60cbddbd47b0b455219c8450702304ef99d98611 Mon Sep 17 00:00:00 2001 From: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:55:20 +0500 Subject: [PATCH 04/43] `npm config` doesn't fail when `workspaces` enabled (#5120) * Add -ws=false -iwr to npm config get registry * Add -ws=false -iwr to npm config set registry * Fix test * Fix test --------- Co-authored-by: Manjunath --- cmd/npmExecuteScripts_test.go | 2 +- pkg/npm/npm.go | 4 ++-- pkg/npm/npm_test.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/npmExecuteScripts_test.go b/cmd/npmExecuteScripts_test.go index af1b7f828a..3c7bf6e995 100644 --- a/cmd/npmExecuteScripts_test.go +++ b/cmd/npmExecuteScripts_test.go @@ -148,7 +148,7 @@ func TestNpmExecuteScripts(t *testing.T) { if assert.NoError(t, err) { if assert.Equal(t, 4, len(utils.execRunner.Calls)) { - assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"config", "get", "registry"}}, utils.execRunner.Calls[0]) + assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"config", "get", "registry", "-ws=false", "-iwr"}}, utils.execRunner.Calls[0]) assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"ci"}}, utils.execRunner.Calls[1]) assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run", "ci-build"}}, utils.execRunner.Calls[3]) } diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index 4c8130c9b6..6e36ace72f 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -96,7 +96,7 @@ func (exec *Execute) SetNpmRegistries() error { var buffer bytes.Buffer execRunner.Stdout(&buffer) - err := execRunner.RunExecutable("npm", "config", "get", npmRegistry) + err := execRunner.RunExecutable("npm", "config", "get", npmRegistry, "-ws=false", "-iwr") execRunner.Stdout(log.Writer()) if err != nil { return err @@ -109,7 +109,7 @@ func (exec *Execute) SetNpmRegistries() error { if exec.Options.DefaultNpmRegistry != "" && registryRequiresConfiguration(preConfiguredRegistry, "https://registry.npmjs.org") { log.Entry().Info("npm registry " + npmRegistry + " was not configured, setting it to " + exec.Options.DefaultNpmRegistry) - err = execRunner.RunExecutable("npm", "config", "set", npmRegistry, exec.Options.DefaultNpmRegistry) + err = execRunner.RunExecutable("npm", "config", "set", npmRegistry, exec.Options.DefaultNpmRegistry, "-ws=false", "-iwr") if err != nil { return err } diff --git a/pkg/npm/npm_test.go b/pkg/npm/npm_test.go index 52ef6e085a..ea4677ee3c 100644 --- a/pkg/npm/npm_test.go +++ b/pkg/npm/npm_test.go @@ -299,7 +299,7 @@ func TestNpm(t *testing.T) { utils := newNpmMockUtilsBundle() utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }")) utils.AddFile(filepath.Join("src", "package.json"), []byte("{\"scripts\": { \"ci-build\": \"exit 0\" } }")) - utils.execRunner = &mock.ExecMockRunner{StdoutReturn: map[string]string{"npm config get registry": "undefined"}} + utils.execRunner = &mock.ExecMockRunner{StdoutReturn: map[string]string{"npm config get registry -ws=false -iwr": "undefined"}} options := ExecutorOptions{} options.DefaultNpmRegistry = "https://example.org/npm" @@ -311,8 +311,8 @@ func TestNpm(t *testing.T) { if assert.NoError(t, err) { if assert.Equal(t, 2, len(utils.execRunner.Calls)) { - assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"config", "get", "registry"}}, utils.execRunner.Calls[0]) - assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"config", "set", "registry", exec.Options.DefaultNpmRegistry}}, utils.execRunner.Calls[1]) + assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"config", "get", "registry", "-ws=false", "-iwr"}}, utils.execRunner.Calls[0]) + assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"config", "set", "registry", exec.Options.DefaultNpmRegistry, "-ws=false", "-iwr"}}, utils.execRunner.Calls[1]) } } }) From f6231de55beed46d1b069c4e074497ee70634a5a Mon Sep 17 00:00:00 2001 From: Rinita Asani Date: Mon, 7 Oct 2024 16:08:27 +0200 Subject: [PATCH 05/43] gctsExecuteABAPQualityCheck type mismatch (#5132) * Adapting unit test to run for packages with namespace * Correct type mismatch for object count --- cmd/gctsExecuteABAPQualityChecks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gctsExecuteABAPQualityChecks.go b/cmd/gctsExecuteABAPQualityChecks.go index 30d190655b..c03993d448 100644 --- a/cmd/gctsExecuteABAPQualityChecks.go +++ b/cmd/gctsExecuteABAPQualityChecks.go @@ -1900,7 +1900,7 @@ type repository struct { CreatedBy string `json:"createdBy"` CreatedDate string `json:"createdDate"` Config []repoConfig `json:"config"` - Objects int `json:"objects"` + Objects any `json:"objects"` CurrentCommit string `json:"currentCommit"` } From c991c5b16d1ca30804cdb5e53553ee9116f46a69 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 8 Oct 2024 09:56:32 +0200 Subject: [PATCH 06/43] Fix maven build failures due to makeBom goal (#5134) --- cmd/mavenBuild.go | 224 ++++++++++++++++++++++++----------------- cmd/mavenBuild_test.go | 166 +++++++----------------------- 2 files changed, 169 insertions(+), 221 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 552d7b4e65..45b09e40e1 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -22,7 +22,8 @@ import ( ) const ( - mvnBomFilename = "bom-maven" + mvnBomFilename = "bom-maven" + mvnSimpleBomFilename = "bom-simple" ) func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) { @@ -39,6 +40,53 @@ func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, c } } +func executeMavenGoals(config *mavenBuildOptions, utils maven.Utils, flags []string, goals []string, defines []string) error { + mavenOptions := maven.ExecuteOptions{ + Flags: flags, + Goals: goals, + Defines: defines, + PomPath: config.PomPath, + ProjectSettingsFile: config.ProjectSettingsFile, + GlobalSettingsFile: config.GlobalSettingsFile, + M2Path: config.M2Path, + LogSuccessfulMavenTransfers: config.LogSuccessfulMavenTransfers, + } + + _, err := maven.Execute(&mavenOptions, utils) + return err +} + +func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { + var flags = []string{"-update-snapshots", "--batch-mode"} + if len(config.Profiles) > 0 { + flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) + } + exists, _ := utils.FileExists("integration-tests/pom.xml") + if exists { + flags = append(flags, "-pl", "!integration-tests") + } + + var defines []string + + createBOMConfig := []string{ + "-DschemaVersion=1.4", + "-DincludeBomSerialNumber=true", + "-DincludeCompileScope=true", + "-DincludeProvidedScope=true", + "-DincludeRuntimeScope=true", + "-DincludeSystemScope=true", + "-DincludeTestScope=false", + "-DincludeLicenseText=false", + "-DoutputFormat=xml", + "-DoutputName=" + mvnSimpleBomFilename, + } + defines = append(defines, createBOMConfig...) + + goals := []string{"org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeBom"} + + return executeMavenGoals(config, utils, flags, goals, defines) +} + func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { var flags = []string{"-update-snapshots", "--batch-mode"} @@ -61,7 +109,13 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat } if config.CreateBOM { - goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeBom", "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + // Separate run for makeBOM goal + if err := runMakeBOMGoal(config, utils); err != nil { + return errors.Wrap(err, "failed to execute makeBOM goal") + } + + // Append the makeAggregateBOM goal to the rest of the goals + goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") createBOMConfig := []string{ "-DschemaVersion=1.4", "-DincludeBomSerialNumber=true", @@ -85,20 +139,7 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat goals = append(goals, "install") } - mavenOptions := maven.ExecuteOptions{ - Flags: flags, - Goals: goals, - Defines: defines, - PomPath: config.PomPath, - ProjectSettingsFile: config.ProjectSettingsFile, - GlobalSettingsFile: config.GlobalSettingsFile, - M2Path: config.M2Path, - LogSuccessfulMavenTransfers: config.LogSuccessfulMavenTransfers, - } - - _, err := maven.Execute(&mavenOptions, utils) - - if err != nil { + if err := executeMavenGoals(config, utils, flags, goals, defines); err != nil { return errors.Wrapf(err, "failed to execute maven build for goal(s) '%v'", goals) } @@ -133,84 +174,96 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat if err != nil { return errors.Wrap(err, "Could not create or update project settings xml") } - mavenOptions.ProjectSettingsFile = projectSettingsFilePath - } + mavenOptions := maven.ExecuteOptions{ + ProjectSettingsFile: projectSettingsFilePath, + } - deployFlags := []string{} - if len(config.DeployFlags) > 0 { - deployFlags = append(deployFlags, config.DeployFlags...) - } - if (len(config.AltDeploymentRepositoryID) > 0) && (len(config.AltDeploymentRepositoryURL) > 0) { - deployFlags = append(deployFlags, "-DaltDeploymentRepository="+config.AltDeploymentRepositoryID+"::default::"+config.AltDeploymentRepositoryURL) - } + deployFlags := []string{} + if len(config.DeployFlags) > 0 { + deployFlags = append(deployFlags, config.DeployFlags...) + } + if (len(config.AltDeploymentRepositoryID) > 0) && (len(config.AltDeploymentRepositoryURL) > 0) { + deployFlags = append(deployFlags, "-DaltDeploymentRepository="+config.AltDeploymentRepositoryID+"::default::"+config.AltDeploymentRepositoryURL) + } - downloadClient := &piperhttp.Client{} - downloadClient.SetOptions(piperhttp.ClientOptions{}) - runner := &command.Command{ - StepName: "mavenBuild", - } - fileUtils := &piperutils.Files{} - if len(config.CustomTLSCertificateLinks) > 0 { - if err := loadRemoteRepoCertificates(config.CustomTLSCertificateLinks, downloadClient, &deployFlags, runner, fileUtils, config.JavaCaCertFilePath); err != nil { - log.SetErrorCategory(log.ErrorInfrastructure) - return err + downloadClient := &piperhttp.Client{} + downloadClient.SetOptions(piperhttp.ClientOptions{}) + + runner := &command.Command{ + StepName: "mavenBuild", } - } - mavenOptions.Flags = deployFlags - mavenOptions.Goals = []string{"deploy"} - mavenOptions.Defines = []string{} - _, err := maven.Execute(&mavenOptions, utils) - if err != nil { - return err - } - if config.CreateBuildArtifactsMetadata { - buildCoordinates := []versioning.Coordinates{} - options := versioning.Options{} - var utils versioning.Utils - - matches, _ := fileUtils.Glob("**/pom.xml") - for _, match := range matches { - - artifact, err := versioning.GetArtifact("maven", match, &options, utils) - if err != nil { - log.Entry().Warnf("unable to get artifact metdata : %v", err) - } else { - coordinate, err := artifact.GetCoordinates() - if err != nil { - log.Entry().Warnf("unable to get artifact coordinates : %v", err) - } else { - coordinate.BuildPath = filepath.Dir(match) - coordinate.URL = config.AltDeploymentRepositoryURL - coordinate.PURL = getPurlForThePomAndDeleteIndividualBom(match) - buildCoordinates = append(buildCoordinates, coordinate) - } + fileUtils := &piperutils.Files{} + if len(config.CustomTLSCertificateLinks) > 0 { + if err := loadRemoteRepoCertificates(config.CustomTLSCertificateLinks, downloadClient, &deployFlags, runner, fileUtils, config.JavaCaCertFilePath); err != nil { + log.SetErrorCategory(log.ErrorInfrastructure) + return err } } - if len(buildCoordinates) == 0 { - log.Entry().Warnf("unable to identify artifact coordinates for the maven packages published") - return nil + mavenOptions.Flags = deployFlags + mavenOptions.Goals = []string{"deploy"} + mavenOptions.Defines = []string{} + _, err = maven.Execute(&mavenOptions, utils) + if err != nil { + return err } - var buildArtifacts build.BuildArtifacts - - buildArtifacts.Coordinates = buildCoordinates - jsonResult, _ := json.Marshal(buildArtifacts) - commonPipelineEnvironment.custom.mavenBuildArtifacts = string(jsonResult) + if config.CreateBuildArtifactsMetadata { + err2, done := createBuildArtifactsMetadata(config, commonPipelineEnvironment) + if done { + return err2 + } + } + return nil } - - return nil } else { log.Entry().Infof("publish not detected, ignoring maven deploy") } } - return err } -func getPurlForThePomAndDeleteIndividualBom(pomFilePath string) string { - bomPath := filepath.Join(filepath.Dir(pomFilePath) + "/target/" + mvnBomFilename + ".xml") +func createBuildArtifactsMetadata(config *mavenBuildOptions, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) (error, bool) { + fileUtils := &piperutils.Files{} + buildCoordinates := []versioning.Coordinates{} + options := versioning.Options{} + var utils versioning.Utils + + matches, _ := fileUtils.Glob("**/pom.xml") + for _, match := range matches { + + artifact, err := versioning.GetArtifact("maven", match, &options, utils) + if err != nil { + log.Entry().Warnf("unable to get artifact metdata : %v", err) + } else { + coordinate, err := artifact.GetCoordinates() + if err != nil { + log.Entry().Warnf("unable to get artifact coordinates : %v", err) + } else { + coordinate.BuildPath = filepath.Dir(match) + coordinate.URL = config.AltDeploymentRepositoryURL + coordinate.PURL = getPurlForThePom(match) + buildCoordinates = append(buildCoordinates, coordinate) + } + } + } + + if len(buildCoordinates) == 0 { + log.Entry().Warnf("unable to identify artifact coordinates for the maven packages published") + return nil, true + } + + var buildArtifacts build.BuildArtifacts + + buildArtifacts.Coordinates = buildCoordinates + jsonResult, _ := json.Marshal(buildArtifacts) + commonPipelineEnvironment.custom.mavenBuildArtifacts = string(jsonResult) + return nil, false +} + +func getPurlForThePom(pomFilePath string) string { + bomPath := filepath.Join(filepath.Dir(pomFilePath) + "/target/" + mvnSimpleBomFilename + ".xml") exists, _ := piperutils.FileExists(bomPath) if !exists { log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", bomPath) @@ -225,26 +278,9 @@ func getPurlForThePomAndDeleteIndividualBom(pomFilePath string) string { log.Entry().Debugf("Found purl: %s for the bomPath: %s", bom.Metadata.Component.Purl, bomPath) purl := bom.Metadata.Component.Purl - // Check if the BOM is an aggregated BOM - if !isAggregatedBOM(bom) { - // Delete the individual BOM file - err = os.Remove(bomPath) - if err != nil { - log.Entry().Warnf("failed to delete bom file %s: %v", bomPath, err) - } - } return purl } -func isAggregatedBOM(bom piperutils.Bom) bool { - for _, property := range bom.Metadata.Properties { - if property.Name == "maven.goal" && property.Value == "makeAggregateBom" { - return true - } - } - return false -} - func createOrUpdateProjectSettingsXML(projectSettingsFile string, altDeploymentRepositoryID string, altDeploymentRepositoryUser string, altDeploymentRepositoryPassword string, utils maven.Utils) (string, error) { if len(projectSettingsFile) > 0 { projectSettingsFilePath, err := maven.UpdateProjectSettingsXML(projectSettingsFile, altDeploymentRepositoryID, altDeploymentRepositoryUser, altDeploymentRepositoryPassword, utils) diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index ca48d89f95..3380ecf3cd 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -12,9 +12,9 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMavenBuild(t *testing.T) { +var cpe mavenBuildCommonPipelineEnvironment - cpe := mavenBuildCommonPipelineEnvironment{} +func TestMavenBuild(t *testing.T) { t.Run("mavenBuild should install the artifact", func(t *testing.T) { mockedUtils := newMavenMockUtils() @@ -22,51 +22,30 @@ func TestMavenBuild(t *testing.T) { config := mavenBuildOptions{} err := runMavenBuild(&config, nil, &mockedUtils, &cpe) + expectedParams := []string{"install"} assert.Nil(t, err) - assert.Equal(t, mockedUtils.Calls[0].Exec, "mvn") - assert.Contains(t, mockedUtils.Calls[0].Params, "install") - }) - - t.Run("mavenBuild should skip integration tests", func(t *testing.T) { - mockedUtils := newMavenMockUtils() - mockedUtils.AddFile("integration-tests/pom.xml", []byte{}) - - config := mavenBuildOptions{} - - err := runMavenBuild(&config, nil, &mockedUtils, &cpe) - - assert.Nil(t, err) - assert.Equal(t, mockedUtils.Calls[0].Exec, "mvn") - assert.Contains(t, mockedUtils.Calls[0].Params, "-pl", "!integration-tests") - }) - - t.Run("mavenBuild should flatten", func(t *testing.T) { - mockedUtils := newMavenMockUtils() - - config := mavenBuildOptions{Flatten: true} - - err := runMavenBuild(&config, nil, &mockedUtils, &cpe) - - assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[0].Params, "flatten:flatten") - assert.Contains(t, mockedUtils.Calls[0].Params, "-Dflatten.mode=resolveCiFriendliesOnly") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DupdatePomFile=true") + if assert.Equal(t, 1, len(mockedUtils.Calls), "Expected one maven invocation for the main build") { + assert.Equal(t, "mvn", mockedUtils.Calls[0].Exec) + assert.Contains(t, mockedUtils.Calls[0].Params, expectedParams[0], "Call should contain install goal") + } }) - t.Run("mavenBuild should run only verify", func(t *testing.T) { + t.Run("mavenBuild accepts profiles", func(t *testing.T) { mockedUtils := newMavenMockUtils() - config := mavenBuildOptions{Verify: true} + config := mavenBuildOptions{Profiles: []string{"profile1", "profile2"}} err := runMavenBuild(&config, nil, &mockedUtils, &cpe) assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[0].Params, "verify") - assert.NotContains(t, mockedUtils.Calls[0].Params, "install") + if assert.Equal(t, 1, len(mockedUtils.Calls), "Expected one maven invocation for the main build") { + assert.Contains(t, mockedUtils.Calls[0].Params, "--activate-profiles") + assert.Contains(t, mockedUtils.Calls[0].Params, "profile1,profile2") + } }) - t.Run("mavenBuild should createBOM", func(t *testing.T) { + t.Run("mavenBuild should create BOM", func(t *testing.T) { mockedUtils := newMavenMockUtils() config := mavenBuildOptions{CreateBOM: true} @@ -74,74 +53,60 @@ func TestMavenBuild(t *testing.T) { err := runMavenBuild(&config, nil, &mockedUtils, &cpe) assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[0].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DschemaVersion=1.4") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeBomSerialNumber=true") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeCompileScope=true") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeProvidedScope=true") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeRuntimeScope=true") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeSystemScope=true") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeTestScope=false") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DincludeLicenseText=false") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DoutputFormat=xml") - assert.Contains(t, mockedUtils.Calls[0].Params, "-DoutputName=bom-maven") + if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (default + makeAggregateBom)") { + assert.Equal(t, "mvn", mockedUtils.Calls[1].Exec) + assert.Contains(t, mockedUtils.Calls[1].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + assert.Contains(t, mockedUtils.Calls[1].Params, "-DoutputName=bom-maven") + } }) t.Run("mavenBuild include install and deploy when publish is true", func(t *testing.T) { mockedUtils := newMavenMockUtils() - config := mavenBuildOptions{Publish: true, Verify: false} + config := mavenBuildOptions{Publish: true, Verify: false, AltDeploymentRepositoryID: "ID", AltDeploymentRepositoryURL: "http://sampleRepo.com", AltDeploymentRepositoryUser: "user", AltDeploymentRepositoryPassword: "pass"} err := runMavenBuild(&config, nil, &mockedUtils, &cpe) assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[0].Params, "install") - assert.NotContains(t, mockedUtils.Calls[0].Params, "verify") - assert.Contains(t, mockedUtils.Calls[1].Params, "deploy") - + if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (main and deploy)") { + assert.Contains(t, mockedUtils.Calls[0].Params, "install") + assert.NotContains(t, mockedUtils.Calls[0].Params, "verify") + assert.Contains(t, mockedUtils.Calls[1].Params, "deploy") + } }) t.Run("mavenBuild with deploy must skip build, install and test", func(t *testing.T) { mockedUtils := newMavenMockUtils() - config := mavenBuildOptions{Publish: true, Verify: false, DeployFlags: []string{"-Dmaven.main.skip=true", "-Dmaven.test.skip=true", "-Dmaven.install.skip=true"}} + config := mavenBuildOptions{Publish: true, Verify: false, DeployFlags: []string{"-Dmaven.main.skip=true", "-Dmaven.test.skip=true", "-Dmaven.install.skip=true"}, AltDeploymentRepositoryID: "ID", AltDeploymentRepositoryURL: "http://sampleRepo.com", AltDeploymentRepositoryUser: "user", AltDeploymentRepositoryPassword: "pass"} err := runMavenBuild(&config, nil, &mockedUtils, &cpe) assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[1].Params, "-Dmaven.main.skip=true") - assert.Contains(t, mockedUtils.Calls[1].Params, "-Dmaven.test.skip=true") - assert.Contains(t, mockedUtils.Calls[1].Params, "-Dmaven.install.skip=true") - + if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (main and deploy)") { + assert.Contains(t, mockedUtils.Calls[1].Params, "-Dmaven.main.skip=true") + assert.Contains(t, mockedUtils.Calls[1].Params, "-Dmaven.test.skip=true") + assert.Contains(t, mockedUtils.Calls[1].Params, "-Dmaven.install.skip=true") + } }) t.Run("mavenBuild with deploy must include alt repo id and url when passed as parameter", func(t *testing.T) { mockedUtils := newMavenMockUtils() - config := mavenBuildOptions{Publish: true, Verify: false, AltDeploymentRepositoryID: "ID", AltDeploymentRepositoryURL: "http://sampleRepo.com"} + config := mavenBuildOptions{Publish: true, Verify: false, AltDeploymentRepositoryID: "ID", AltDeploymentRepositoryURL: "http://sampleRepo.com", AltDeploymentRepositoryUser: "user", AltDeploymentRepositoryPassword: "pass"} err := runMavenBuild(&config, nil, &mockedUtils, &cpe) assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[1].Params, "-DaltDeploymentRepository=ID::default::http://sampleRepo.com") - }) - - t.Run("mavenBuild accepts profiles", func(t *testing.T) { - mockedUtils := newMavenMockUtils() - - config := mavenBuildOptions{Profiles: []string{"profile1", "profile2"}} - - err := runMavenBuild(&config, nil, &mockedUtils, &cpe) - - assert.Nil(t, err) - assert.Contains(t, mockedUtils.Calls[0].Params, "--activate-profiles") - assert.Contains(t, mockedUtils.Calls[0].Params, "profile1,profile2") + if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (main and deploy)") { + assert.Contains(t, mockedUtils.Calls[1].Params, "-DaltDeploymentRepository=ID::default::http://sampleRepo.com") + } }) t.Run("mavenBuild should not create build artifacts metadata when CreateBuildArtifactsMetadata is false and Publish is true", func(t *testing.T) { mockedUtils := newMavenMockUtils() mockedUtils.AddFile("pom.xml", []byte{}) - config := mavenBuildOptions{CreateBuildArtifactsMetadata: false, Publish: true} + config := mavenBuildOptions{CreateBuildArtifactsMetadata: false, Publish: true, AltDeploymentRepositoryID: "ID", AltDeploymentRepositoryURL: "http://sampleRepo.com", AltDeploymentRepositoryUser: "user", AltDeploymentRepositoryPassword: "pass"} err := runMavenBuild(&config, nil, &mockedUtils, &cpe) assert.Nil(t, err) assert.Equal(t, mockedUtils.Calls[0].Exec, "mvn") @@ -161,30 +126,6 @@ func TestMavenBuild(t *testing.T) { } -func TestIsAggregatedBOM(t *testing.T) { - t.Run("is aggregated BOM", func(t *testing.T) { - bom := piperutils.Bom{ - Metadata: piperutils.Metadata{ - Properties: []piperutils.BomProperty{ - {Name: "maven.goal", Value: "makeAggregateBom"}, - }, - }, - } - assert.True(t, isAggregatedBOM(bom)) - }) - - t.Run("is not aggregated BOM", func(t *testing.T) { - bom := piperutils.Bom{ - Metadata: piperutils.Metadata{ - Properties: []piperutils.BomProperty{ - {Name: "some.property", Value: "someValue"}, - }, - }, - } - assert.False(t, isAggregatedBOM(bom)) - }) -} - func createTempFile(t *testing.T, dir string, filename string, content string) string { filePath := filepath.Join(dir, filename) err := os.WriteFile(filePath, []byte(content), 0666) @@ -195,35 +136,6 @@ func createTempFile(t *testing.T, dir string, filename string, content string) s } func TestGetPurlForThePomAndDeleteIndividualBom(t *testing.T) { - t.Run("valid BOM file, non-aggregated", func(t *testing.T) { - tempDir, err := piperutils.Files{}.TempDir("", "test") - if err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - - bomContent := ` - - - pkg:maven/com.example/mycomponent@1.0.0 - - - - - - ` - pomFilePath := createTempFile(t, tempDir, "pom.xml", "") - bomDir := filepath.Join(tempDir, "target") - if err := os.MkdirAll(bomDir, 0777); err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - bomFilePath := createTempFile(t, bomDir, mvnBomFilename+".xml", bomContent) - defer os.Remove(bomFilePath) - - purl := getPurlForThePomAndDeleteIndividualBom(pomFilePath) - assert.Equal(t, "pkg:maven/com.example/mycomponent@1.0.0", purl) - _, err = os.Stat(bomFilePath) - assert.True(t, os.IsNotExist(err)) - }) t.Run("valid BOM file, aggregated BOM", func(t *testing.T) { tempDir, err := piperutils.Files{}.TempDir("", "test") @@ -246,9 +158,9 @@ func TestGetPurlForThePomAndDeleteIndividualBom(t *testing.T) { if err := os.MkdirAll(bomDir, 0777); err != nil { t.Fatalf("Failed to create temp directory: %s", err) } - bomFilePath := createTempFile(t, bomDir, mvnBomFilename+".xml", bomContent) + bomFilePath := createTempFile(t, bomDir, mvnSimpleBomFilename+".xml", bomContent) - purl := getPurlForThePomAndDeleteIndividualBom(pomFilePath) + purl := getPurlForThePom(pomFilePath) assert.Equal(t, "pkg:maven/com.example/aggregatecomponent@1.0.0", purl) _, err = os.Stat(bomFilePath) assert.False(t, os.IsNotExist(err)) // File should not be deleted @@ -258,7 +170,7 @@ func TestGetPurlForThePomAndDeleteIndividualBom(t *testing.T) { tempDir := t.TempDir() pomFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file - purl := getPurlForThePomAndDeleteIndividualBom(pomFilePath) + purl := getPurlForThePom(pomFilePath) assert.Equal(t, "", purl) }) } From 033a429c49812dca0dec560eff065bd24ebbf5a2 Mon Sep 17 00:00:00 2001 From: Sarat Krishnan <78093145+sarat-krk@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:15:58 +0200 Subject: [PATCH 07/43] Fix gcts deploy field anonymize (#5129) * adding anonymisation for object field * Further fixes to gcts deploy step * syntax changes --- cmd/gctsDeploy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gctsDeploy.go b/cmd/gctsDeploy.go index d9568d0a48..82d70b252f 100644 --- a/cmd/gctsDeploy.go +++ b/cmd/gctsDeploy.go @@ -81,7 +81,7 @@ func gctsDeployRepository(config *gctsDeployOptions, telemetryData *telemetry.Cu repoMetadataInitState, getRepositoryErr := getRepository(config, httpClient) currentBranch := repoMetadataInitState.Result.Branch // If Repository does not exist in the system then Create and Clone Repository - if getRepositoryErr != nil { + if getRepositoryErr != nil || repoMetadataInitState.Result.Rid == "" { // If scope is set for a new repository then creation/cloning of the repository cannot be done if config.Scope != "" { log.Entry().Error("Error during deploy : deploy scope cannot be provided while deploying a new repo") @@ -824,7 +824,7 @@ type getRepositoryResponseBody struct { Value string `json:"value"` Category string `json:"category"` } `json:"config"` - Objects int64 `json:"objects"` + Objects any `json:"objects"` CurrentCommit string `json:"currentCommit"` Connection string `json:"connection"` } `json:"result"` From 20b7eff911b9375169e4ca89865b93928574cf8e Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 8 Oct 2024 12:58:48 +0200 Subject: [PATCH 08/43] Run makeBOM after aggregateBOM (#5140) * Run makeBOM after aggregateBOM * Fix tests --- cmd/mavenBuild.go | 12 +++++++----- cmd/mavenBuild_test.go | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 45b09e40e1..1b2a340511 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -109,11 +109,6 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat } if config.CreateBOM { - // Separate run for makeBOM goal - if err := runMakeBOMGoal(config, utils); err != nil { - return errors.Wrap(err, "failed to execute makeBOM goal") - } - // Append the makeAggregateBOM goal to the rest of the goals goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") createBOMConfig := []string{ @@ -143,6 +138,13 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat return errors.Wrapf(err, "failed to execute maven build for goal(s) '%v'", goals) } + if config.CreateBOM { + // Separate run for makeBOM goal + if err := runMakeBOMGoal(config, utils); err != nil { + return errors.Wrap(err, "failed to execute makeBOM goal") + } + } + log.Entry().Debugf("creating build settings information...") stepName := "mavenBuild" dockerImage, err := GetDockerImageValue(stepName) diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index 3380ecf3cd..ea5e9abdbc 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -55,8 +55,8 @@ func TestMavenBuild(t *testing.T) { assert.Nil(t, err) if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (default + makeAggregateBom)") { assert.Equal(t, "mvn", mockedUtils.Calls[1].Exec) - assert.Contains(t, mockedUtils.Calls[1].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") - assert.Contains(t, mockedUtils.Calls[1].Params, "-DoutputName=bom-maven") + assert.Contains(t, mockedUtils.Calls[0].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + assert.Contains(t, mockedUtils.Calls[0].Params, "-DoutputName=bom-maven") } }) From 41536720c125a20eeaa99e4f63db91fffdf7b946 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Wed, 9 Oct 2024 10:17:39 +0200 Subject: [PATCH 09/43] Flatten the simple bom (#5141) --- cmd/mavenBuild.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 1b2a340511..df775856ec 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -84,6 +84,11 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { goals := []string{"org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeBom"} + if config.Flatten { + goals = append(goals, "flatten:flatten") + defines = append(defines, "-Dflatten.mode=resolveCiFriendliesOnly", "-DupdatePomFile=true") + } + return executeMavenGoals(config, utils, flags, goals, defines) } From 129e20d7139d0158d6232a2b6b324a5589b0e765 Mon Sep 17 00:00:00 2001 From: Srinikitha Kondreddy Date: Wed, 9 Oct 2024 11:34:54 +0200 Subject: [PATCH 10/43] feat: add validation for application name (#5123) * feat: add validation for application name * improve error message --------- Co-authored-by: Oliver Feldmann --- pkg/transportrequest/cts/upload.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/transportrequest/cts/upload.go b/pkg/transportrequest/cts/upload.go index fb22b4260e..a50458df70 100644 --- a/pkg/transportrequest/cts/upload.go +++ b/pkg/transportrequest/cts/upload.go @@ -2,10 +2,12 @@ package cts import ( "fmt" + "regexp" + "strings" + "github.com/SAP/jenkins-library/pkg/command" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" - "strings" ) type fileUtils interface { @@ -60,6 +62,7 @@ const ( abapUserKey = "ABAP_USER" abapPasswordKey = "ABAP_PASSWORD" defaultConfigFileName = "ui5-deploy.yaml" + pattern = "^[a-zA-Z0-9_]+$" ) // WithConnection ... @@ -189,6 +192,10 @@ func getFioriDeployStatement( log.Entry().Debug("No application package found in piper config.") } if len(app.Name) > 0 { + re := regexp.MustCompile(pattern) + if !re.MatchString(app.Name) { + fmt.Errorf("application name '%s' contains spaces or special characters. It is not according to the '%s'", app.Name, pattern) + } log.Entry().Debugf("application name '%s' used from piper config", app.Name) cmd = append(cmd, "--name", app.Name) } else { From b9050b53a586feb11c78ec4ab17feeb350b28679 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Wed, 9 Oct 2024 12:13:43 +0200 Subject: [PATCH 11/43] Handle options properly (#5142) --- cmd/mavenBuild.go | 103 ++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 53 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index df775856ec..3280ce7ce7 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -40,23 +40,12 @@ func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, c } } -func executeMavenGoals(config *mavenBuildOptions, utils maven.Utils, flags []string, goals []string, defines []string) error { - mavenOptions := maven.ExecuteOptions{ - Flags: flags, - Goals: goals, - Defines: defines, - PomPath: config.PomPath, - ProjectSettingsFile: config.ProjectSettingsFile, - GlobalSettingsFile: config.GlobalSettingsFile, - M2Path: config.M2Path, - LogSuccessfulMavenTransfers: config.LogSuccessfulMavenTransfers, - } - - _, err := maven.Execute(&mavenOptions, utils) +func executeMavenGoals(config *mavenBuildOptions, utils maven.Utils, flags []string, goals []string, defines []string, mavenOptions *maven.ExecuteOptions) error { + _, err := maven.Execute(mavenOptions, utils) return err } -func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { +func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils, mavenOptions *maven.ExecuteOptions) error { var flags = []string{"-update-snapshots", "--batch-mode"} if len(config.Profiles) > 0 { flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) @@ -89,7 +78,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { defines = append(defines, "-Dflatten.mode=resolveCiFriendliesOnly", "-DupdatePomFile=true") } - return executeMavenGoals(config, utils, flags, goals, defines) + return executeMavenGoals(config, utils, flags, goals, defines, mavenOptions) } func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { @@ -139,13 +128,24 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat goals = append(goals, "install") } - if err := executeMavenGoals(config, utils, flags, goals, defines); err != nil { + mavenOptions := maven.ExecuteOptions{ + Flags: flags, + Goals: goals, + Defines: defines, + PomPath: config.PomPath, + ProjectSettingsFile: config.ProjectSettingsFile, + GlobalSettingsFile: config.GlobalSettingsFile, + M2Path: config.M2Path, + LogSuccessfulMavenTransfers: config.LogSuccessfulMavenTransfers, + } + + if err := executeMavenGoals(config, utils, flags, goals, defines, &mavenOptions); err != nil { return errors.Wrapf(err, "failed to execute maven build for goal(s) '%v'", goals) } if config.CreateBOM { // Separate run for makeBOM goal - if err := runMakeBOMGoal(config, utils); err != nil { + if err := runMakeBOMGoal(config, utils, &mavenOptions); err != nil { return errors.Wrap(err, "failed to execute makeBOM goal") } } @@ -181,53 +181,50 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat if err != nil { return errors.Wrap(err, "Could not create or update project settings xml") } - mavenOptions := maven.ExecuteOptions{ - ProjectSettingsFile: projectSettingsFilePath, - } - - deployFlags := []string{} - if len(config.DeployFlags) > 0 { - deployFlags = append(deployFlags, config.DeployFlags...) - } - if (len(config.AltDeploymentRepositoryID) > 0) && (len(config.AltDeploymentRepositoryURL) > 0) { - deployFlags = append(deployFlags, "-DaltDeploymentRepository="+config.AltDeploymentRepositoryID+"::default::"+config.AltDeploymentRepositoryURL) - } - - downloadClient := &piperhttp.Client{} - downloadClient.SetOptions(piperhttp.ClientOptions{}) - - runner := &command.Command{ - StepName: "mavenBuild", - } + mavenOptions.ProjectSettingsFile = projectSettingsFilePath + } - fileUtils := &piperutils.Files{} - if len(config.CustomTLSCertificateLinks) > 0 { - if err := loadRemoteRepoCertificates(config.CustomTLSCertificateLinks, downloadClient, &deployFlags, runner, fileUtils, config.JavaCaCertFilePath); err != nil { - log.SetErrorCategory(log.ErrorInfrastructure) - return err - } - } + deployFlags := []string{} + if len(config.DeployFlags) > 0 { + deployFlags = append(deployFlags, config.DeployFlags...) + } + if (len(config.AltDeploymentRepositoryID) > 0) && (len(config.AltDeploymentRepositoryURL) > 0) { + deployFlags = append(deployFlags, "-DaltDeploymentRepository="+config.AltDeploymentRepositoryID+"::default::"+config.AltDeploymentRepositoryURL) + } - mavenOptions.Flags = deployFlags - mavenOptions.Goals = []string{"deploy"} - mavenOptions.Defines = []string{} - _, err = maven.Execute(&mavenOptions, utils) - if err != nil { + downloadClient := &piperhttp.Client{} + downloadClient.SetOptions(piperhttp.ClientOptions{}) + runner := &command.Command{ + StepName: "mavenBuild", + } + fileUtils := &piperutils.Files{} + if len(config.CustomTLSCertificateLinks) > 0 { + if err := loadRemoteRepoCertificates(config.CustomTLSCertificateLinks, downloadClient, &deployFlags, runner, fileUtils, config.JavaCaCertFilePath); err != nil { + log.SetErrorCategory(log.ErrorInfrastructure) return err } + } - if config.CreateBuildArtifactsMetadata { - err2, done := createBuildArtifactsMetadata(config, commonPipelineEnvironment) - if done { - return err2 - } + mavenOptions.Flags = deployFlags + mavenOptions.Goals = []string{"deploy"} + mavenOptions.Defines = []string{} + _, err := maven.Execute(&mavenOptions, utils) + if err != nil { + return err + } + if config.CreateBuildArtifactsMetadata { + err2, done := createBuildArtifactsMetadata(config, commonPipelineEnvironment) + if done { + return err2 } - return nil } + + return nil } else { log.Entry().Infof("publish not detected, ignoring maven deploy") } } + return err } From 3225a7ddc06ff1f0a2645b190ac98a48a180ea1e Mon Sep 17 00:00:00 2001 From: Simon Dold <48808400+doldsimo@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:42:09 +0200 Subject: [PATCH 12/43] Adding log download as zip archive (#5121) * adding LogOutput to clone step * delete comments * adding stepName * change step name * adding multiple log archive outputs files * changing file name * change filename * change time format * adding second file for testing * adding second file * change structure for PersistReportsAndLinks * change to pointer * change pointer * cleanup * changing file name * adding logArchive for pull action * adding logArchive for checkoutBranch * refactor zip archive log * change structure * adding PersistArchiveLogsForPiperStep function * adding persist structure to checkout * adding FileNameStep * adding unit tests * correct name * change whitespace in yaml * fixing unit tests * fixing createTag unit test * fixing unit test * fixing unit test * rename ArchiveOutputLogs to LogOutputManager * refactor pointer structure * adopt tests to pointer structure * fixing / error in repo name * adding log overview also after archive log * change log output structure * adding always execution log * update unit tests --------- Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> --- cmd/abapEnvironmentCheckoutBranch.go | 31 +++-- ...abapEnvironmentCheckoutBranch_generated.go | 11 ++ cmd/abapEnvironmentCheckoutBranch_test.go | 70 ++++++++++- cmd/abapEnvironmentCloneGitRepo.go | 32 +++-- cmd/abapEnvironmentCloneGitRepo_generated.go | 11 ++ cmd/abapEnvironmentCloneGitRepo_test.go | 112 ++++++++++++++++-- cmd/abapEnvironmentCreateTag.go | 9 +- cmd/abapEnvironmentCreateTag_test.go | 31 +++-- cmd/abapEnvironmentPullGitRepo.go | 29 +++-- cmd/abapEnvironmentPullGitRepo_generated.go | 11 ++ cmd/abapEnvironmentPullGitRepo_test.go | 107 +++++++++++++++-- pkg/abaputils/manageGitRepositoryUtils.go | 70 +++++++++-- .../manageGitRepositoryUtils_test.go | 19 ++- pkg/abaputils/sap_com_0510.go | 9 ++ pkg/abaputils/sap_com_0948.go | 26 ++++ pkg/abaputils/sap_com_0948_test.go | 22 ++++ pkg/abaputils/softwareComponentApiManager.go | 2 + .../abapEnvironmentPipelineDefaults.yml | 1 + .../abapEnvironmentCheckoutBranch.yaml | 11 ++ .../metadata/abapEnvironmentCloneGitRepo.yaml | 11 ++ .../metadata/abapEnvironmentPullGitRepo.yaml | 11 ++ 21 files changed, 556 insertions(+), 80 deletions(-) diff --git a/cmd/abapEnvironmentCheckoutBranch.go b/cmd/abapEnvironmentCheckoutBranch.go index 684b03a2da..17804ec21b 100644 --- a/cmd/abapEnvironmentCheckoutBranch.go +++ b/cmd/abapEnvironmentCheckoutBranch.go @@ -9,6 +9,7 @@ import ( "github.com/SAP/jenkins-library/pkg/command" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/pkg/errors" ) @@ -30,14 +31,23 @@ func abapEnvironmentCheckoutBranch(options abapEnvironmentCheckoutBranchOptions, PollIntervall: 5 * time.Second, } + var reports []piperutils.Path + + logOutputManager := abaputils.LogOutputManager{ + LogOutput: options.LogOutput, + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, + } + // error situations should stop execution through log.Entry().Fatal() call which leads to an os.Exit(1) in the end - err := runAbapEnvironmentCheckoutBranch(&options, &autils, &apiManager) + err := runAbapEnvironmentCheckoutBranch(&options, &autils, &apiManager, &logOutputManager) if err != nil { log.Entry().WithError(err).Fatal("step execution failed") } } -func runAbapEnvironmentCheckoutBranch(options *abapEnvironmentCheckoutBranchOptions, com abaputils.Communication, apiManager abaputils.SoftwareComponentApiManagerInterface) (err error) { +func runAbapEnvironmentCheckoutBranch(options *abapEnvironmentCheckoutBranchOptions, com abaputils.Communication, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) (err error) { // Mapping for options subOptions := convertCheckoutConfig(options) @@ -58,19 +68,24 @@ func runAbapEnvironmentCheckoutBranch(options *abapEnvironmentCheckoutBranchOpti if err != nil { return errors.Wrap(err, "Could not read repositories") } - err = checkoutBranches(repositories, connectionDetails, apiManager) + + err = checkoutBranches(repositories, connectionDetails, apiManager, logOutputManager) if err != nil { return fmt.Errorf("Something failed during the checkout: %w", err) } + + // Persist log archive + abaputils.PersistArchiveLogsForPiperStep(logOutputManager) + log.Entry().Infof("-------------------------") log.Entry().Info("All branches were checked out successfully") return nil } -func checkoutBranches(repositories []abaputils.Repository, checkoutConnectionDetails abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface) (err error) { +func checkoutBranches(repositories []abaputils.Repository, checkoutConnectionDetails abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) (err error) { log.Entry().Infof("Start switching %v branches", len(repositories)) for _, repo := range repositories { - err = handleCheckout(repo, checkoutConnectionDetails, apiManager) + err = handleCheckout(repo, checkoutConnectionDetails, apiManager, logOutputManager) if err != nil { break } @@ -96,7 +111,7 @@ func checkCheckoutBranchRepositoryConfiguration(options abapEnvironmentCheckoutB return nil } -func handleCheckout(repo abaputils.Repository, checkoutConnectionDetails abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface) (err error) { +func handleCheckout(repo abaputils.Repository, checkoutConnectionDetails abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) (err error) { if reflect.DeepEqual(abaputils.Repository{}, repo) { return fmt.Errorf("Failed to read repository configuration: %w", errors.New("Error in configuration, most likely you have entered empty or wrong configuration values. Please make sure that you have correctly specified the branches in the repositories to be checked out")) @@ -113,8 +128,10 @@ func handleCheckout(repo abaputils.Repository, checkoutConnectionDetails abaputi return fmt.Errorf("Failed to trigger Checkout: %w", errors.New("Checkout of "+repo.Branch+" for software component "+repo.Name+" failed on the ABAP System")) } + // set correct filename for archive file + logOutputManager.FileNameStep = "checkoutBranch" // Polling the status of the repository import on the ABAP Environment system - status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall()) + status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall(), logOutputManager) if errorPollEntity != nil { return fmt.Errorf("Failed to poll Checkout: %w", errors.New("Status of checkout action on repository"+repo.Name+" failed on the ABAP System")) } diff --git a/cmd/abapEnvironmentCheckoutBranch_generated.go b/cmd/abapEnvironmentCheckoutBranch_generated.go index 988ebfc9d4..6b3384c279 100644 --- a/cmd/abapEnvironmentCheckoutBranch_generated.go +++ b/cmd/abapEnvironmentCheckoutBranch_generated.go @@ -22,6 +22,7 @@ type abapEnvironmentCheckoutBranchOptions struct { BranchName string `json:"branchName,omitempty"` Host string `json:"host,omitempty"` Repositories string `json:"repositories,omitempty"` + LogOutput string `json:"logOutput,omitempty" validate:"possible-values=ZIP STANDARD"` CfAPIEndpoint string `json:"cfApiEndpoint,omitempty"` CfOrg string `json:"cfOrg,omitempty"` CfSpace string `json:"cfSpace,omitempty"` @@ -142,6 +143,7 @@ func addAbapEnvironmentCheckoutBranchFlags(cmd *cobra.Command, stepConfig *abapE cmd.Flags().StringVar(&stepConfig.BranchName, "branchName", os.Getenv("PIPER_branchName"), "Specifies a Branch of a Repository (Software Component) on the SAP BTP ABAP Environment system") cmd.Flags().StringVar(&stepConfig.Host, "host", os.Getenv("PIPER_host"), "Specifies the host address of the SAP BTP ABAP Environment system") cmd.Flags().StringVar(&stepConfig.Repositories, "repositories", os.Getenv("PIPER_repositories"), "Specifies a YAML file containing the repositories configuration") + cmd.Flags().StringVar(&stepConfig.LogOutput, "logOutput", `STANDARD`, "Specifies how the clone logs from the Manage Software Components App are displayed or saved") cmd.Flags().StringVar(&stepConfig.CfAPIEndpoint, "cfApiEndpoint", os.Getenv("PIPER_cfApiEndpoint"), "Cloud Foundry API Enpoint") cmd.Flags().StringVar(&stepConfig.CfOrg, "cfOrg", os.Getenv("PIPER_cfOrg"), "Cloud Foundry target organization") cmd.Flags().StringVar(&stepConfig.CfSpace, "cfSpace", os.Getenv("PIPER_cfSpace"), "Cloud Foundry target space") @@ -233,6 +235,15 @@ func abapEnvironmentCheckoutBranchMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_repositories"), }, + { + Name: "logOutput", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `STANDARD`, + }, { Name: "cfApiEndpoint", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/abapEnvironmentCheckoutBranch_test.go b/cmd/abapEnvironmentCheckoutBranch_test.go index 87bf02fe43..8ca763b722 100644 --- a/cmd/abapEnvironmentCheckoutBranch_test.go +++ b/cmd/abapEnvironmentCheckoutBranch_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/abaputils" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) @@ -51,6 +52,7 @@ func TestCheckoutBranchStep(t *testing.T) { Password: "testPassword", RepositoryName: "testRepo1", BranchName: "testBranch", + LogOutput: "STANDARD", } logResultSuccess := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -67,8 +69,16 @@ func TestCheckoutBranchStep(t *testing.T) { StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager) + err := runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager, &logOutputManager) assert.NoError(t, err, "Did not expect error") }) t.Run("Run Step Failure - empty config", func(t *testing.T) { @@ -96,8 +106,16 @@ func TestCheckoutBranchStep(t *testing.T) { StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: "STANDARD", + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager) + err := runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager, &logOutputManager) assert.EqualError(t, err, expectedErrorMessage) }) t.Run("Run Step Failure - wrong status", func(t *testing.T) { @@ -120,6 +138,7 @@ func TestCheckoutBranchStep(t *testing.T) { Password: "testPassword", RepositoryName: "testRepo1", BranchName: "testBranch", + LogOutput: "STANDARD", } logResultError := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -130,13 +149,22 @@ func TestCheckoutBranchStep(t *testing.T) { `{"d" : { "status" : "E" } }`, `{"d" : { "status" : "E" } }`, `{"d" : { "status" : "E" } }`, + `{"d" : { "status" : "E" } }`, }, Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager) + err := runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager, &logOutputManager) assert.EqualError(t, err, expectedErrorMessage) }) t.Run("Success case: checkout Branches from file config", func(t *testing.T) { @@ -183,9 +211,19 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "repositoriesTest.yml", + LogOutput: "STANDARD", + } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager) + err = runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager, &logOutputManager) assert.NoError(t, err) }) t.Run("Failure case: checkout Branches from empty file config", func(t *testing.T) { @@ -227,9 +265,19 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "repositoriesTest.yml", + LogOutput: "STANDARD", + } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager) + err = runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager, &logOutputManager) assert.EqualError(t, err, expectedErrorMessage) }) t.Run("Failure case: checkout Branches from wrong file config", func(t *testing.T) { @@ -276,9 +324,19 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "repositoriesTest.yml", + LogOutput: "STANDARD", + } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "checkoutBranch", + FileNameStep: "checkoutBranch", + StepReports: reports, } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager) + err = runAbapEnvironmentCheckoutBranch(&config, &autils, apiManager, &logOutputManager) assert.EqualError(t, err, expectedErrorMessage) }) } diff --git a/cmd/abapEnvironmentCloneGitRepo.go b/cmd/abapEnvironmentCloneGitRepo.go index 71cfe60abd..d8c3763c2c 100644 --- a/cmd/abapEnvironmentCloneGitRepo.go +++ b/cmd/abapEnvironmentCloneGitRepo.go @@ -7,6 +7,7 @@ import ( "github.com/SAP/jenkins-library/pkg/command" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/pkg/errors" ) @@ -26,14 +27,23 @@ func abapEnvironmentCloneGitRepo(config abapEnvironmentCloneGitRepoOptions, _ *t Client: &piperhttp.Client{}, PollIntervall: 5 * time.Second, } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + // error situations should stop execution through log.Entry().Fatal() call which leads to an os.Exit(1) in the end - err := runAbapEnvironmentCloneGitRepo(&config, &autils, &apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, &apiManager, &logOutputManager) if err != nil { log.Entry().WithError(err).Fatal("step execution failed") } } -func runAbapEnvironmentCloneGitRepo(config *abapEnvironmentCloneGitRepoOptions, com abaputils.Communication, apiManager abaputils.SoftwareComponentApiManagerInterface) error { +func runAbapEnvironmentCloneGitRepo(config *abapEnvironmentCloneGitRepoOptions, com abaputils.Communication, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) error { // Mapping for options subOptions := convertCloneConfig(config) @@ -55,19 +65,23 @@ func runAbapEnvironmentCloneGitRepo(config *abapEnvironmentCloneGitRepoOptions, connectionDetails.CertificateNames = config.CertificateNames log.Entry().Infof("Start cloning %v repositories", len(repositories)) + for _, repo := range repositories { - cloneError := cloneSingleRepo(apiManager, connectionDetails, repo, config, com) + cloneError := cloneSingleRepo(apiManager, connectionDetails, repo, config, com, logOutputManager) if cloneError != nil { return cloneError } } + // Persist log archive + abaputils.PersistArchiveLogsForPiperStep(logOutputManager) + abaputils.AddDefaultDashedLine(1) log.Entry().Info("All repositories were cloned successfully") return nil } -func cloneSingleRepo(apiManager abaputils.SoftwareComponentApiManagerInterface, connectionDetails abaputils.ConnectionDetailsHTTP, repo abaputils.Repository, config *abapEnvironmentCloneGitRepoOptions, com abaputils.Communication) error { +func cloneSingleRepo(apiManager abaputils.SoftwareComponentApiManagerInterface, connectionDetails abaputils.ConnectionDetailsHTTP, repo abaputils.Repository, config *abapEnvironmentCloneGitRepoOptions, com abaputils.Communication, logOutputManager *abaputils.LogOutputManager) error { // New API instance for each request // Triggering the Clone of the repository into the ABAP Environment system @@ -98,8 +112,9 @@ func cloneSingleRepo(apiManager abaputils.SoftwareComponentApiManagerInterface, if errClone != nil { return errors.Wrapf(errClone, errorString) } - - status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall()) + // set correct filename for archive file + logOutputManager.FileNameStep = "clone" + status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall(), logOutputManager) if errorPollEntity != nil { return errors.Wrapf(errorPollEntity, errorString) } @@ -114,13 +129,13 @@ func cloneSingleRepo(apiManager abaputils.SoftwareComponentApiManagerInterface, abaputils.AddDefaultDashedLine(2) var returnedError error if repo.Branch != "" && !(activeBranch == repo.Branch) { - returnedError = runAbapEnvironmentCheckoutBranch(getCheckoutOptions(config, repo), com, apiManager) + returnedError = runAbapEnvironmentCheckoutBranch(getCheckoutOptions(config, repo), com, apiManager, logOutputManager) abaputils.AddDefaultDashedLine(2) if returnedError != nil { return returnedError } } - returnedError = runAbapEnvironmentPullGitRepo(getPullOptions(config, repo), com, apiManager) + returnedError = runAbapEnvironmentPullGitRepo(getPullOptions(config, repo), com, apiManager, logOutputManager) return returnedError } return nil @@ -154,6 +169,7 @@ func getPullOptions(config *abapEnvironmentCloneGitRepoOptions, repo abaputils.R CfServiceInstance: config.CfServiceInstance, CfServiceKeyName: config.CfServiceKeyName, CfSpace: config.CfSpace, + LogOutput: config.LogOutput, } return &pullOptions } diff --git a/cmd/abapEnvironmentCloneGitRepo_generated.go b/cmd/abapEnvironmentCloneGitRepo_generated.go index 4a474950ed..a8c26a5223 100644 --- a/cmd/abapEnvironmentCloneGitRepo_generated.go +++ b/cmd/abapEnvironmentCloneGitRepo_generated.go @@ -25,6 +25,7 @@ type abapEnvironmentCloneGitRepoOptions struct { RepositoryName string `json:"repositoryName,omitempty"` BranchName string `json:"branchName,omitempty"` Host string `json:"host,omitempty"` + LogOutput string `json:"logOutput,omitempty" validate:"possible-values=ZIP STANDARD"` CfAPIEndpoint string `json:"cfApiEndpoint,omitempty"` CfOrg string `json:"cfOrg,omitempty"` CfSpace string `json:"cfSpace,omitempty"` @@ -150,6 +151,7 @@ func addAbapEnvironmentCloneGitRepoFlags(cmd *cobra.Command, stepConfig *abapEnv cmd.Flags().StringVar(&stepConfig.RepositoryName, "repositoryName", os.Getenv("PIPER_repositoryName"), "Specifies a repository (Software Components) on the SAP BTP ABAP Environment system") cmd.Flags().StringVar(&stepConfig.BranchName, "branchName", os.Getenv("PIPER_branchName"), "Specifies a branch of a repository (Software Components) on the SAP BTP ABAP Environment system") cmd.Flags().StringVar(&stepConfig.Host, "host", os.Getenv("PIPER_host"), "Specifies the host address of the SAP BTP ABAP Environment system") + cmd.Flags().StringVar(&stepConfig.LogOutput, "logOutput", `STANDARD`, "Specifies how the clone logs from the Manage Software Components App are displayed or saved") cmd.Flags().StringVar(&stepConfig.CfAPIEndpoint, "cfApiEndpoint", os.Getenv("PIPER_cfApiEndpoint"), "Cloud Foundry API Enpoint") cmd.Flags().StringVar(&stepConfig.CfOrg, "cfOrg", os.Getenv("PIPER_cfOrg"), "Cloud Foundry target organization") cmd.Flags().StringVar(&stepConfig.CfSpace, "cfSpace", os.Getenv("PIPER_cfSpace"), "Cloud Foundry target space") @@ -283,6 +285,15 @@ func abapEnvironmentCloneGitRepoMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_host"), }, + { + Name: "logOutput", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `STANDARD`, + }, { Name: "cfApiEndpoint", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/abapEnvironmentCloneGitRepo_test.go b/cmd/abapEnvironmentCloneGitRepo_test.go index ae6f47f75f..e40cbb0122 100644 --- a/cmd/abapEnvironmentCloneGitRepo_test.go +++ b/cmd/abapEnvironmentCloneGitRepo_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/abaputils" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) @@ -75,6 +76,7 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "filename.yaml", + LogOutput: "STANDARD", } logResultSuccess := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -100,8 +102,16 @@ repositories: Token: "myToken", } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) assert.NoError(t, err, "Did not expect error") assert.Equal(t, 0, len(client.BodyList), "Not all requests were done") }) @@ -123,6 +133,7 @@ repositories: Username: "testUser", Password: "testPassword", RepositoryName: "testRepo1", + LogOutput: "STANDARD", } logResultSuccess := `{"d": { "sc_name": "testRepo1", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -139,9 +150,16 @@ repositories: Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) assert.NoError(t, err, "Did not expect error") assert.Equal(t, 0, len(client.BodyList), "Not all requests were done") }) @@ -164,21 +182,30 @@ repositories: Password: "testPassword", RepositoryName: "testRepo1", BranchName: "testBranch1", + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ BodyList: []string{ + `{"d" : {} }`, `{"d" : {} }`, `{"d" : { "status" : "R" } }`, `{"d" : { "sc_name" : "testRepo1", "avail_on_instance" : true, "active_branch": "testBranch1" } }`, - `{"d" : [] }`, + `{"d" : {} }`, }, Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Clone of repository / software component 'testRepo1', branch 'testBranch1' failed on the ABAP system: Request to ABAP System not successful", err.Error(), "Expected different error message") } @@ -224,6 +251,7 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "filename.yaml", + LogOutput: "STANDARD", } logResultError := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -239,8 +267,16 @@ repositories: Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Clone of repository / software component '/DMO/REPO_A', branch 'branchA', commit 'ABCD1234' failed on the ABAP system: Request to ABAP System not successful", err.Error(), "Expected different error message") } @@ -264,6 +300,7 @@ repositories: Password: "testPassword", RepositoryName: "testRepo1", BranchName: "testBranch1", + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ @@ -275,8 +312,16 @@ repositories: Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Clone of repository / software component 'testRepo1', branch 'testBranch1' failed on the ABAP system: Request to ABAP System not successful", err.Error(), "Expected different error message") } @@ -300,6 +345,7 @@ repositories: Password: "testPassword", RepositoryName: "testRepo1", BranchName: "testBranch1", + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ @@ -310,8 +356,16 @@ repositories: Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Clone of repository / software component 'testRepo1', branch 'testBranch1' failed on the ABAP system: Request to ABAP System not successful", err.Error(), "Expected different error message") } @@ -334,6 +388,7 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "filename.yaml", + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ @@ -344,8 +399,16 @@ repositories: Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Could not read repositories: Could not find filename.yaml", err.Error(), "Expected different error message") } @@ -371,6 +434,7 @@ repositories: Repositories: "filename.yaml", RepositoryName: "/DMO/REPO", BranchName: "Branch", + LogOutput: "STANDARD", } logResultError := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Error", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -385,8 +449,16 @@ repositories: Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentCloneGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "The provided configuration is not allowed: It is not allowed to configure the parameters `repositories`and `repositoryName` at the same time", err.Error(), "Expected different error message") } @@ -412,6 +484,7 @@ func TestALreadyCloned(t *testing.T) { CfServiceKeyName: "testServiceKey", Username: "testUser", Password: "testPassword", + LogOutput: "STANDARD", } logResultSuccess := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -441,8 +514,16 @@ func TestALreadyCloned(t *testing.T) { CommitID: "abcd1234", } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := cloneSingleRepo(apiManager, autils.ReturnedConnectionDetailsHTTP, repo, &config, &autils) + err := cloneSingleRepo(apiManager, autils.ReturnedConnectionDetailsHTTP, repo, &config, &autils, &logOutputManager) assert.NoError(t, err, "Did not expect error") }) @@ -464,6 +545,7 @@ func TestALreadyCloned(t *testing.T) { CfServiceKeyName: "testServiceKey", Username: "testUser", Password: "testPassword", + LogOutput: "STANDARD", } logResultSuccess := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -487,8 +569,16 @@ func TestALreadyCloned(t *testing.T) { CommitID: "abcd1234", } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "clone", + FileNameStep: "clone", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := cloneSingleRepo(apiManager, autils.ReturnedConnectionDetailsHTTP, repo, &config, &autils) + err := cloneSingleRepo(apiManager, autils.ReturnedConnectionDetailsHTTP, repo, &config, &autils, &logOutputManager) assert.NoError(t, err, "Did not expect error") }) diff --git a/cmd/abapEnvironmentCreateTag.go b/cmd/abapEnvironmentCreateTag.go index 5adebcda7d..3bcda4c0c3 100644 --- a/cmd/abapEnvironmentCreateTag.go +++ b/cmd/abapEnvironmentCreateTag.go @@ -97,7 +97,14 @@ func createSingleTag(item abaputils.CreateTagBacklog, index int, con abaputils.C return errors.Wrapf(err, "Creation of Tag failed on the ABAP system") } - status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall()) + logOutputManager := abaputils.LogOutputManager{ + LogOutput: "STANDARD", + PiperStep: "createTag", + FileNameStep: "createTag", + StepReports: nil, + } + + status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall(), &logOutputManager) if errorPollEntity == nil && status == "S" { log.Entry().Info("Created tag " + item.Tags[index].TagName + " for repository " + item.RepositoryName + " with commitID " + item.CommitID) diff --git a/cmd/abapEnvironmentCreateTag_test.go b/cmd/abapEnvironmentCreateTag_test.go index cdd1310062..b02acff9a6 100644 --- a/cmd/abapEnvironmentCreateTag_test.go +++ b/cmd/abapEnvironmentCreateTag_test.go @@ -104,11 +104,11 @@ repositories: apiManager := &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} err = runAbapEnvironmentCreateTag(config, autils, apiManager) - assert.NoError(t, err, "Did not expect error") - assert.Equal(t, 25, len(hook.Entries), "Expected a different number of entries") - assert.Equal(t, `Created tag v4.5.6 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[12].Message, "Expected a different message") - assert.Equal(t, `Created tag -DMO-PRODUCT-1.2.3 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[18].Message, "Expected a different message") - assert.Equal(t, `Created tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[24].Message, "Expected a different message") + assert.Error(t, err, "Did expect error") + assert.Equal(t, 18, len(hook.Entries), "Expected a different number of entries") + assert.Equal(t, `Created tag v4.5.6 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[6].Message, "Expected a different message") + assert.Equal(t, `NOT created: Tag -DMO-PRODUCT-1.2.3 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[11].Message, "Expected a different message") + assert.Equal(t, `NOT created: Tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[16].Message, "Expected a different message") hook.Reset() }) @@ -179,12 +179,11 @@ repositories: apiManager := &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} err = runAbapEnvironmentCreateTag(config, autils, apiManager) - assert.Error(t, err, "Did expect error") - assert.Equal(t, 40, len(hook.Entries), "Expected a different number of entries") - assert.Equal(t, `NOT created: Tag v4.5.6 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[12].Message, "Expected a different message") - assert.Equal(t, `NOT created: Tag -DMO-PRODUCT-1.2.3 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[25].Message, "Expected a different message") - assert.Equal(t, `NOT created: Tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[38].Message, "Expected a different message") - assert.Equal(t, `At least one tag has not been created`, hook.AllEntries()[39].Message, "Expected a different message") + assert.NoError(t, err, "Did expect error") + assert.Equal(t, 21, len(hook.Entries), "Expected a different number of entries") + assert.Equal(t, `Created tag v4.5.6 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[6].Message, "Expected a different message") + assert.Equal(t, `Created tag -DMO-PRODUCT-1.2.3 for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[13].Message, "Expected a different message") + assert.Equal(t, `Created tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[20].Message, "Expected a different message") hook.Reset() }) @@ -232,8 +231,8 @@ func TestRunAbapEnvironmentCreateTagConfigurations(t *testing.T) { err := runAbapEnvironmentCreateTag(config, autils, apiManager) assert.NoError(t, err, "Did not expect error") - assert.Equal(t, 13, len(hook.Entries), "Expected a different number of entries") - assert.Equal(t, `Created tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[12].Message, "Expected a different message") + assert.Equal(t, 7, len(hook.Entries), "Expected a different number of entries") + assert.Equal(t, `Created tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[6].Message, "Expected a different message") hook.Reset() }) @@ -362,9 +361,9 @@ repositories: apiManager := &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} err = runAbapEnvironmentCreateTag(config, autils, apiManager) - assert.NoError(t, err, "Did not expect error") - assert.Equal(t, 6, len(hook.Entries), "Expected a different number of entries") - assert.Equal(t, `Created tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[5].Message, "Expected a different message") + assert.Error(t, err, "Did expect error") + assert.Equal(t, 8, len(hook.Entries), "Expected a different number of entries") + assert.Equal(t, `NOT created: Tag tag for repository /DMO/SWC with commitID 1234abcd`, hook.AllEntries()[6].Message, "Expected a different message") hook.Reset() }) diff --git a/cmd/abapEnvironmentPullGitRepo.go b/cmd/abapEnvironmentPullGitRepo.go index 93e51a3048..7b851ec5cf 100644 --- a/cmd/abapEnvironmentPullGitRepo.go +++ b/cmd/abapEnvironmentPullGitRepo.go @@ -8,6 +8,7 @@ import ( "github.com/SAP/jenkins-library/pkg/command" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/telemetry" "github.com/pkg/errors" ) @@ -28,15 +29,22 @@ func abapEnvironmentPullGitRepo(options abapEnvironmentPullGitRepoOptions, _ *te Client: &piperhttp.Client{}, PollIntervall: 5 * time.Second, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: options.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } // error situations should stop execution through log.Entry().Fatal() call which leads to an os.Exit(1) in the end - err := runAbapEnvironmentPullGitRepo(&options, &autils, &apiManager) + err := runAbapEnvironmentPullGitRepo(&options, &autils, &apiManager, &logOutputManager) if err != nil { log.Entry().WithError(err).Fatal("step execution failed") } } -func runAbapEnvironmentPullGitRepo(options *abapEnvironmentPullGitRepoOptions, com abaputils.Communication, apiManager abaputils.SoftwareComponentApiManagerInterface) (err error) { +func runAbapEnvironmentPullGitRepo(options *abapEnvironmentPullGitRepoOptions, com abaputils.Communication, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) (err error) { subOptions := convertPullConfig(options) @@ -52,21 +60,26 @@ func runAbapEnvironmentPullGitRepo(options *abapEnvironmentPullGitRepoOptions, c if err != nil { return err } + repositories, err = abaputils.GetRepositories(&abaputils.RepositoriesConfig{RepositoryNames: options.RepositoryNames, Repositories: options.Repositories, RepositoryName: options.RepositoryName, CommitID: options.CommitID}, false) handleIgnoreCommit(repositories, options.IgnoreCommit) if err != nil { return err } - err = pullRepositories(repositories, connectionDetails, apiManager) + err = pullRepositories(repositories, connectionDetails, apiManager, logOutputManager) + + // Persist log archive + abaputils.PersistArchiveLogsForPiperStep(logOutputManager) + return err } -func pullRepositories(repositories []abaputils.Repository, pullConnectionDetails abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface) (err error) { +func pullRepositories(repositories []abaputils.Repository, pullConnectionDetails abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) (err error) { log.Entry().Infof("Start pulling %v repositories", len(repositories)) for _, repo := range repositories { - err = handlePull(repo, pullConnectionDetails, apiManager) + err = handlePull(repo, pullConnectionDetails, apiManager, logOutputManager) if err != nil { break } @@ -77,7 +90,7 @@ func pullRepositories(repositories []abaputils.Repository, pullConnectionDetails return err } -func handlePull(repo abaputils.Repository, con abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface) (err error) { +func handlePull(repo abaputils.Repository, con abaputils.ConnectionDetailsHTTP, apiManager abaputils.SoftwareComponentApiManagerInterface, logOutputManager *abaputils.LogOutputManager) (err error) { logString := repo.GetPullLogString() errorString := "Pull of the " + logString + " failed on the ABAP system" @@ -96,8 +109,10 @@ func handlePull(repo abaputils.Repository, con abaputils.ConnectionDetailsHTTP, return errors.Wrapf(err, errorString) } + // set correct filename for archive file + logOutputManager.FileNameStep = "pull" // Polling the status of the repository import on the ABAP Environment system - status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall()) + status, errorPollEntity := abaputils.PollEntity(api, apiManager.GetPollIntervall(), logOutputManager) if errorPollEntity != nil { return errors.Wrapf(errorPollEntity, errorString) } diff --git a/cmd/abapEnvironmentPullGitRepo_generated.go b/cmd/abapEnvironmentPullGitRepo_generated.go index 3575ef4fb2..8cc70c1961 100644 --- a/cmd/abapEnvironmentPullGitRepo_generated.go +++ b/cmd/abapEnvironmentPullGitRepo_generated.go @@ -23,6 +23,7 @@ type abapEnvironmentPullGitRepoOptions struct { RepositoryName string `json:"repositoryName,omitempty"` CommitID string `json:"commitID,omitempty"` Host string `json:"host,omitempty"` + LogOutput string `json:"logOutput,omitempty" validate:"possible-values=ZIP STANDARD"` CfAPIEndpoint string `json:"cfApiEndpoint,omitempty"` CfOrg string `json:"cfOrg,omitempty"` CfSpace string `json:"cfSpace,omitempty"` @@ -145,6 +146,7 @@ func addAbapEnvironmentPullGitRepoFlags(cmd *cobra.Command, stepConfig *abapEnvi cmd.Flags().StringVar(&stepConfig.RepositoryName, "repositoryName", os.Getenv("PIPER_repositoryName"), "Specifies a repository (Software Component) on the SAP BTP ABAP Environment system") cmd.Flags().StringVar(&stepConfig.CommitID, "commitID", os.Getenv("PIPER_commitID"), "Specifies a commitID of the repository, configured via \"repositoryName\" on the SAP BTP ABAP Environment system") cmd.Flags().StringVar(&stepConfig.Host, "host", os.Getenv("PIPER_host"), "Specifies the host address of the SAP BTP ABAP Environment system") + cmd.Flags().StringVar(&stepConfig.LogOutput, "logOutput", `STANDARD`, "Specifies how the clone logs from the Manage Software Components App are displayed or saved") cmd.Flags().StringVar(&stepConfig.CfAPIEndpoint, "cfApiEndpoint", os.Getenv("PIPER_cfApiEndpoint"), "Cloud Foundry API Enpoint") cmd.Flags().StringVar(&stepConfig.CfOrg, "cfOrg", os.Getenv("PIPER_cfOrg"), "Cloud Foundry target organization") cmd.Flags().StringVar(&stepConfig.CfSpace, "cfSpace", os.Getenv("PIPER_cfSpace"), "Cloud Foundry target space") @@ -246,6 +248,15 @@ func abapEnvironmentPullGitRepoMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_host"), }, + { + Name: "logOutput", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: `STANDARD`, + }, { Name: "cfApiEndpoint", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/abapEnvironmentPullGitRepo_test.go b/cmd/abapEnvironmentPullGitRepo_test.go index cd805f37d5..f589381795 100644 --- a/cmd/abapEnvironmentPullGitRepo_test.go +++ b/cmd/abapEnvironmentPullGitRepo_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/abaputils" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) @@ -53,6 +54,7 @@ func TestPullStep(t *testing.T) { Username: "testUser", Password: "testPassword", RepositoryNames: []string{"testRepo1"}, + LogOutput: "STANDARD", } logResultSuccess := `{"d": { "sc_name": "/DMO/SWC", "status": "S", "to_Log_Overview": { "results": [ { "log_index": 1, "log_name": "Main Import", "type_of_found_issues": "Success", "timestamp": "/Date(1644332299000+0000)/", "to_Log_Protocol": { "results": [ { "log_index": 1, "index_no": "1", "log_name": "", "type": "Info", "descr": "Main import", "timestamp": null, "criticality": 0 } ] } } ] } } }` @@ -70,7 +72,16 @@ func TestPullStep(t *testing.T) { } apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + + err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) assert.NoError(t, err, "Did not expect error") assert.Equal(t, 0, len(client.BodyList), "Not all requests were done") }) @@ -96,8 +107,16 @@ func TestPullStep(t *testing.T) { config := abapEnvironmentPullGitRepoOptions{} + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: "STANDARD", + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) assert.Equal(t, expectedErrorMessage, err.Error(), "Different error message expected") }) @@ -146,9 +165,19 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "repositoriesTest.yml", + LogOutput: "STANDARD", + } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) assert.NoError(t, err) }) @@ -189,6 +218,7 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "filename.yaml", + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ BodyList: []string{ @@ -198,13 +228,22 @@ repositories: `{"d" : { "status" : "E" } }`, `{"d" : { "status" : "R" } }`, `{"d" : { "status" : "R" } }`, + `{"d" : { "status" : "R" } }`, }, Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Pull of the repository / software component '/DMO/REPO_A', commit 'ABCD1234' failed on the ABAP system", err.Error(), "Expected different error message") } @@ -248,6 +287,7 @@ repositories: Password: "testPassword", Repositories: "filename.yaml", IgnoreCommit: true, + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ BodyList: []string{ @@ -257,13 +297,22 @@ repositories: `{"d" : { "status" : "E" } }`, `{"d" : { "status" : "R" } }`, `{"d" : { "status" : "R" } }`, + `{"d" : { "status" : "R" } }`, }, Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Pull of the repository / software component '/DMO/REPO_A', tag 'v-1.0.1-build-0001' failed on the ABAP system", err.Error(), "Expected different error message") } @@ -288,6 +337,7 @@ repositories: RepositoryName: "/DMO/SWC", CommitID: "123456", IgnoreCommit: false, + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ BodyList: []string{ @@ -297,13 +347,22 @@ repositories: `{"d" : { "status" : "E" } }`, `{"d" : { "status" : "R" } }`, `{"d" : { "status" : "R" } }`, + `{"d" : { "status" : "R" } }`, }, Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Pull of the repository / software component '/DMO/SWC', commit '123456' failed on the ABAP system", err.Error(), "Expected different error message") } @@ -327,6 +386,7 @@ repositories: Password: "testPassword", RepositoryName: "/DMO/SWC", IgnoreCommit: false, + LogOutput: "STANDARD", } client := &abaputils.ClientMock{ BodyList: []string{ @@ -336,13 +396,22 @@ repositories: `{"d" : { "status" : "E" } }`, `{"d" : { "status" : "R" } }`, `{"d" : { "status" : "R" } }`, + `{"d" : { "status" : "R" } }`, }, Token: "myToken", StatusCode: 200, } + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err := runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) if assert.Error(t, err, "Expected error") { assert.Equal(t, "Pull of the repository / software component '/DMO/SWC' failed on the ABAP system", err.Error(), "Expected different error message") } @@ -387,9 +456,19 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "repositoriesTest.yml", + LogOutput: "STANDARD", } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) assert.EqualError(t, err, expectedErrorMessage) }) @@ -437,9 +516,19 @@ repositories: Username: "testUser", Password: "testPassword", Repositories: "repositoriesTest.yml", + LogOutput: "STANDARD", } + + var reports []piperutils.Path + logOutputManager := abaputils.LogOutputManager{ + LogOutput: config.LogOutput, + PiperStep: "pull", + FileNameStep: "pull", + StepReports: reports, + } + apiManager = &abaputils.SoftwareComponentApiManager{Client: client, PollIntervall: 1 * time.Nanosecond, Force0510: true} - err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager) + err = runAbapEnvironmentPullGitRepo(&config, &autils, apiManager, &logOutputManager) assert.EqualError(t, err, expectedErrorMessage) }) } diff --git a/pkg/abaputils/manageGitRepositoryUtils.go b/pkg/abaputils/manageGitRepositoryUtils.go index 73da35c043..f9a0b11514 100644 --- a/pkg/abaputils/manageGitRepositoryUtils.go +++ b/pkg/abaputils/manageGitRepositoryUtils.go @@ -2,12 +2,14 @@ package abaputils import ( "fmt" + "os" "reflect" "sort" "strings" "time" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/pkg/errors" ) @@ -15,13 +17,37 @@ const numberOfEntriesPerPage = 100000 const logOutputStatusLength = 10 const logOutputTimestampLength = 29 +// Specifies which output option is used for logs +type LogOutputManager struct { + LogOutput string + PiperStep string + FileNameStep string + StepReports []piperutils.Path +} + +func PersistArchiveLogsForPiperStep(logOutputManager *LogOutputManager) { + fileUtils := piperutils.Files{} + switch logOutputManager.PiperStep { + case "clone": + piperutils.PersistReportsAndLinks("abapEnvironmentCloneGitRepo", "", fileUtils, logOutputManager.StepReports, nil) + case "pull": + piperutils.PersistReportsAndLinks("abapEnvironmentPullGitRepo", "", fileUtils, logOutputManager.StepReports, nil) + case "checkoutBranch": + piperutils.PersistReportsAndLinks("abapEnvironmentCheckoutBranch", "", fileUtils, logOutputManager.StepReports, nil) + default: + log.Entry().Info("Cannot save log archive because no piper step was defined.") + } +} + // PollEntity periodically polls the action entity to get the status. Check if the import is still running -func PollEntity(api SoftwareComponentApiInterface, pollIntervall time.Duration) (string, error) { +func PollEntity(api SoftwareComponentApiInterface, pollIntervall time.Duration, logOutputManager *LogOutputManager) (string, error) { log.Entry().Info("Start polling the status...") var statusCode string = "R" var err error + api.initialRequest() + for { // pullEntity, responseStatus, err := api.GetStatus(failureMessageClonePull+repositoryName, connectionDetails, client) statusCode, err = api.GetAction() @@ -31,7 +57,7 @@ func PollEntity(api SoftwareComponentApiInterface, pollIntervall time.Duration) if statusCode != "R" && statusCode != "Q" { - PrintLogs(api) + PrintLogs(api, logOutputManager) break } time.Sleep(pollIntervall) @@ -39,7 +65,7 @@ func PollEntity(api SoftwareComponentApiInterface, pollIntervall time.Duration) return statusCode, nil } -func PrintLogs(api SoftwareComponentApiInterface) { +func PrintLogs(api SoftwareComponentApiInterface, logOutputManager *LogOutputManager) { // Get Execution Logs executionLogs, err := api.GetExecutionLog() @@ -47,11 +73,7 @@ func PrintLogs(api SoftwareComponentApiInterface) { printExecutionLogs(executionLogs) } - results, err := api.GetLogOverview() - if err != nil || len(results) == 0 { - // return if no logs are available - return - } + results, _ := api.GetLogOverview() // Sort logs sort.SliceStable(results, func(i, j int) bool { @@ -60,13 +82,31 @@ func PrintLogs(api SoftwareComponentApiInterface) { printOverview(results, api) - // Print Details - for _, logEntryForDetails := range results { - printLog(logEntryForDetails, api) + if logOutputManager.LogOutput == "ZIP" { + // get zip file as byte array + zipfile, err := api.GetLogArchive() + // Saving logs in file and adding to piperutils to archive file + if err == nil { + fileName := "LogArchive-" + logOutputManager.FileNameStep + "-" + strings.Replace(api.getRepositoryName(), "/", "_", -1) + "-" + api.getUUID() + "_" + time.Now().Format("2006-01-02T15:04:05") + ".zip" + + err = os.WriteFile(fileName, zipfile, 0o644) + + if err == nil { + log.Entry().Infof("Writing %s file was successful", fileName) + logOutputManager.StepReports = append(logOutputManager.StepReports, piperutils.Path{Target: fileName, Name: "Log_Archive_" + api.getUUID(), Mandatory: true}) + } + } + + } else { + // Print Details + if len(results) != 0 { + for _, logEntryForDetails := range results { + printLog(logEntryForDetails, api) + } + } + AddDefaultDashedLine(1) } - AddDefaultDashedLine(1) - return } func printExecutionLogs(executionLogs ExecutionLog) { @@ -82,6 +122,10 @@ func printExecutionLogs(executionLogs ExecutionLog) { func printOverview(results []LogResultsV2, api SoftwareComponentApiInterface) { + if len(results) == 0 { + return + } + logOutputPhaseLength, logOutputLineLength := calculateLenghts(results) log.Entry().Infof("\n") diff --git a/pkg/abaputils/manageGitRepositoryUtils_test.go b/pkg/abaputils/manageGitRepositoryUtils_test.go index 557c742cea..e1173ec6ab 100644 --- a/pkg/abaputils/manageGitRepositoryUtils_test.go +++ b/pkg/abaputils/manageGitRepositoryUtils_test.go @@ -10,6 +10,7 @@ import ( "os" "testing" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) @@ -63,7 +64,14 @@ func TestPollEntity(t *testing.T) { repo := Repository{Name: "testRepo1"} api, _ := swcManager.GetAPI(con, repo) - status, _ := PollEntity(api, 0) + var reports []piperutils.Path + logOutputManager := LogOutputManager{ + LogOutput: "STANDARD", + PiperStep: "pull", + StepReports: reports, + } + + status, _ := PollEntity(api, 0, &logOutputManager) assert.Equal(t, "S", status) assert.Equal(t, 0, len(client.BodyList), "Not all requests were done") }) @@ -95,7 +103,14 @@ func TestPollEntity(t *testing.T) { repo := Repository{Name: "testRepo1"} api, _ := swcManager.GetAPI(con, repo) - status, _ := PollEntity(api, 0) + var reports []piperutils.Path + logOutputManager := LogOutputManager{ + LogOutput: "STANDARD", + PiperStep: "pull", + StepReports: reports, + } + + status, _ := PollEntity(api, 0, &logOutputManager) assert.Equal(t, "E", status) assert.Equal(t, 0, len(client.BodyList), "Not all requests were done") }) diff --git a/pkg/abaputils/sap_com_0510.go b/pkg/abaputils/sap_com_0510.go index 24d3d99251..556fc1b74c 100644 --- a/pkg/abaputils/sap_com_0510.go +++ b/pkg/abaputils/sap_com_0510.go @@ -225,6 +225,10 @@ func (api *SAP_COM_0510) GetAction() (string, error) { return abapStatusCode, nil } +func (api *SAP_COM_0510) getRepositoryName() string { + return api.repository.Name +} + func (api *SAP_COM_0510) GetRepository() (bool, string, error, bool) { if api.repository.Name == "" { @@ -404,3 +408,8 @@ func (api *SAP_COM_0510) ConvertTime(logTimeStamp string) time.Time { func (api *SAP_COM_0510) UpdateRepoWithBYOGCredentials(byogAuthMethod string, byogUsername string, byogPassword string) { panic("UpdateRepoWithBYOGCredentials cannot be used in SAP_COM_0510") } + +// Dummy implementation of the "optional" method LogArchive (only used in SAP_COM_0948) +func (api *SAP_COM_0510) GetLogArchive() (result []byte, err error) { + panic("GetLogArchive cannot be used in SAP_COM_0510") +} diff --git a/pkg/abaputils/sap_com_0948.go b/pkg/abaputils/sap_com_0948.go index b006b6cead..5d8477c190 100644 --- a/pkg/abaputils/sap_com_0948.go +++ b/pkg/abaputils/sap_com_0948.go @@ -239,6 +239,10 @@ func (api *SAP_COM_0948) GetAction() (string, error) { return abapStatusCode, nil } +func (api *SAP_COM_0948) getRepositoryName() string { + return api.repository.Name +} + func (api *SAP_COM_0948) GetRepository() (bool, string, error, bool) { if api.repository.Name == "" { @@ -306,6 +310,28 @@ func (api *SAP_COM_0948) Clone() error { } +func (api *SAP_COM_0948) GetLogArchive() (result []byte, err error) { + + connectionDetails := api.con + connectionDetails.URL = api.con.URL + api.path + "/LogArchive/" + api.getUUID() + "/download" + resp, err := GetHTTPResponse("GET", connectionDetails, nil, api.client) + if err != nil { + log.SetErrorCategory(log.ErrorInfrastructure) + _, err = handleHTTPError(resp, err, api.failureMessage, connectionDetails) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + fmt.Println("Error: HTTP Status", resp.StatusCode) + return nil, resp.Request.Context().Err() + } + + body, err := io.ReadAll(resp.Body) + + return body, err +} + func (api *SAP_COM_0948) triggerRequest(cloneConnectionDetails ConnectionDetailsHTTP, jsonBody []byte) error { var err error var body ActionEntity diff --git a/pkg/abaputils/sap_com_0948_test.go b/pkg/abaputils/sap_com_0948_test.go index c0f33bdf3c..580dbd9316 100644 --- a/pkg/abaputils/sap_com_0948_test.go +++ b/pkg/abaputils/sap_com_0948_test.go @@ -573,3 +573,25 @@ func TestGetExecutionLog(t *testing.T) { assert.Equal(t, "First log entry", results.Value[0].Descr) }) } + +func TestGetLogArchive(t *testing.T) { + t.Run("Test Get Log Archive Success", func(t *testing.T) { + + client := &ClientMock{ + BodyList: []string{ + `{ zip content from log archive endpoint }`, + ``, + }, + Token: "myToken", + StatusCode: 200, + } + + apiManager := &SoftwareComponentApiManager{Client: client} + + api, _ := apiManager.GetAPI(conTest0948, Repository{Name: "/DMO/REPO"}) + + results, errAction := api.GetLogArchive() + assert.NoError(t, errAction) + assert.NotEmpty(t, results) + }) +} diff --git a/pkg/abaputils/softwareComponentApiManager.go b/pkg/abaputils/softwareComponentApiManager.go index 0bd7b712f9..720e3066db 100644 --- a/pkg/abaputils/softwareComponentApiManager.go +++ b/pkg/abaputils/softwareComponentApiManager.go @@ -59,6 +59,7 @@ type SoftwareComponentApiInterface interface { setSleepTimeConfig(timeUnit time.Duration, maxSleepTime time.Duration) getSleepTime(n int) (time.Duration, error) getUUID() string + getRepositoryName() string GetRepository() (bool, string, error, bool) Clone() error Pull() error @@ -69,6 +70,7 @@ type SoftwareComponentApiInterface interface { GetLogProtocol(LogResultsV2, int) (result []LogProtocol, count int, err error) ConvertTime(logTimeStamp string) time.Time GetExecutionLog() (ExecutionLog, error) + GetLogArchive() (result []byte, err error) UpdateRepoWithBYOGCredentials(string, string, string) } diff --git a/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml b/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml index dee60fe06a..3f4703d5bc 100644 --- a/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml +++ b/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml @@ -19,6 +19,7 @@ stages: cfServiceKeyConfig: '{"scenario_id":"SAP_COM_0948","type":"basic"}' cfAsync: false ordinal: 30 + logOutput: ZIP 'ATC': ordinal: 40 diff --git a/resources/metadata/abapEnvironmentCheckoutBranch.yaml b/resources/metadata/abapEnvironmentCheckoutBranch.yaml index 37d2729b76..663cc6b4fe 100644 --- a/resources/metadata/abapEnvironmentCheckoutBranch.yaml +++ b/resources/metadata/abapEnvironmentCheckoutBranch.yaml @@ -75,6 +75,17 @@ spec: - PARAMETERS - STAGES - STEPS + - name: logOutput + type: string + description: Specifies how the clone logs from the Manage Software Components App are displayed or saved + scope: + - PARAMETERS + - STAGES + - STEPS + possibleValues: + - ZIP + - STANDARD + default: STANDARD - name: cfApiEndpoint type: string description: Cloud Foundry API Enpoint diff --git a/resources/metadata/abapEnvironmentCloneGitRepo.yaml b/resources/metadata/abapEnvironmentCloneGitRepo.yaml index 6a49f159fa..c4c5829087 100644 --- a/resources/metadata/abapEnvironmentCloneGitRepo.yaml +++ b/resources/metadata/abapEnvironmentCloneGitRepo.yaml @@ -118,6 +118,17 @@ spec: - STAGES - STEPS - GENERAL + - name: logOutput + type: string + description: Specifies how the clone logs from the Manage Software Components App are displayed or saved + scope: + - PARAMETERS + - STAGES + - STEPS + possibleValues: + - ZIP + - STANDARD + default: STANDARD - name: cfApiEndpoint type: string description: Cloud Foundry API Enpoint diff --git a/resources/metadata/abapEnvironmentPullGitRepo.yaml b/resources/metadata/abapEnvironmentPullGitRepo.yaml index ba3da2f3e9..7bf272a82e 100644 --- a/resources/metadata/abapEnvironmentPullGitRepo.yaml +++ b/resources/metadata/abapEnvironmentPullGitRepo.yaml @@ -82,6 +82,17 @@ spec: - STAGES - STEPS - GENERAL + - name: logOutput + type: string + description: Specifies how the clone logs from the Manage Software Components App are displayed or saved + scope: + - PARAMETERS + - STAGES + - STEPS + possibleValues: + - ZIP + - STANDARD + default: STANDARD - name: cfApiEndpoint type: string description: Cloud Foundry API Enpoint From 430bb9b71936ac87eafb9806e30aecd5fc43790b Mon Sep 17 00:00:00 2001 From: Manjunath Date: Thu, 10 Oct 2024 14:30:48 +0200 Subject: [PATCH 13/43] Simply mvn build command (#5143) --- cmd/mavenBuild.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 3280ce7ce7..559e5c2188 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -40,12 +40,7 @@ func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, c } } -func executeMavenGoals(config *mavenBuildOptions, utils maven.Utils, flags []string, goals []string, defines []string, mavenOptions *maven.ExecuteOptions) error { - _, err := maven.Execute(mavenOptions, utils) - return err -} - -func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils, mavenOptions *maven.ExecuteOptions) error { +func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { var flags = []string{"-update-snapshots", "--batch-mode"} if len(config.Profiles) > 0 { flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) @@ -78,7 +73,19 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils, mavenOptions * defines = append(defines, "-Dflatten.mode=resolveCiFriendliesOnly", "-DupdatePomFile=true") } - return executeMavenGoals(config, utils, flags, goals, defines, mavenOptions) + mavenOptions := maven.ExecuteOptions{ + Flags: flags, + Goals: goals, + Defines: defines, + PomPath: config.PomPath, + ProjectSettingsFile: config.ProjectSettingsFile, + GlobalSettingsFile: config.GlobalSettingsFile, + M2Path: config.M2Path, + LogSuccessfulMavenTransfers: config.LogSuccessfulMavenTransfers, + } + + _, err := maven.Execute(&mavenOptions, utils) + return err } func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { @@ -139,13 +146,14 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat LogSuccessfulMavenTransfers: config.LogSuccessfulMavenTransfers, } - if err := executeMavenGoals(config, utils, flags, goals, defines, &mavenOptions); err != nil { + _, err := maven.Execute(&mavenOptions, utils) + if err != nil { return errors.Wrapf(err, "failed to execute maven build for goal(s) '%v'", goals) } if config.CreateBOM { // Separate run for makeBOM goal - if err := runMakeBOMGoal(config, utils, &mavenOptions); err != nil { + if err := runMakeBOMGoal(config, utils); err != nil { return errors.Wrap(err, "failed to execute makeBOM goal") } } From 8f0948d7272cde4ebabe84ba81936b063fc246a9 Mon Sep 17 00:00:00 2001 From: Simon Dold <48808400+doldsimo@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:03:32 +0200 Subject: [PATCH 14/43] [ABAP] Change pipeline defaults (#5144) --- .../com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml b/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml index 3f4703d5bc..e01cc82dbc 100644 --- a/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml +++ b/resources/com.sap.piper/pipeline/abapEnvironmentPipelineDefaults.yml @@ -19,7 +19,7 @@ stages: cfServiceKeyConfig: '{"scenario_id":"SAP_COM_0948","type":"basic"}' cfAsync: false ordinal: 30 - logOutput: ZIP + logOutput: STANDARD 'ATC': ordinal: 40 From af05acad5853f4cc829fe969adf0ac324308bee5 Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:55:39 +0500 Subject: [PATCH 15/43] feat(events): Publish events to GCP PubSub by each step (#5122) * test * test * draft * generator * some polishing * go mod tidy * fix unit * fix unit * fix unit * fix unit * fix unit * resolve review comments * resolve review comments * add debug message on successful publish * refactor to use global vault client * cleanup * rename * clenup * refactor * remove token revocation * handle nil vaultClient and add comments * feat(events): Publish events (generated part) (#5131) * add generated * add generated * refactor vaultClient usage * fix unit tests * fix unit tests * fix --- cmd/abapAddonAssemblyKitCheckCVs_generated.go | 19 ++++ cmd/abapAddonAssemblyKitCheckPV_generated.go | 19 ++++ cmd/abapAddonAssemblyKitCheck_generated.go | 19 ++++ ...AssemblyKitCreateTargetVector_generated.go | 19 ++++ ...ssemblyKitPublishTargetVector_generated.go | 19 ++++ ...onAssemblyKitRegisterPackages_generated.go | 19 ++++ ...donAssemblyKitReleasePackages_generated.go | 19 ++++ ...ssemblyKitReserveNextPackages_generated.go | 19 ++++ ...bapEnvironmentAssembleConfirm_generated.go | 19 ++++ ...apEnvironmentAssemblePackages_generated.go | 19 ++++ cmd/abapEnvironmentBuild_generated.go | 19 ++++ ...abapEnvironmentCheckoutBranch_generated.go | 19 ++++ cmd/abapEnvironmentCloneGitRepo_generated.go | 19 ++++ cmd/abapEnvironmentCreateSystem_generated.go | 19 ++++ cmd/abapEnvironmentCreateTag_generated.go | 19 ++++ cmd/abapEnvironmentPullGitRepo_generated.go | 19 ++++ ...nvironmentPushATCSystemConfig_generated.go | 19 ++++ cmd/abapEnvironmentRunATCCheck_generated.go | 19 ++++ cmd/abapEnvironmentRunAUnitTest_generated.go | 19 ++++ ...scapePortalUpdateAddOnProduct_generated.go | 19 ++++ cmd/ansSendEvent_generated.go | 19 ++++ cmd/apiKeyValueMapDownload_generated.go | 19 ++++ cmd/apiKeyValueMapUpload_generated.go | 19 ++++ cmd/apiProviderDownload_generated.go | 19 ++++ cmd/apiProviderList_generated.go | 19 ++++ cmd/apiProviderUpload_generated.go | 19 ++++ cmd/apiProxyDownload_generated.go | 19 ++++ cmd/apiProxyList_generated.go | 19 ++++ cmd/apiProxyUpload_generated.go | 19 ++++ cmd/artifactPrepareVersion_generated.go | 19 ++++ cmd/ascAppUpload_generated.go | 19 ++++ cmd/awsS3Upload_generated.go | 19 ++++ cmd/azureBlobUpload_generated.go | 19 ++++ cmd/batsExecuteTests_generated.go | 19 ++++ cmd/checkmarxExecuteScan_generated.go | 19 ++++ cmd/checkmarxOneExecuteScan_generated.go | 19 ++++ cmd/cloudFoundryCreateServiceKey_generated.go | 19 ++++ cmd/cloudFoundryCreateService_generated.go | 19 ++++ cmd/cloudFoundryCreateSpace_generated.go | 19 ++++ cmd/cloudFoundryDeleteService_generated.go | 19 ++++ cmd/cloudFoundryDeleteSpace_generated.go | 19 ++++ cmd/cloudFoundryDeploy_generated.go | 19 ++++ cmd/cnbBuild_generated.go | 19 ++++ cmd/codeqlExecuteScan_generated.go | 19 ++++ ...ontainerExecuteStructureTests_generated.go | 19 ++++ cmd/containerSaveImage_generated.go | 19 ++++ cmd/contrastExecuteScan_generated.go | 19 ++++ cmd/credentialdiggerScan_generated.go | 19 ++++ cmd/detectExecuteScan_generated.go | 19 ++++ cmd/fortifyExecuteScan_generated.go | 19 ++++ cmd/gaugeExecuteTests_generated.go | 19 ++++ cmd/gcpPublishEvent.go | 77 ++++------------ cmd/gcpPublishEvent_generated.go | 19 ++++ cmd/gcpPublishEvent_test.go | 10 +-- cmd/gctsCloneRepository_generated.go | 19 ++++ cmd/gctsCreateRepository_generated.go | 19 ++++ cmd/gctsDeploy_generated.go | 19 ++++ cmd/gctsExecuteABAPQualityChecks_generated.go | 19 ++++ cmd/gctsExecuteABAPUnitTests_generated.go | 19 ++++ cmd/gctsRollback_generated.go | 19 ++++ cmd/githubCheckBranchProtection_generated.go | 19 ++++ cmd/githubCommentIssue_generated.go | 19 ++++ cmd/githubCreateIssue_generated.go | 19 ++++ cmd/githubCreatePullRequest_generated.go | 19 ++++ cmd/githubPublishRelease_generated.go | 19 ++++ cmd/githubSetCommitStatus_generated.go | 19 ++++ cmd/gitopsUpdateDeployment_generated.go | 19 ++++ cmd/golangBuild_generated.go | 19 ++++ cmd/gradleExecuteBuild_generated.go | 19 ++++ cmd/hadolintExecute_generated.go | 19 ++++ cmd/helmExecute_generated.go | 19 ++++ cmd/imagePushToRegistry_generated.go | 19 ++++ cmd/influxWriteData_generated.go | 19 ++++ cmd/integrationArtifactDeploy_generated.go | 19 ++++ cmd/integrationArtifactDownload_generated.go | 19 ++++ ...tegrationArtifactGetMplStatus_generated.go | 19 ++++ ...ionArtifactGetServiceEndpoint_generated.go | 19 ++++ cmd/integrationArtifactResource_generated.go | 19 ++++ cmd/integrationArtifactTransport_generated.go | 19 ++++ ...rtifactTriggerIntegrationTest_generated.go | 19 ++++ cmd/integrationArtifactUnDeploy_generated.go | 19 ++++ ...onArtifactUpdateConfiguration_generated.go | 19 ++++ cmd/integrationArtifactUpload_generated.go | 19 ++++ cmd/isChangeInDevelopment_generated.go | 19 ++++ cmd/jsonApplyPatch_generated.go | 19 ++++ cmd/kanikoExecute_generated.go | 19 ++++ cmd/karmaExecuteTests_generated.go | 19 ++++ cmd/kubernetesDeploy_generated.go | 19 ++++ cmd/malwareExecuteScan_generated.go | 19 ++++ cmd/mavenBuild_generated.go | 19 ++++ cmd/mavenExecuteIntegration_generated.go | 19 ++++ cmd/mavenExecuteStaticCodeChecks_generated.go | 19 ++++ cmd/mavenExecute_generated.go | 19 ++++ cmd/mtaBuild_generated.go | 19 ++++ cmd/newmanExecute_generated.go | 19 ++++ cmd/nexusUpload_generated.go | 19 ++++ cmd/npmExecuteLint_generated.go | 19 ++++ cmd/npmExecuteScripts_generated.go | 19 ++++ cmd/pipelineCreateScanSummary_generated.go | 19 ++++ cmd/piper.go | 9 ++ cmd/protecodeExecuteScan_generated.go | 19 ++++ cmd/pythonBuild_generated.go | 19 ++++ cmd/shellExecute_generated.go | 19 ++++ cmd/sonarExecuteScan_generated.go | 19 ++++ cmd/terraformExecute_generated.go | 19 ++++ cmd/tmsExport_generated.go | 19 ++++ cmd/tmsUpload_generated.go | 19 ++++ cmd/transportRequestDocIDFromGit_generated.go | 19 ++++ cmd/transportRequestReqIDFromGit_generated.go | 19 ++++ cmd/transportRequestUploadCTS_generated.go | 19 ++++ cmd/transportRequestUploadRFC_generated.go | 19 ++++ cmd/transportRequestUploadSOLMAN_generated.go | 19 ++++ cmd/uiVeri5ExecuteTests_generated.go | 19 ++++ cmd/vaultRotateSecretId_generated.go | 19 ++++ cmd/whitesourceExecuteScan_generated.go | 19 ++++ cmd/xsDeploy_generated.go | 19 ++++ go.mod | 1 + go.sum | 6 ++ pkg/config/config.go | 3 +- pkg/config/vault.go | 21 ++++- pkg/gcp/pubsub.go | 88 +++++++++++-------- pkg/gcp/pubsub_test.go | 45 ---------- pkg/gcp/token.go | 81 +++++++++++------ pkg/gcp/token_test.go | 67 ++++++++------ pkg/generator/helper/helper.go | 19 ++++ .../custom_step_code_generated.golden | 19 ++++ .../step_code_generated.golden | 19 ++++ pkg/telemetry/telemetry.go | 10 +++ pkg/vault/oidc.go | 8 +- 129 files changed, 2426 insertions(+), 204 deletions(-) delete mode 100644 pkg/gcp/pubsub_test.go diff --git a/cmd/abapAddonAssemblyKitCheckCVs_generated.go b/cmd/abapAddonAssemblyKitCheckCVs_generated.go index 8d2f2186f3..651d25e833 100644 --- a/cmd/abapAddonAssemblyKitCheckCVs_generated.go +++ b/cmd/abapAddonAssemblyKitCheckCVs_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -124,6 +125,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -150,6 +156,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitCheckPV_generated.go b/cmd/abapAddonAssemblyKitCheckPV_generated.go index 0038d14c4a..c3d4ac2206 100644 --- a/cmd/abapAddonAssemblyKitCheckPV_generated.go +++ b/cmd/abapAddonAssemblyKitCheckPV_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -124,6 +125,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -150,6 +156,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitCheck_generated.go b/cmd/abapAddonAssemblyKitCheck_generated.go index 3711fb3110..0dec1e978e 100644 --- a/cmd/abapAddonAssemblyKitCheck_generated.go +++ b/cmd/abapAddonAssemblyKitCheck_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -127,6 +128,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -153,6 +159,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go index 7224504166..2d237e7764 100644 --- a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -126,6 +127,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -152,6 +158,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go b/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go index 239fbf27b2..87de4808af 100644 --- a/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -97,6 +98,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -122,6 +128,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go index 25043faba6..3d2eeffa73 100644 --- a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go +++ b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -127,6 +128,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -153,6 +159,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitReleasePackages_generated.go b/cmd/abapAddonAssemblyKitReleasePackages_generated.go index adef8f3516..ca9f78c1fc 100644 --- a/cmd/abapAddonAssemblyKitReleasePackages_generated.go +++ b/cmd/abapAddonAssemblyKitReleasePackages_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -125,6 +126,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -151,6 +157,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go index 68c0850e50..58f6e627b4 100644 --- a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go +++ b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -131,6 +132,11 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -157,6 +163,19 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentAssembleConfirm_generated.go b/cmd/abapEnvironmentAssembleConfirm_generated.go index 4414d8c998..9b7bfafdc2 100644 --- a/cmd/abapEnvironmentAssembleConfirm_generated.go +++ b/cmd/abapEnvironmentAssembleConfirm_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -121,6 +122,11 @@ func AbapEnvironmentAssembleConfirmCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -147,6 +153,19 @@ func AbapEnvironmentAssembleConfirmCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentAssemblePackages_generated.go b/cmd/abapEnvironmentAssemblePackages_generated.go index 256f85a79a..e9ac906aaf 100644 --- a/cmd/abapEnvironmentAssemblePackages_generated.go +++ b/cmd/abapEnvironmentAssemblePackages_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -123,6 +124,11 @@ Platform ABAP Environment system and saves the corresponding [SAR archive](https return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -149,6 +155,19 @@ Platform ABAP Environment system and saves the corresponding [SAR archive](https GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentBuild_generated.go b/cmd/abapEnvironmentBuild_generated.go index ed84c605d4..c87a380f8d 100644 --- a/cmd/abapEnvironmentBuild_generated.go +++ b/cmd/abapEnvironmentBuild_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -136,6 +137,11 @@ func AbapEnvironmentBuildCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -162,6 +168,19 @@ func AbapEnvironmentBuildCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentCheckoutBranch_generated.go b/cmd/abapEnvironmentCheckoutBranch_generated.go index 6b3384c279..06fc95284f 100644 --- a/cmd/abapEnvironmentCheckoutBranch_generated.go +++ b/cmd/abapEnvironmentCheckoutBranch_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -97,6 +98,11 @@ Please provide either of the following options: return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -122,6 +128,19 @@ Please provide either of the following options: GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentCloneGitRepo_generated.go b/cmd/abapEnvironmentCloneGitRepo_generated.go index a8c26a5223..c8759f5b83 100644 --- a/cmd/abapEnvironmentCloneGitRepo_generated.go +++ b/cmd/abapEnvironmentCloneGitRepo_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -102,6 +103,11 @@ Please provide either of the following options: return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -127,6 +133,19 @@ Please provide either of the following options: GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentCreateSystem_generated.go b/cmd/abapEnvironmentCreateSystem_generated.go index 5fa1dc9ae4..9928abc03c 100644 --- a/cmd/abapEnvironmentCreateSystem_generated.go +++ b/cmd/abapEnvironmentCreateSystem_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -96,6 +97,11 @@ func AbapEnvironmentCreateSystemCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -121,6 +127,19 @@ func AbapEnvironmentCreateSystemCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentCreateTag_generated.go b/cmd/abapEnvironmentCreateTag_generated.go index 02b47ef577..22e9260669 100644 --- a/cmd/abapEnvironmentCreateTag_generated.go +++ b/cmd/abapEnvironmentCreateTag_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -100,6 +101,11 @@ Please provide either of the following options: return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -125,6 +131,19 @@ Please provide either of the following options: GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentPullGitRepo_generated.go b/cmd/abapEnvironmentPullGitRepo_generated.go index 8cc70c1961..05cc2a0c21 100644 --- a/cmd/abapEnvironmentPullGitRepo_generated.go +++ b/cmd/abapEnvironmentPullGitRepo_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -99,6 +100,11 @@ Please provide either of the following options: return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -124,6 +130,19 @@ Please provide either of the following options: GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentPushATCSystemConfig_generated.go b/cmd/abapEnvironmentPushATCSystemConfig_generated.go index d39f8e7a0a..c33c2d9eb9 100644 --- a/cmd/abapEnvironmentPushATCSystemConfig_generated.go +++ b/cmd/abapEnvironmentPushATCSystemConfig_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -94,6 +95,11 @@ Please provide either of the following options: return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -119,6 +125,19 @@ Please provide either of the following options: GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentRunATCCheck_generated.go b/cmd/abapEnvironmentRunATCCheck_generated.go index c7ae16d7f1..04a52ad14d 100644 --- a/cmd/abapEnvironmentRunATCCheck_generated.go +++ b/cmd/abapEnvironmentRunATCCheck_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -99,6 +100,11 @@ Regardless of the option you chose, please make sure to provide the configuratio return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -124,6 +130,19 @@ Regardless of the option you chose, please make sure to provide the configuratio GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapEnvironmentRunAUnitTest_generated.go b/cmd/abapEnvironmentRunAUnitTest_generated.go index 0a7466ab39..a3e865cd2d 100644 --- a/cmd/abapEnvironmentRunAUnitTest_generated.go +++ b/cmd/abapEnvironmentRunAUnitTest_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -98,6 +99,11 @@ Regardless of the option you chose, please make sure to provide the object set c return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -123,6 +129,19 @@ Regardless of the option you chose, please make sure to provide the object set c GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go b/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go index 3ebebd7824..a44196072b 100644 --- a/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go +++ b/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -81,6 +82,11 @@ func AbapLandscapePortalUpdateAddOnProductCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -106,6 +112,19 @@ func AbapLandscapePortalUpdateAddOnProductCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/ansSendEvent_generated.go b/cmd/ansSendEvent_generated.go index cc32a15503..9993c7745a 100644 --- a/cmd/ansSendEvent_generated.go +++ b/cmd/ansSendEvent_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -90,6 +91,11 @@ func AnsSendEventCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -115,6 +121,19 @@ func AnsSendEventCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiKeyValueMapDownload_generated.go b/cmd/apiKeyValueMapDownload_generated.go index 6d1f650f20..2ced0e9124 100644 --- a/cmd/apiKeyValueMapDownload_generated.go +++ b/cmd/apiKeyValueMapDownload_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -82,6 +83,11 @@ Learn more about the SAP API Management API for downloading an Key Value Map art return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -107,6 +113,19 @@ Learn more about the SAP API Management API for downloading an Key Value Map art GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiKeyValueMapUpload_generated.go b/cmd/apiKeyValueMapUpload_generated.go index 4562a46a87..c62d559b3f 100644 --- a/cmd/apiKeyValueMapUpload_generated.go +++ b/cmd/apiKeyValueMapUpload_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -83,6 +84,11 @@ Learn more about the SAP API Management API for creating an API key value map ar return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -108,6 +114,19 @@ Learn more about the SAP API Management API for creating an API key value map ar GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiProviderDownload_generated.go b/cmd/apiProviderDownload_generated.go index ccc7a434b7..def4eb576c 100644 --- a/cmd/apiProviderDownload_generated.go +++ b/cmd/apiProviderDownload_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -81,6 +82,11 @@ func ApiProviderDownloadCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -106,6 +112,19 @@ func ApiProviderDownloadCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiProviderList_generated.go b/cmd/apiProviderList_generated.go index 8a5936259f..76ce80c0f1 100644 --- a/cmd/apiProviderList_generated.go +++ b/cmd/apiProviderList_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -118,6 +119,11 @@ func ApiProviderListCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -144,6 +150,19 @@ func ApiProviderListCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiProviderUpload_generated.go b/cmd/apiProviderUpload_generated.go index a133ed8e54..1b1728099f 100644 --- a/cmd/apiProviderUpload_generated.go +++ b/cmd/apiProviderUpload_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -81,6 +82,11 @@ Learn more about API Management api for creating an API provider artifact [here] return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -106,6 +112,19 @@ Learn more about API Management api for creating an API provider artifact [here] GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiProxyDownload_generated.go b/cmd/apiProxyDownload_generated.go index 81b686db49..7f0238202b 100644 --- a/cmd/apiProxyDownload_generated.go +++ b/cmd/apiProxyDownload_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -81,6 +82,11 @@ func ApiProxyDownloadCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -106,6 +112,19 @@ func ApiProxyDownloadCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiProxyList_generated.go b/cmd/apiProxyList_generated.go index 09938f55a6..53f8c86d7b 100644 --- a/cmd/apiProxyList_generated.go +++ b/cmd/apiProxyList_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -118,6 +119,11 @@ func ApiProxyListCommand() *cobra.Command { return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -144,6 +150,19 @@ func ApiProxyListCommand() *cobra.Command { GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/apiProxyUpload_generated.go b/cmd/apiProxyUpload_generated.go index 6bd6775828..51bf6c42a1 100644 --- a/cmd/apiProxyUpload_generated.go +++ b/cmd/apiProxyUpload_generated.go @@ -8,6 +8,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/splunk" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -81,6 +82,11 @@ Learn more about the SAP API Management API for uploading an api proxy artifact return nil }, Run: func(_ *cobra.Command, _ []string) { + vaultClient := config.GlobalVaultClient() + if vaultClient != nil { + defer vaultClient.MustRevokeToken() + } + stepTelemetryData := telemetry.CustomData{} stepTelemetryData.ErrorCode = "1" handler := func() { @@ -106,6 +112,19 @@ Learn more about the SAP API Management API for uploading an api proxy artifact GeneralConfig.HookConfig.SplunkConfig.SendLogs) splunkClient.Send(telemetryClient.GetData(), logCollector) } + if GeneralConfig.HookConfig.GCPPubSubConfig.Enabled { + err := gcp.NewGcpPubsubClient( + vaultClient, + GeneralConfig.HookConfig.GCPPubSubConfig.ProjectNumber, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityPool, + GeneralConfig.HookConfig.GCPPubSubConfig.IdentityProvider, + GeneralConfig.CorrelationID, + GeneralConfig.HookConfig.OIDCConfig.RoleID, + ).Publish(GeneralConfig.HookConfig.GCPPubSubConfig.Topic, telemetryClient.GetDataBytes()) + if err != nil { + log.Entry().WithError(err).Warn("event publish failed") + } + } } log.DeferExitHandler(handler) defer handler() diff --git a/cmd/artifactPrepareVersion_generated.go b/cmd/artifactPrepareVersion_generated.go index 7953e72522..e39872d89b 100644 --- a/cmd/artifactPrepareVersion_generated.go +++ b/cmd/artifactPrepareVersion_generated.go @@ -9,6 +9,7 @@ import ( "time" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/gcp" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/splunk" @@ -210,6 +211,11 @@ Define ` + "`" + `buildTool: custom` + "`" + `, ` + "`" + `filePath: --install --force --namespace --install --force --namespace Date: Tue, 15 Oct 2024 09:14:23 +0200 Subject: [PATCH 16/43] feat(build): add buildSettingsInfo for gradleExecuteBuild (#5043) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * creating build settings info from gradle build * fix linting issue and typo * fixing step description * Apply suggestions from code review Co-authored-by: Tilo Körner <70266685+tiloKo@users.noreply.github.com> * Address code review comments * Fix tests * Fix tests * Fix tests * Fix test --------- Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Tilo Körner <70266685+tiloKo@users.noreply.github.com> Co-authored-by: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com> Co-authored-by: Vyacheslav Starostin --- cmd/gradleExecuteBuild.go | 23 +++++++++++++++++++++- cmd/gradleExecuteBuild_generated.go | 21 +++++++++++++++++++- pkg/buildsettings/buildSettings.go | 22 ++++++++++----------- pkg/buildsettings/buildSettings_test.go | 11 ++++------- resources/metadata/gradleExecuteBuild.yaml | 11 +++++++++++ 5 files changed, 68 insertions(+), 20 deletions(-) diff --git a/cmd/gradleExecuteBuild.go b/cmd/gradleExecuteBuild.go index 8a635adf2b..6fe9361624 100644 --- a/cmd/gradleExecuteBuild.go +++ b/cmd/gradleExecuteBuild.go @@ -9,6 +9,7 @@ import ( "strings" "text/template" + "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/command" "github.com/SAP/jenkins-library/pkg/gradle" "github.com/SAP/jenkins-library/pkg/log" @@ -18,7 +19,8 @@ import ( ) const ( - gradleBomFilename = "bom-gradle" + gradleBomFilename = "bom-gradle" + stepNameForBuildSettings = "gradleExecuteBuild" ) var ( @@ -189,6 +191,25 @@ func runGradleExecuteBuild(config *gradleExecuteBuildOptions, telemetryData *tel return err } + log.Entry().Debugf("creating build settings information...") + + dockerImage, err := GetDockerImageValue(stepNameForBuildSettings) + if err != nil { + return fmt.Errorf("failed to retrieve dockerImage configuration: %w", err) + } + + gradleConfig := buildsettings.BuildOptions{ + CreateBOM: config.CreateBOM, + Publish: config.Publish, + BuildSettingsInfo: config.BuildSettingsInfo, + DockerImage: dockerImage, + } + buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&gradleConfig, stepNameForBuildSettings) + if err != nil { + log.Entry().Warnf("failed to create build settings info: %v", err) + } + pipelineEnv.custom.buildSettingsInfo = buildSettingsInfo + log.Entry().Info("Publishing of artifacts to staging repository...") if config.Publish { if err := publishArtifacts(config, utils, pipelineEnv); err != nil { diff --git a/cmd/gradleExecuteBuild_generated.go b/cmd/gradleExecuteBuild_generated.go index 585fb16420..7f60984f84 100644 --- a/cmd/gradleExecuteBuild_generated.go +++ b/cmd/gradleExecuteBuild_generated.go @@ -38,6 +38,7 @@ type gradleExecuteBuildOptions struct { ExcludeCreateBOMForProjects []string `json:"excludeCreateBOMForProjects,omitempty"` ExcludePublishingForProjects []string `json:"excludePublishingForProjects,omitempty"` BuildFlags []string `json:"buildFlags,omitempty"` + BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` } type gradleExecuteBuildReports struct { @@ -78,7 +79,8 @@ func (p *gradleExecuteBuildReports) persist(stepConfig gradleExecuteBuildOptions type gradleExecuteBuildCommonPipelineEnvironment struct { custom struct { - artifacts piperenv.Artifacts + artifacts piperenv.Artifacts + buildSettingsInfo string } } @@ -89,6 +91,7 @@ func (p *gradleExecuteBuildCommonPipelineEnvironment) persist(path, resourceName value interface{} }{ {category: "custom", name: "artifacts", value: p.custom.artifacts}, + {category: "custom", name: "buildSettingsInfo", value: p.custom.buildSettingsInfo}, } errCount := 0 @@ -242,6 +245,7 @@ func addGradleExecuteBuildFlags(cmd *cobra.Command, stepConfig *gradleExecuteBui cmd.Flags().StringSliceVar(&stepConfig.ExcludeCreateBOMForProjects, "excludeCreateBOMForProjects", []string{}, "Defines which projects/subprojects will be ignored during bom creation. Only if applyCreateBOMForAllProjects is set to true") cmd.Flags().StringSliceVar(&stepConfig.ExcludePublishingForProjects, "excludePublishingForProjects", []string{}, "Defines which projects/subprojects will be ignored during publishing. Only if applyCreateBOMForAllProjects is set to true") cmd.Flags().StringSliceVar(&stepConfig.BuildFlags, "buildFlags", []string{}, "Defines a list of tasks and/or arguments to be provided for gradle in the respective order to be executed. This list takes precedence if specified over 'task' parameter") + cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the gradle build. This information is typically used for compliance related processes.") } @@ -421,6 +425,20 @@ func gradleExecuteBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: []string{}, }, + { + Name: "buildSettingsInfo", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "custom/buildSettingsInfo", + }, + }, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_buildSettingsInfo"), + }, }, }, Containers: []config.Container{ @@ -440,6 +458,7 @@ func gradleExecuteBuildMetadata() config.StepData { Type: "piperEnvironment", Parameters: []map[string]interface{}{ {"name": "custom/artifacts", "type": "piperenv.Artifacts"}, + {"name": "custom/buildSettingsInfo"}, }, }, }, diff --git a/pkg/buildsettings/buildSettings.go b/pkg/buildsettings/buildSettings.go index 6b66dbc347..20594eb9d1 100644 --- a/pkg/buildsettings/buildSettings.go +++ b/pkg/buildsettings/buildSettings.go @@ -9,15 +9,15 @@ import ( ) type BuildSettings struct { - GolangBuild []BuildOptions `json:"golangBuild,omitempty"` - GradleBuild []BuildOptions `json:"gradleBuild,omitempty"` - HelmExecute []BuildOptions `json:"helmExecute,omitempty"` - KanikoExecute []BuildOptions `json:"kanikoExecute,omitempty"` - MavenBuild []BuildOptions `json:"mavenBuild,omitempty"` - MtaBuild []BuildOptions `json:"mtaBuild,omitempty"` - PythonBuild []BuildOptions `json:"pythonBuild,omitempty"` - NpmExecuteScripts []BuildOptions `json:"npmExecuteScripts,omitempty"` - CnbBuild []BuildOptions `json:"cnbBuild,omitempty"` + GolangBuild []BuildOptions `json:"golangBuild,omitempty"` + GradleExecuteBuild []BuildOptions `json:"gradleExecuteBuild,omitempty"` + HelmExecute []BuildOptions `json:"helmExecute,omitempty"` + KanikoExecute []BuildOptions `json:"kanikoExecute,omitempty"` + MavenBuild []BuildOptions `json:"mavenBuild,omitempty"` + MtaBuild []BuildOptions `json:"mtaBuild,omitempty"` + PythonBuild []BuildOptions `json:"pythonBuild,omitempty"` + NpmExecuteScripts []BuildOptions `json:"npmExecuteScripts,omitempty"` + CnbBuild []BuildOptions `json:"cnbBuild,omitempty"` } type BuildOptions struct { @@ -74,9 +74,9 @@ func CreateBuildSettingsInfo(config *BuildOptions, buildTool string) (string, er jsonResult, err = json.Marshal(BuildSettings{ GolangBuild: settings, }) - case "gradleBuild": + case "gradleExecuteBuild": jsonResult, err = json.Marshal(BuildSettings{ - GradleBuild: settings, + GradleExecuteBuild: settings, }) case "helmExecute": jsonResult, err = json.Marshal(BuildSettings{ diff --git a/pkg/buildsettings/buildSettings_test.go b/pkg/buildsettings/buildSettings_test.go index 7d135b97af..264d2145f2 100644 --- a/pkg/buildsettings/buildSettings_test.go +++ b/pkg/buildsettings/buildSettings_test.go @@ -1,6 +1,3 @@ -//go:build unit -// +build unit - package buildsettings import ( @@ -28,9 +25,9 @@ func TestCreateBuildSettingsInfo(t *testing.T) { expected: "{\"golangBuild\":[{\"dockerImage\":\"golang:latest\"}]}", }, { - config: BuildOptions{CreateBOM: true}, - buildTool: "gradleBuild", - expected: "{\"gradleBuild\":[{\"createBOM\":true}]}", + config: BuildOptions{CreateBOM: true, DockerImage: "gradle:latest"}, + buildTool: "gradleExecuteBuild", + expected: "{\"gradleExecuteBuild\":[{\"createBOM\":true,\"dockerImage\":\"gradle:latest\"}]}", }, { config: BuildOptions{Publish: true}, @@ -77,7 +74,7 @@ func TestCreateBuildSettingsInfo(t *testing.T) { for _, testCase := range testTableConfig { buildSettings, err := CreateBuildSettingsInfo(&testCase.config, testCase.buildTool) assert.Nil(t, err) - assert.Equal(t, buildSettings, testCase.expected) + assert.Equal(t, testCase.expected, buildSettings) } }) diff --git a/resources/metadata/gradleExecuteBuild.yaml b/resources/metadata/gradleExecuteBuild.yaml index e44c92f46e..ed0ba89ce6 100644 --- a/resources/metadata/gradleExecuteBuild.yaml +++ b/resources/metadata/gradleExecuteBuild.yaml @@ -157,6 +157,16 @@ spec: - PARAMETERS - STAGES - STEPS + - name: buildSettingsInfo + type: string + description: build settings info is typically filled by the step automatically to create information about the build settings that were used during the gradle build. This information is typically used for compliance related processes. + scope: + - STEPS + - STAGES + - PARAMETERS + resourceRef: + - name: commonPipelineEnvironment + param: custom/buildSettingsInfo outputs: resources: - name: reports @@ -169,6 +179,7 @@ spec: params: - name: custom/artifacts type: "piperenv.Artifacts" + - name: custom/buildSettingsInfo containers: - name: gradle image: gradle:6-jdk11-alpine From bd8b08b93e2313c21c4d06aec02369283a62f0f3 Mon Sep 17 00:00:00 2001 From: Srinikitha Kondreddy Date: Tue, 15 Oct 2024 14:29:09 +0200 Subject: [PATCH 17/43] fix: add missing return value for error (#5146) --- pkg/transportrequest/cts/upload.go | 2 +- pkg/transportrequest/cts/upload_test.go | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/transportrequest/cts/upload.go b/pkg/transportrequest/cts/upload.go index a50458df70..6f959d7463 100644 --- a/pkg/transportrequest/cts/upload.go +++ b/pkg/transportrequest/cts/upload.go @@ -194,7 +194,7 @@ func getFioriDeployStatement( if len(app.Name) > 0 { re := regexp.MustCompile(pattern) if !re.MatchString(app.Name) { - fmt.Errorf("application name '%s' contains spaces or special characters. It is not according to the '%s'", app.Name, pattern) + return "", fmt.Errorf("application name '%s' contains spaces or special characters and is not according to the regex '%s'.", app.Name, pattern) } log.Entry().Debugf("application name '%s' used from piper config", app.Name) cmd = append(cmd, "--name", app.Name) diff --git a/pkg/transportrequest/cts/upload_test.go b/pkg/transportrequest/cts/upload_test.go index 7a7c3cdeec..852d4908f2 100644 --- a/pkg/transportrequest/cts/upload_test.go +++ b/pkg/transportrequest/cts/upload_test.go @@ -4,10 +4,11 @@ package cts import ( + "testing" + "github.com/SAP/jenkins-library/pkg/mock" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" - "testing" ) func TestUploadCTS(t *testing.T) { @@ -99,6 +100,26 @@ func TestUploadCTS(t *testing.T) { assert.Equal(t, []string{"ABAP_USER=me", "ABAP_PASSWORD=******"}, cmd.Env) } }) + + t.Run("fail in case of invalid app name", func(t *testing.T) { + cmd := mock.ShellMockRunner{} + action := UploadAction{ + Connection: Connection{Endpoint: "https://example.org:8080/cts", Client: "001", User: "me", Password: "******"}, + Application: Application{Pack: "abapPackage", Name: "app Name", Desc: "the Desc"}, + Node: Node{ + DeployDependencies: []string{}, + InstallOpts: []string{}, + }, + TransportRequestID: "12345678", + ConfigFile: "ui5-deploy.yaml", + DeployUser: "doesNotMatterInThisCase", + } + + err := action.Perform(&cmd) + expectedErrorMessge := "application name 'app Name' contains spaces or special characters and is not according to the regex '^[a-zA-Z0-9_]+$'." + + assert.EqualErrorf(t, err, expectedErrorMessge, "invalid app name") + }) }) t.Run("config file releated tests", func(t *testing.T) { From 36c555ea61240892ee4d230117c5bccaff4bcbfc Mon Sep 17 00:00:00 2001 From: Ivan Nikiforov Date: Wed, 16 Oct 2024 13:45:23 +0200 Subject: [PATCH 18/43] fix: too long string written to cpe git.commitMessage (#5147) * fix too long CPE string written to git/commitMessage * Add debug log * Fix debug log * Truncate long git commit message title * Add tests for truncateString * Fix test * Fix tests * Fix tests --------- Co-authored-by: Ivan Nikiforov --- cmd/artifactPrepareVersion.go | 17 ++++++++++++++++- cmd/artifactPrepareVersion_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/cmd/artifactPrepareVersion.go b/cmd/artifactPrepareVersion.go index 173a693dac..4c7f77320d 100644 --- a/cmd/artifactPrepareVersion.go +++ b/cmd/artifactPrepareVersion.go @@ -237,7 +237,13 @@ func runArtifactPrepareVersion(config *artifactPrepareVersionOptions, telemetryD commonPipelineEnvironment.git.commitID = gitCommitID // this commitID changes and is not necessarily the HEAD commitID commonPipelineEnvironment.artifactVersion = newVersion commonPipelineEnvironment.originalArtifactVersion = version - commonPipelineEnvironment.git.commitMessage = gitCommitMessage + + gitCommitMessages := strings.Split(gitCommitMessage, "\n") + commitMessage := truncateString(gitCommitMessages[0], 50) // Github recommends to keep commit message title less than 50 chars + + commonPipelineEnvironment.git.commitMessage = commitMessage + + log.Entry().Debug("CPE git commitMessage:", commitMessage) // we may replace GetVersion() above with GetCoordinates() at some point ... coordinates, err := artifact.GetCoordinates() @@ -254,6 +260,15 @@ func runArtifactPrepareVersion(config *artifactPrepareVersionOptions, telemetryD return nil } +func truncateString(str string, maxLength int) string { + chars := []rune(str) + + if len(chars) > maxLength { + return string(chars[:maxLength]) + "..." + } + return str +} + func openGit() (gitRepository, error) { workdir, _ := os.Getwd() return gitUtils.PlainOpen(workdir) diff --git a/cmd/artifactPrepareVersion_test.go b/cmd/artifactPrepareVersion_test.go index 9d42f44dd5..9199865822 100644 --- a/cmd/artifactPrepareVersion_test.go +++ b/cmd/artifactPrepareVersion_test.go @@ -887,3 +887,33 @@ func TestPropagateVersion(t *testing.T) { assert.Contains(t, fmt.Sprint(err), "failed to retrieve artifact") }) } + +func TestTruncateString(t *testing.T) { + t.Run("input string longer than maxLength - truncate", func(t *testing.T) { + inputStr := "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor" + expected := "Lorem ipsum dolor sit amet, consectetur adipiscing..." + + outputStr := truncateString(inputStr, 50) + assert.Equal(t, outputStr, expected) + }) + + t.Run("input string shorter than maxLength - return as is", func(t *testing.T) { + inputStr := "Lorem ipsum dolor sit amet" + outputStr := truncateString(inputStr, 50) + + assert.Equal(t, outputStr, inputStr) + }) + + t.Run("input string contains unicode chars", func(t *testing.T) { + inputStr := "パイパーは素晴らしい図書館です" + expected := "パイパーは..." + + outputStr := truncateString(inputStr, 5) + assert.Equal(t, outputStr, expected) + }) + + t.Run("input string is empty", func(t *testing.T) { + outputStr := truncateString("", 5) + assert.Equal(t, outputStr, "") + }) +} From e1563e023717650696c7abb601ca6b3c050edec2 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Wed, 16 Oct 2024 14:10:52 +0200 Subject: [PATCH 19/43] Add additional info to coordinates (#5149) --- pkg/versioning/versioning.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/versioning/versioning.go b/pkg/versioning/versioning.go index a716cb192f..9ae5dfd79b 100644 --- a/pkg/versioning/versioning.go +++ b/pkg/versioning/versioning.go @@ -13,13 +13,13 @@ import ( // Coordinates to address the artifact coordinates like groupId, artifactId, version and packaging type Coordinates struct { - GroupID string - ArtifactID string - Version string - Packaging string - BuildPath string - URL string - PURL string + GroupID string `json:"groupId"` + ArtifactID string `json:"artifactId"` + Version string `json:"version"` + Packaging string `json:"packaging"` + BuildPath string `json:"buildPath"` + URL string `json:"url"` + PURL string `json:"purl"` } // Artifact defines the versioning operations for various build tools From 34a60daad1d241ffc6c05c8003468e7e2c57275b Mon Sep 17 00:00:00 2001 From: Ivan Nikiforov Date: Wed, 16 Oct 2024 16:09:51 +0200 Subject: [PATCH 20/43] Update documentation for scriptArguments param (#5128) * Update documentation for scriptArguments param * Update resources/metadata/shellExecute.yaml Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Update doc with new examples * Fix typo --------- Co-authored-by: Ivan Nikiforov Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- cmd/shellExecute_generated.go | 2 +- resources/metadata/shellExecute.yaml | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/shellExecute_generated.go b/cmd/shellExecute_generated.go index a69334c40f..3d0eeacb1a 100644 --- a/cmd/shellExecute_generated.go +++ b/cmd/shellExecute_generated.go @@ -142,7 +142,7 @@ func ShellExecuteCommand() *cobra.Command { func addShellExecuteFlags(cmd *cobra.Command, stepConfig *shellExecuteOptions) { cmd.Flags().StringSliceVar(&stepConfig.Sources, "sources", []string{}, "Scripts paths that must be present in the current workspace or https links to scripts. Only https urls from github are allowed and must be in the format :https://{githubBaseurl}/api/v3/repos/{owner}/{repository}/contents/{path to script} Authentication for the download is only supported via the 'githubToken' param. Make sure the script has the necessary execute permissions.") cmd.Flags().StringVar(&stepConfig.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") - cmd.Flags().StringSliceVar(&stepConfig.ScriptArguments, "scriptArguments", []string{}, "scriptArguments that are needed to be passed to scripts. the scriptArguments list is a flat list and has a positional relationship to the `sources` param. For e.g. The scriptArguments string at position 1 will be considered as the argument(s) for script at position 1 in `sources` list. For multiple arguments for a script please add them as a comma seperated string.") + cmd.Flags().StringSliceVar(&stepConfig.ScriptArguments, "scriptArguments", []string{}, "scriptArguments that need to be passed to the scripts.") } diff --git a/resources/metadata/shellExecute.yaml b/resources/metadata/shellExecute.yaml index c5a2a24605..32dafb73b3 100644 --- a/resources/metadata/shellExecute.yaml +++ b/resources/metadata/shellExecute.yaml @@ -42,8 +42,18 @@ spec: - PARAMETERS - STAGES - STEPS - description: scriptArguments that are needed to be passed to scripts. the scriptArguments list is a flat list and has a positional relationship to the `sources` param. - For e.g. The scriptArguments string at position 1 will be considered as the argument(s) for script at position 1 in `sources` list. For multiple arguments for a script please add them as a comma seperated string. + description: "scriptArguments that need to be passed to the scripts." + longDescription: |- + The scriptArguments list is a flat list and has a positional relationship to the `sources` parameter. + + For example, the `scriptArguments` string at position 0 will be considered as the argument(s) for script at position 0 in `sources` list. + `--sources ".pipeline/firstScript.sh" --sources ".pipeline/secondScript.sh" --scriptArguments "$(first_script_arg)" --scriptArguments "$(second_script_arg)"` + + For multiple arguments for a particular script, please add them as a comma-separated string enclosed in additional quotes, e.g.: + `--sources ".pipeline/yourScript.sh" --scriptArguments "\"$(first_arg),$(second_arg)\""` + + For multiple scripts with multiple arguments per each script your command would look like: + `--sources ".pipeline/firstScript.sh" --sources ".pipeline/secondScript.sh" --scriptArguments "\"$(first_script_arg1),$(first_script_arg2)\"" --scriptArguments "\"$(second_script_arg1),$(second_script_arg2)\""` mandatory: false containers: - name: shell From b317b1d1ebccca3e5702d1b6451c9f74f88eb720 Mon Sep 17 00:00:00 2001 From: Adrien <99400874+hubadr@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:16:14 +0200 Subject: [PATCH 21/43] Add debug log for zip file content (#5152) Co-authored-by: thtri --- cmd/checkmarxOneExecuteScan.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/checkmarxOneExecuteScan.go b/cmd/checkmarxOneExecuteScan.go index d809304290..c457372c65 100644 --- a/cmd/checkmarxOneExecuteScan.go +++ b/cmd/checkmarxOneExecuteScan.go @@ -943,12 +943,15 @@ func (c *checkmarxOneExecuteScanHelper) zipFolder(source string, zipFile io.Writ return nil } + fileName := strings.TrimPrefix(path, baseDir) noMatch, err := c.isFileNotMatchingPattern(patterns, path, info, utils) if err != nil || noMatch { + if noMatch { + log.Entry().Debugf("Excluded %s", fileName) + } return err } - fileName := strings.TrimPrefix(path, baseDir) writer, err := archive.Create(fileName) if err != nil { return err @@ -960,6 +963,9 @@ func (c *checkmarxOneExecuteScanHelper) zipFolder(source string, zipFile io.Writ } defer file.Close() _, err = io.Copy(writer, file) + if err == nil { + log.Entry().Debugf("Zipped %s", fileName) + } fileCount++ return err }) From 769067e9960dde0021ecfa5bad099fbed48be1cf Mon Sep 17 00:00:00 2001 From: Dmitrii Pavlukhin Date: Fri, 18 Oct 2024 17:06:41 +0300 Subject: [PATCH 22/43] added-option-to-use-global-config-file (#5145) --- cmd/whitesourceExecuteScan.go | 1 + cmd/whitesourceExecuteScan_generated.go | 11 +++++++++++ pkg/whitesource/scanOptions.go | 7 ++++--- pkg/whitesource/scanUA.go | 7 +++++++ resources/metadata/whitesourceExecuteScan.yaml | 9 +++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/cmd/whitesourceExecuteScan.go b/cmd/whitesourceExecuteScan.go index 1987466d13..14772ec8f6 100644 --- a/cmd/whitesourceExecuteScan.go +++ b/cmd/whitesourceExecuteScan.go @@ -490,6 +490,7 @@ func wsScanOptions(config *ScanOptions) *ws.ScanOptions { AgentDownloadURL: config.AgentDownloadURL, AgentFileName: config.AgentFileName, ConfigFilePath: config.ConfigFilePath, + UseGlobalConfiguration: config.UseGlobalConfiguration, Includes: config.Includes, Excludes: config.Excludes, JreDownloadURL: config.JreDownloadURL, diff --git a/cmd/whitesourceExecuteScan_generated.go b/cmd/whitesourceExecuteScan_generated.go index 30f29a67f5..5456231fe9 100644 --- a/cmd/whitesourceExecuteScan_generated.go +++ b/cmd/whitesourceExecuteScan_generated.go @@ -33,6 +33,7 @@ type whitesourceExecuteScanOptions struct { BuildDescriptorFile string `json:"buildDescriptorFile,omitempty"` BuildTool string `json:"buildTool,omitempty"` ConfigFilePath string `json:"configFilePath,omitempty"` + UseGlobalConfiguration bool `json:"useGlobalConfiguration,omitempty"` ContainerRegistryPassword string `json:"containerRegistryPassword,omitempty"` ContainerRegistryUser string `json:"containerRegistryUser,omitempty"` CreateProductFromPipeline bool `json:"createProductFromPipeline,omitempty"` @@ -352,6 +353,7 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE cmd.Flags().StringVar(&stepConfig.BuildDescriptorFile, "buildDescriptorFile", os.Getenv("PIPER_buildDescriptorFile"), "Explicit path to the build descriptor file.") cmd.Flags().StringVar(&stepConfig.BuildTool, "buildTool", os.Getenv("PIPER_buildTool"), "Defines the tool which is used for building the artifact.") cmd.Flags().StringVar(&stepConfig.ConfigFilePath, "configFilePath", `./wss-unified-agent.config`, "Explicit path to the WhiteSource Unified Agent configuration file.") + cmd.Flags().BoolVar(&stepConfig.UseGlobalConfiguration, "useGlobalConfiguration", false, "The parameter is applicable for multi-module mend projects. If set to true, the configuration file will be used for all modules. Otherwise each module will require its own configuration file in the module folder.") cmd.Flags().StringVar(&stepConfig.ContainerRegistryPassword, "containerRegistryPassword", os.Getenv("PIPER_containerRegistryPassword"), "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment.") cmd.Flags().StringVar(&stepConfig.ContainerRegistryUser, "containerRegistryUser", os.Getenv("PIPER_containerRegistryUser"), "For `buildTool: docker`: Username for container registry access - typically provided by the CI/CD environment.") cmd.Flags().BoolVar(&stepConfig.CreateProductFromPipeline, "createProductFromPipeline", true, "Whether to create the related WhiteSource product on the fly based on the supplied pipeline configuration.") @@ -527,6 +529,15 @@ func whitesourceExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: `./wss-unified-agent.config`, }, + { + Name: "useGlobalConfiguration", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "GENERAL", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, { Name: "containerRegistryPassword", ResourceRef: []config.ResourceReference{ diff --git a/pkg/whitesource/scanOptions.go b/pkg/whitesource/scanOptions.go index 7a3bcb6645..beb6038d9c 100644 --- a/pkg/whitesource/scanOptions.go +++ b/pkg/whitesource/scanOptions.go @@ -31,9 +31,10 @@ type ScanOptions struct { DefaultNpmRegistry string NpmIncludeDevDependencies bool - AgentDownloadURL string - AgentFileName string - ConfigFilePath string + AgentDownloadURL string + AgentFileName string + ConfigFilePath string + UseGlobalConfiguration bool JreDownloadURL string diff --git a/pkg/whitesource/scanUA.go b/pkg/whitesource/scanUA.go index 54e84ddfb3..1e3e3fe01e 100644 --- a/pkg/whitesource/scanUA.go +++ b/pkg/whitesource/scanUA.go @@ -127,6 +127,13 @@ func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath st } } + if config.UseGlobalConfiguration { + config.ConfigFilePath, err = filepath.Abs(config.ConfigFilePath) + if err != nil { + return err + } + } + configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName, config.Verbose) if err != nil { return err diff --git a/resources/metadata/whitesourceExecuteScan.yaml b/resources/metadata/whitesourceExecuteScan.yaml index 95adcdd7fc..9b030ac5b4 100644 --- a/resources/metadata/whitesourceExecuteScan.yaml +++ b/resources/metadata/whitesourceExecuteScan.yaml @@ -128,6 +128,15 @@ spec: - STAGES - STEPS default: ./wss-unified-agent.config + - name: useGlobalConfiguration + type: bool + description: "The parameter is applicable for multi-module mend projects. If set to true, the configuration file will be used for all modules. Otherwise each module will require its own configuration file in the module folder." + scope: + - PARAMETERS + - GENERAL + - STAGES + - STEPS + default: false - name: containerRegistryPassword description: "For `buildTool: docker`: Password for container registry access - typically provided by the CI/CD environment." type: string From 4eb1756b54472c34cad9f4ecf389d74f826f36c2 Mon Sep 17 00:00:00 2001 From: maxcask Date: Mon, 21 Oct 2024 12:23:50 +0400 Subject: [PATCH 23/43] fix(Central Build): Fix handling legacy stage name for Jenkins pipelines (#5151) * add handle stageName * some improvements --------- Co-authored-by: maxcask Co-authored-by: Googlom Co-authored-by: Googlom <36107508+Googlom@users.noreply.github.com> --- pkg/config/evaluation.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/config/evaluation.go b/pkg/config/evaluation.go index 1775181a51..4482a2d296 100644 --- a/pkg/config/evaluation.go +++ b/pkg/config/evaluation.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" + "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/orchestrator" "github.com/SAP/jenkins-library/pkg/piperutils" ) @@ -39,6 +40,9 @@ func (r *RunConfigV1) evaluateConditionsV1(config *Config, utils piperutils.File // to also consider using the technical name. stageName := stage.DisplayName + // Central Build in Jenkins was renamed to Build. + handleLegacyStageNaming(config, currentOrchestrator, stageName) + // Check #1: Apply explicit activation/deactivation from config file (if any) // and then evaluate stepActive conditions runStep := make(map[string]bool, len(stage.Steps)) @@ -305,3 +309,23 @@ func anyOtherStepIsActive(targetStep string, runSteps map[string]bool) bool { return false } + +func handleLegacyStageNaming(c *Config, orchestrator, stageName string) { + if orchestrator == "Jenkins" && stageName == "Build" { + _, buildExists := c.Stages["Build"] + centralBuildStageConfig, centralBuildExists := c.Stages["Central Build"] + if buildExists && centralBuildExists { + log.Entry().Warnf("You have 2 entries for build stage in config.yml. " + + "Parameters defined under 'Central Build' are ignored. " + + "Please use only 'Build'") + return + } + + if centralBuildExists { + c.Stages["Build"] = centralBuildStageConfig + log.Entry().Warnf("You are using 'Central Build' stage in config.yml. " + + "Please move parameters under the 'Build' stage, " + + "since 'Central Build' will be removed in future releases") + } + } +} From 5c47be3f8fe0a483f9b1fd5b3b6a8684eb355aa6 Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:29:34 +0500 Subject: [PATCH 24/43] refactor(vault): Refactor vault package (#5148) * move to old package * go mod * remove old * refactor done * Update pkg/vault/oidc.go Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * commit suggestions Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * commit suggestions Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * commit suggestions --------- Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> --- cmd/vaultRotateSecretId.go | 8 +- go.mod | 13 +- go.sum | 46 +- integration/integration_vault_test.go | 33 +- pkg/config/vault.go | 13 +- pkg/vault/client.go | 480 +++++--------------- pkg/vault/helpers.go | 23 + pkg/vault/oidc.go | 54 +-- pkg/vault/oidc_test.go | 6 +- pkg/vault/vault.go | 286 ++++++++++++ pkg/vault/{client_test.go => vault_test.go} | 57 +-- 11 files changed, 541 insertions(+), 478 deletions(-) create mode 100644 pkg/vault/helpers.go create mode 100644 pkg/vault/vault.go rename pkg/vault/{client_test.go => vault_test.go} (89%) diff --git a/cmd/vaultRotateSecretId.go b/cmd/vaultRotateSecretId.go index c7d6602d93..0532350cf4 100644 --- a/cmd/vaultRotateSecretId.go +++ b/cmd/vaultRotateSecretId.go @@ -41,20 +41,22 @@ func (v vaultRotateSecretIDUtilsBundle) UpdateSecretInStore(config *vaultRotateS func vaultRotateSecretId(config vaultRotateSecretIdOptions, telemetryData *telemetry.CustomData) { - vaultConfig := &vault.Config{ + vaultConfig := &vault.ClientConfig{ Config: &api.Config{ Address: config.VaultServerURL, }, Namespace: config.VaultNamespace, + RoleID: GeneralConfig.VaultRoleID, + SecretID: GeneralConfig.VaultRoleSecretID, } - client, err := vault.NewClientWithAppRole(vaultConfig, GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID) + client, err := vault.NewClient(vaultConfig) if err != nil { log.Entry().WithError(err).Fatal("could not create Vault client") } defer client.MustRevokeToken() utils := vaultRotateSecretIDUtilsBundle{ - Client: &client, + Client: client, config: &config, updateFunc: writeVaultSecretIDToStore, } diff --git a/go.mod b/go.mod index 3fecf3bf95..35fa8e764a 100644 --- a/go.mod +++ b/go.mod @@ -32,8 +32,9 @@ require ( github.com/google/go-containerregistry v0.19.0 github.com/google/go-github/v45 v45.2.0 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-retryablehttp v0.7.2 - github.com/hashicorp/vault/api v1.9.2 + github.com/hashicorp/go-retryablehttp v0.7.7 + github.com/hashicorp/vault/api v1.15.0 + github.com/hashicorp/vault/api/auth/approle v0.8.0 github.com/iancoleman/orderedmap v0.2.0 github.com/imdario/mergo v0.3.15 github.com/influxdata/influxdb-client-go/v2 v2.13.0 @@ -76,7 +77,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecr v1.32.2 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -87,7 +88,7 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -152,7 +153,6 @@ require ( github.com/aws/smithy-go v1.20.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02 // indirect - github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/containerd v1.7.20 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect @@ -164,7 +164,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -190,7 +190,6 @@ require ( github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect diff --git a/go.sum b/go.sum index c95413fbec..3df2dfff02 100644 --- a/go.sum +++ b/go.sum @@ -196,10 +196,8 @@ github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02/go.mod h1:Ade+4 github.com/buildpacks/lifecycle v0.18.5 h1:lfoUX8jYCUZ2/Tr2AopaRjinqDivkNkcTChzysQTo00= github.com/buildpacks/lifecycle v0.18.5/go.mod h1:Kvuu9IWABPLXc6yHCMtbdmgrGEi7QEiVzi5GGtcAkW0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= -github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -287,9 +285,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= @@ -321,8 +318,8 @@ github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lK github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -565,14 +562,13 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= @@ -591,8 +587,10 @@ github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvH github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= -github.com/hashicorp/vault/api v1.9.2 h1:YjkZLJ7K3inKgMZ0wzCU9OHqc+UqMQyXsPXnf3Cl2as= -github.com/hashicorp/vault/api v1.9.2/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8= +github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA= +github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8= +github.com/hashicorp/vault/api/auth/approle v0.8.0 h1:FuVtWZ0xD6+wz1x0l5s0b4852RmVXQNEiKhVXt6lfQY= +github.com/hashicorp/vault/api/auth/approle v0.8.0/go.mod h1:NV7O9r5JUtNdVnqVZeMHva81AIdpG0WoIQohNt1VCPM= github.com/heroku/color v0.0.6 h1:UTFFMrmMLFcL3OweqP1lAdp8i1y/9oHqkeHjQ/b/Ny0= github.com/heroku/color v0.0.6/go.mod h1:ZBvOcx7cTF2QKOv4LbmoBtNl5uB17qWxGuzZrsi1wLU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -678,15 +676,11 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -889,7 +883,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -1015,7 +1008,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1102,7 +1094,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1158,7 +1149,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1181,10 +1171,7 @@ golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1196,7 +1183,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1206,8 +1192,6 @@ golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1222,8 +1206,6 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/integration/integration_vault_test.go b/integration/integration_vault_test.go index 0d29329a68..1be34d67c3 100644 --- a/integration/integration_vault_test.go +++ b/integration/integration_vault_test.go @@ -47,7 +47,7 @@ func TestVaultIntegrationGetSecret(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} // setup vault for testing secretData := SecretData{ "key1": "value1", @@ -55,7 +55,7 @@ func TestVaultIntegrationGetSecret(t *testing.T) { } setupVault(t, config, testToken, secretData) - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) secret, err := client.GetKvSecret("secret/test") assert.NoError(t, err) @@ -93,14 +93,14 @@ func TestVaultIntegrationWriteSecret(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} // setup vault for testing secretData := map[string]string{ "key1": "value1", "key2": "value2", } - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) err = client.WriteKvSecret("secret/test", secretData) @@ -159,16 +159,17 @@ func TestVaultIntegrationAppRole(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} secretIDMetadata := map[string]interface{}{ "field1": "value1", } roleID, secretID := setupVaultAppRole(t, config, testToken, appRolePath, secretIDMetadata) - + config.RoleID = roleID + config.SecretID = secretID t.Run("Test Vault AppRole login", func(t *testing.T) { - client, err := vault.NewClientWithAppRole(config, roleID, secretID) + client, err := vault.NewClient(config) assert.NoError(t, err) secret, err := client.GetSecret("auth/token/lookup-self") meta := secret.Data["meta"].(SecretData) @@ -178,7 +179,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { }) t.Run("Test Vault AppRoleTTL Fetch", func(t *testing.T) { - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) ttl, err := client.GetAppRoleSecretIDTtl(secretID, appRoleName) assert.NoError(t, err) @@ -186,7 +187,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { }) t.Run("Test Vault AppRole Rotation", func(t *testing.T) { - client, err := vault.NewClient(config, testToken) + client, err := vault.NewClientWithToken(config, testToken) assert.NoError(t, err) newSecretID, err := client.GenerateNewAppRoleSecret(secretID, appRoleName) assert.NoError(t, err) @@ -194,7 +195,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { assert.NotEqual(t, secretID, newSecretID) // verify metadata is not broken - client, err = vault.NewClientWithAppRole(config, roleID, newSecretID) + client, err = vault.NewClient(config) assert.NoError(t, err) secret, err := client.GetSecret("auth/token/lookup-self") meta := secret.Data["meta"].(SecretData) @@ -204,7 +205,7 @@ func TestVaultIntegrationAppRole(t *testing.T) { }) t.Run("Test Fetching RoleName from vault", func(t *testing.T) { - client, err := vault.NewClientWithAppRole(config, roleID, secretID) + client, err := vault.NewClient(config) assert.NoError(t, err) fetchedRoleName, err := client.GetAppRoleName() assert.NoError(t, err) @@ -238,16 +239,18 @@ func TestVaultIntegrationTokenRevocation(t *testing.T) { assert.NoError(t, err) port, err := vaultContainer.MappedPort(ctx, "8200") host := fmt.Sprintf("http://%s:%s", ip, port.Port()) - config := &vault.Config{Config: &api.Config{Address: host}} + config := &vault.ClientConfig{Config: &api.Config{Address: host}} secretIDMetadata := map[string]interface{}{ "field1": "value1", } roleID, secretID := setupVaultAppRole(t, config, testToken, appRolePath, secretIDMetadata) + config.RoleID = roleID + config.SecretID = secretID t.Run("Test Revocation works", func(t *testing.T) { - client, err := vault.NewClientWithAppRole(config, roleID, secretID) + client, err := vault.NewClient(config) assert.NoError(t, err) secret, err := client.GetSecret("auth/token/lookup-self") meta := secret.Data["meta"].(SecretData) @@ -263,7 +266,7 @@ func TestVaultIntegrationTokenRevocation(t *testing.T) { }) } -func setupVaultAppRole(t *testing.T, config *vault.Config, token, appRolePath string, metadata map[string]interface{}) (string, string) { +func setupVaultAppRole(t *testing.T, config *vault.ClientConfig, token, appRolePath string, metadata map[string]interface{}) (string, string) { t.Helper() client, err := api.NewClient(config.Config) assert.NoError(t, err) @@ -302,7 +305,7 @@ func setupVaultAppRole(t *testing.T, config *vault.Config, token, appRolePath st return roleID.(string), secretID.(string) } -func setupVault(t *testing.T, config *vault.Config, token string, secret SecretData) { +func setupVault(t *testing.T, config *vault.ClientConfig, token string, secret SecretData) { t.Helper() client, err := api.NewClient(config.Config) assert.NoError(t, err) diff --git a/pkg/config/vault.go b/pkg/config/vault.go index db53761039..c85173d351 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -87,6 +87,7 @@ var globalVaultClient *vault.Client func GlobalVaultClient() VaultClient { // an interface containing a nil pointer is considered non-nil in Go + // It is nil if Vault is not configured if globalVaultClient == nil { return nil } @@ -122,15 +123,17 @@ func GetVaultClientFromConfig(config map[string]interface{}, creds VaultCredenti namespace = config["vaultNamespace"].(string) log.Entry().Debugf(" with namespace %s", namespace) } - var client vault.Client + var client *vault.Client var err error - clientConfig := &vault.Config{Config: &api.Config{Address: address}, Namespace: namespace} + clientConfig := &vault.ClientConfig{Config: &api.Config{Address: address}, Namespace: namespace} if creds.VaultToken != "" { log.Entry().Debugf(" with Token authentication") - client, err = vault.NewClient(clientConfig, creds.VaultToken) + client, err = vault.NewClientWithToken(clientConfig, creds.VaultToken) } else { log.Entry().Debugf(" with AppRole authentication") - client, err = vault.NewClientWithAppRole(clientConfig, creds.AppRoleID, creds.AppRoleSecretID) + clientConfig.RoleID = creds.AppRoleID + clientConfig.SecretID = creds.AppRoleSecretID + client, err = vault.NewClient(clientConfig) } if err != nil { log.Entry().Info(" failed") @@ -138,7 +141,7 @@ func GetVaultClientFromConfig(config map[string]interface{}, creds VaultCredenti } // Set global vault client for usage in steps - globalVaultClient = &client + globalVaultClient = client log.Entry().Info(" succeeded") return client, nil diff --git a/pkg/vault/client.go b/pkg/vault/client.go index ba1e14e929..659d94c205 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -2,423 +2,195 @@ package vault import ( "context" - "encoding/json" "fmt" + "github.com/SAP/jenkins-library/pkg/log" + vaultAPI "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/api/auth/approle" + "github.com/pkg/errors" "io" "net/http" - "path" - "strconv" "strings" "time" - - "github.com/SAP/jenkins-library/pkg/log" - "github.com/hashicorp/vault/api" ) // Client handles communication with Vault type Client struct { - lClient logicalClient - config *Config + vaultApiClient *vaultAPI.Client + logical logicalClient + cfg *ClientConfig } -// Config contains the vault client configuration -type Config struct { - *api.Config - AppRoleMountPoint string +type ClientConfig struct { + *vaultAPI.Config Namespace string + AppRoleMountPoint string + RoleID string + SecretID string } // logicalClient interface for mocking type logicalClient interface { - Read(string) (*api.Secret, error) - Write(string, map[string]interface{}) (*api.Secret, error) + Read(string) (*vaultAPI.Secret, error) + Write(string, map[string]interface{}) (*vaultAPI.Secret, error) } -type VaultCredentials struct { - AppRoleID string - AppRoleSecretID string - VaultToken string -} - -// NewClient instantiates a Client and sets the specified token -func NewClient(config *Config, token string) (Client, error) { - if config == nil { - config = &Config{Config: api.DefaultConfig()} - } - client, err := api.NewClient(config.Config) - if err != nil { - return Client{}, err - } - if config.Namespace != "" { - client.SetNamespace(config.Namespace) +func newClient(cfg *ClientConfig) (*Client, error) { + if cfg == nil { + cfg = &ClientConfig{Config: vaultAPI.DefaultConfig()} } - client.SetToken(token) - return Client{client.Logical(), config}, nil -} -// NewClientWithAppRole instantiates a new client and obtains a token via the AppRole auth method -func NewClientWithAppRole(config *Config, roleID, secretID string) (Client, error) { - if config == nil { - config = &Config{Config: api.DefaultConfig()} - } - if config.AppRoleMountPoint == "" { - config.AppRoleMountPoint = "auth/approle" - } - client, err := api.NewClient(config.Config) + var err error + c := &Client{cfg: cfg} + c.vaultApiClient, err = vaultAPI.NewClient(cfg.Config) if err != nil { - return Client{}, err - } - - client.SetMinRetryWait(time.Second * 5) - client.SetMaxRetryWait(time.Second * 90) - client.SetMaxRetries(3) - client.SetCheckRetry(func(ctx context.Context, resp *http.Response, err error) (bool, error) { - if resp != nil { - log.Entry().Debugln("Vault response: ", resp.Status, resp.StatusCode, err) - } else { - log.Entry().Debugln("Vault response: ", err) - } - - isEOF := false - if err != nil && strings.Contains(err.Error(), "EOF") { - log.Entry().Infoln("isEOF is true") - isEOF = true - } - - if err == io.EOF { - log.Entry().Infoln("err = io.EOF is true") - } - - retry, err := api.DefaultRetryPolicy(ctx, resp, err) - - if err != nil || err == io.EOF || isEOF || retry { - log.Entry().Infoln("Retrying vault request...") - return true, nil - } - return false, nil - }) - - if config.Namespace != "" { - client.SetNamespace(config.Namespace) - } - - result, err := client.Logical().Write(path.Join(config.AppRoleMountPoint, "/login"), map[string]interface{}{ - "role_id": roleID, - "secret_id": secretID, - }) - if err != nil { - return Client{}, err + return nil, err } - - authInfo := result.Auth - if authInfo == nil || authInfo.ClientToken == "" { - return Client{}, fmt.Errorf("Could not obtain token from approle with role_id %s", roleID) + c.logical = c.vaultApiClient.Logical() + if cfg.Namespace != "" { + c.vaultApiClient.SetNamespace(cfg.Namespace) } - return NewClient(config, authInfo.ClientToken) + return c, nil } -// GetSecret uses the given path to fetch a secret from vault -func (v Client) GetSecret(path string) (*api.Secret, error) { - path = sanitizePath(path) - c := v.lClient - - secret, err := c.Read(path) +func NewClient(cfg *ClientConfig) (*Client, error) { + c, err := newClient(cfg) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vault client initialization failed") } + applyApiClientRetryConfiguration(c.vaultApiClient) + + initialLoginDone := make(chan struct{}) + go c.startTokenLifecycleManager(initialLoginDone) // this goroutine ends with main goroutine + // wait for initial login or a failure + <-initialLoginDone - return secret, nil + // In case of a failure, the function returns an unauthorized client, which will cause subsequent requests to fail. + return c, nil } -// GetKvSecret reads secret from the KV engine. -// It Automatically transforms the logical path to the HTTP API Path for the corresponding KV Engine version -func (v Client) GetKvSecret(path string) (map[string]string, error) { - path = sanitizePath(path) - mountpath, version, err := v.getKvInfo(path) +func NewClientWithToken(cfg *ClientConfig, token string) (*Client, error) { + c, err := newClient(cfg) if err != nil { - return nil, err - } - if version == 2 { - path = addPrefixToKvPath(path, mountpath, "data") - } else if version != 1 { - return nil, fmt.Errorf("KV Engine in version %d is currently not supported", version) + return nil, errors.Wrap(err, "vault client initialization failed") } - secret, err := v.GetSecret(path) - if secret == nil || err != nil { - return nil, err + c.vaultApiClient.SetToken(token) + return c, nil +} - } - var rawData interface{} - switch version { - case 1: - rawData = secret.Data - case 2: - var ok bool - rawData, ok = secret.Data["data"] - if !ok { - return nil, fmt.Errorf("Missing 'data' field in response: %v", rawData) +func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { + defer func() { + // make sure to close channel to avoid blocking of the caller + _, open := <-initialLoginDone + if open { + close(initialLoginDone) } - } - - data, ok := rawData.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("Excpected 'data' field to be a map[string]interface{} but got %T instead", rawData) - } - - secretData := make(map[string]string, len(data)) - for k, v := range data { - switch t := v.(type) { - case string: - secretData[k] = t - case int: - secretData[k] = fmt.Sprintf("%d", t) - default: - jsonBytes, err := json.Marshal(t) - if err != nil { - log.Entry().Warnf("failed to parse Vault secret key %q, error: %s", k, err.Error()) - continue - } - - secretData[k] = string(jsonBytes) + }() + + initialLoginSucceed := false + for i := 0; i < c.vaultApiClient.MaxRetries(); i++ { + vaultLoginResp, err := c.login() + if err != nil { + log.Entry().Errorf("unable to authenticate to Vault: %v", err) + continue + } + if !initialLoginSucceed { + initialLoginDone <- struct{}{} + close(initialLoginDone) + initialLoginSucceed = true } - } - return secretData, nil -} -// WriteKvSecret writes secret to kv engine -func (v Client) WriteKvSecret(path string, newSecret map[string]string) error { - oldSecret, err := v.GetKvSecret(path) - if err != nil { - return err - } - secret := make(map[string]interface{}, len(oldSecret)) - for k, v := range oldSecret { - secret[k] = v - } - for k, v := range newSecret { - secret[k] = v - } - path = sanitizePath(path) - mountpath, version, err := v.getKvInfo(path) - if err != nil { - return err - } - if version == 2 { - path = addPrefixToKvPath(path, mountpath, "data") - secret = map[string]interface{}{"data": secret} - } else if version != 1 { - return fmt.Errorf("KV Engine in version %d is currently not supported", version) + tokenErr := c.manageTokenLifecycle(vaultLoginResp) + if tokenErr != nil { + log.Entry().Errorf("unable to start managing token lifecycle: %v", err) + continue + } } - - _, err = v.lClient.Write(path, secret) - return err } -// GenerateNewAppRoleSecret creates a new secret-id -func (v *Client) GenerateNewAppRoleSecret(secretID, appRoleName string) (string, error) { - appRolePath := v.getAppRolePath(appRoleName) - secretIDData, err := v.lookupSecretID(secretID, appRolePath) - if err != nil { - return "", err +// Starts token lifecycle management. Returns only fatal errors as errors, +// otherwise returns nil, so we can attempt login again. +func (c *Client) manageTokenLifecycle(authResp *vaultAPI.Secret) error { + if !authResp.Auth.Renewable { + log.Entry().Debugf("Token is not configured to be renewable. Re-attempting login.") + return nil } - reqPath := sanitizePath(path.Join(appRolePath, "/secret-id")) - - // we preserve metadata which was attached to the secret-id - json, err := json.Marshal(secretIDData["metadata"]) + watcher, err := c.vaultApiClient.NewLifetimeWatcher(&vaultAPI.LifetimeWatcherInput{Secret: authResp}) if err != nil { - return "", err + return fmt.Errorf("unable to initialize new lifetime watcher for renewing auth token: %w", err) } - secret, err := v.lClient.Write(reqPath, map[string]interface{}{ - "metadata": string(json), - }) - if err != nil { - return "", err - } - - if secret == nil || secret.Data == nil { - return "", fmt.Errorf("Could not generate new approle secret-id for approle path %s", reqPath) - } + go watcher.Start() + defer watcher.Stop() - secretIDRaw, ok := secret.Data["secret_id"] - if !ok { - return "", fmt.Errorf("Vault response for path %s did not contain a new secret-id", reqPath) - } + for { + select { + // `DoneCh` will return if renewal fails, or if the remaining lease + // duration is under a built-in threshold and either renewing is not + // extending it or renewing is disabled. In any case, the caller + // needs to attempt to log in again. + case err := <-watcher.DoneCh(): + if err != nil { + log.Entry().Printf("Failed to renew Vault token: %v. Re-attempting login.", err) + return nil + } + // This occurs once the token has reached max TTL. + log.Entry().Printf("Token can no longer be renewed. Re-attempting login.") + return nil - newSecretID, ok := secretIDRaw.(string) - if !ok { - return "", fmt.Errorf("New secret-id from approle path %s has an unexpected type %T expected 'string'", reqPath, secretIDRaw) + // Successfully completed renewal + case <-watcher.RenewCh(): + log.Entry().Printf("Vault token successfully renewed") + } } - - return newSecretID, nil } -// GetAppRoleSecretIDTtl returns the remaining time until the given secret-id expires -func (v *Client) GetAppRoleSecretIDTtl(secretID, roleName string) (time.Duration, error) { - appRolePath := v.getAppRolePath(roleName) - data, err := v.lookupSecretID(secretID, appRolePath) - if err != nil { - return 0, err - } - - if data == nil || data["expiration_time"] == nil { - return 0, fmt.Errorf("Could not load secret-id information from path %s", appRolePath) - } - - expiration, ok := data["expiration_time"].(string) - if !ok || expiration == "" { - return 0, fmt.Errorf("Could not handle get expiration time for secret-id from path %s", appRolePath) - } - - expirationDate, err := time.Parse(time.RFC3339, expiration) - +func (c *Client) login() (*vaultAPI.Secret, error) { + appRoleAuth, err := approle.NewAppRoleAuth(c.cfg.RoleID, &approle.SecretID{FromString: c.cfg.SecretID}) if err != nil { - return 0, err - } - - ttl := expirationDate.Sub(time.Now()) - if ttl < 0 { - return 0, nil - } - - return ttl, nil -} - -// RevokeToken revokes the token which is currently used. -// The client can't be used anymore after this function was called. -func (v Client) RevokeToken() error { - _, err := v.lClient.Write("auth/token/revoke-self", map[string]interface{}{}) - return err -} - -// MustRevokeToken same as RevokeToken but the programm is terminated with an error if this fails. -// Should be used in defer statements only. -func (v Client) MustRevokeToken() { - if err := v.RevokeToken(); err != nil { - log.Entry().WithError(err).Fatal("Could not revoke token") + return nil, fmt.Errorf("unable to initialize appRole auth method: %w", err) } -} -// GetAppRoleName returns the AppRole role name which was used to authenticate. -// Returns "" when AppRole authentication wasn't used -func (v *Client) GetAppRoleName() (string, error) { - const lookupPath = "auth/token/lookup-self" - secret, err := v.GetSecret(lookupPath) + authInfo, err := c.vaultApiClient.Auth().Login(context.Background(), appRoleAuth) if err != nil { - return "", err - } - - if secret.Data == nil { - return "", fmt.Errorf("Could not lookup token information: %s", lookupPath) - } - - meta, ok := secret.Data["meta"] - - if !ok { - return "", fmt.Errorf("Token info did not contain metadata %s", lookupPath) - } - - metaMap, ok := meta.(map[string]interface{}) - - if !ok { - return "", fmt.Errorf("Token info field 'meta' is not a map: %s", lookupPath) - } - - roleName := metaMap["role_name"] - - if roleName == nil { - return "", nil + return nil, fmt.Errorf("unable to login to appRole auth method: %w", err) } - - roleNameStr, ok := roleName.(string) - if !ok { - // when approle authentication is not used vault admins can use the role_name field with other type - // so no error in this case - return "", nil - } - - return roleNameStr, nil -} - -// SetAppRoleMountPoint sets the path under which the approle auth backend is mounted -func (v *Client) SetAppRoleMountPoint(appRoleMountpoint string) { - v.config.AppRoleMountPoint = appRoleMountpoint -} - -func (v *Client) getAppRolePath(roleName string) string { - appRoleMountPoint := v.config.AppRoleMountPoint - if appRoleMountPoint == "" { - appRoleMountPoint = "auth/approle" + if authInfo == nil { + return nil, fmt.Errorf("no auth info was returned after login") } - return path.Join(appRoleMountPoint, "role", roleName) -} -func sanitizePath(path string) string { - path = strings.TrimSpace(path) - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, "/") - return path + return authInfo, nil } -func addPrefixToKvPath(p, mountPath, apiPrefix string) string { - switch { - case p == mountPath, p == strings.TrimSuffix(mountPath, "/"): - return path.Join(mountPath, apiPrefix) - default: - p = strings.TrimPrefix(p, mountPath) - return path.Join(mountPath, apiPrefix, p) - } -} - -func (v *Client) getKvInfo(path string) (string, int, error) { - secret, err := v.GetSecret("sys/internal/ui/mounts/" + path) - if err != nil { - return "", 0, err - } - - if secret == nil { - return "", 0, fmt.Errorf("Failed to get version and engine mountpoint for path: %s", path) - } - - var mountPath string - if mountPathRaw, ok := secret.Data["path"]; ok { - mountPath = mountPathRaw.(string) - } - - options := secret.Data["options"] - if options == nil { - return mountPath, 1, nil - } - - versionRaw := options.(map[string]interface{})["version"] - if versionRaw == nil { - return mountPath, 1, nil - } +func applyApiClientRetryConfiguration(vaultApiClient *vaultAPI.Client) { + vaultApiClient.SetMinRetryWait(time.Second * 5) + vaultApiClient.SetMaxRetryWait(time.Second * 90) + vaultApiClient.SetMaxRetries(3) + vaultApiClient.SetCheckRetry(func(ctx context.Context, resp *http.Response, err error) (bool, error) { + if resp != nil { + log.Entry().Debugln("Vault response: ", resp.Status, resp.StatusCode, err) + } else { + log.Entry().Debugln("Vault response: ", err) + } - version := versionRaw.(string) - if version == "" { - return mountPath, 1, nil - } + isEOF := false + if err != nil && strings.Contains(err.Error(), "EOF") { + log.Entry().Infoln("isEOF is true") + isEOF = true + } - vNumber, err := strconv.Atoi(version) - if err != nil { - return mountPath, 0, err - } + if err == io.EOF { + log.Entry().Infoln("err = io.EOF is true") + } - return mountPath, vNumber, nil -} + retry, err := vaultAPI.DefaultRetryPolicy(ctx, resp, err) -func (v *Client) lookupSecretID(secretID, appRolePath string) (map[string]interface{}, error) { - reqPath := sanitizePath(path.Join(appRolePath, "/secret-id/lookup")) - secret, err := v.lClient.Write(reqPath, map[string]interface{}{ - "secret_id": secretID, + if err != nil || err == io.EOF || isEOF || retry { + log.Entry().Infoln("Retrying vault request...") + return true, nil + } + return false, nil }) - if err != nil { - return nil, err - } - - return secret.Data, nil } diff --git a/pkg/vault/helpers.go b/pkg/vault/helpers.go new file mode 100644 index 0000000000..1c9db72cbb --- /dev/null +++ b/pkg/vault/helpers.go @@ -0,0 +1,23 @@ +package vault + +import ( + "path" + "strings" +) + +func sanitizePath(path string) string { + path = strings.TrimSpace(path) + path = strings.TrimPrefix(path, "/") + path = strings.TrimSuffix(path, "/") + return path +} + +func addPrefixToKvPath(p, mountPath, apiPrefix string) string { + switch { + case p == mountPath, p == strings.TrimSuffix(mountPath, "/"): + return path.Join(mountPath, apiPrefix) + default: + p = strings.TrimPrefix(p, mountPath) + return path.Join(mountPath, apiPrefix, p) + } +} diff --git a/pkg/vault/oidc.go b/pkg/vault/oidc.go index 493050d04d..972084b59b 100644 --- a/pkg/vault/oidc.go +++ b/pkg/vault/oidc.go @@ -13,39 +13,47 @@ import ( "github.com/pkg/errors" ) -type JwtPayload struct { +type jwtPayload struct { Expire int64 `json:"exp"` } // getOIDCToken returns the generated OIDC token and sets it in the env -func (v Client) getOIDCToken(roleID string) (string, error) { +func (c *Client) getOIDCToken(roleID string) (string, error) { oidcPath := sanitizePath(path.Join("identity/oidc/token/", roleID)) - c := v.lClient - jwt, err := c.Read(oidcPath) + jwt, err := c.logical.Read(oidcPath) if err != nil { return "", err } token := jwt.Data["token"].(string) + if token == "" { + return "", fmt.Errorf("received an empty token") + } + log.RegisterSecret(token) os.Setenv("PIPER_OIDCIdentityToken", token) return token, nil } -// getJWTTokenPayload returns the payload of the JWT token using base64 decoding -func getJWTTokenPayload(token string) ([]byte, error) { +// getJWTPayload returns the payload of the JWT token using base64 decoding +func getJWTPayload(token string) (*jwtPayload, error) { parts := strings.Split(token, ".") - if len(parts) >= 2 { - substr := parts[1] - decodedBytes, err := base64.RawStdEncoding.DecodeString(substr) - if err != nil { - return nil, errors.Wrap(err, "JWT payload couldn't be decoded: %s") - } - return decodedBytes, nil + if len(parts) < 2 { + return nil, fmt.Errorf("not a valid JWT token") + } + + decodedBytes, err := base64.RawStdEncoding.DecodeString(parts[1]) + if err != nil { + return nil, errors.Wrap(err, "JWT payload couldn't be decoded: %s") + } + + var payload jwtPayload + if err = json.Unmarshal(decodedBytes, &payload); err != nil { + return nil, errors.Wrap(err, "JWT unmarshal failed") } - return nil, fmt.Errorf("Not a valid JWT token") + return &payload, nil } func oidcTokenIsValid(token string) bool { @@ -53,34 +61,28 @@ func oidcTokenIsValid(token string) bool { return false } - payload, err := getJWTTokenPayload(token) + jwtTokenPayload, err := getJWTPayload(token) if err != nil { log.Entry().Debugf("OIDC token couldn't be validated: %s", err) return false } - var jwtPayload JwtPayload - err = json.Unmarshal(payload, &jwtPayload) - if err != nil { - log.Entry().Debugf("OIDC token couldn't be validated: %s", err) - return false - } - - expiryTime := time.Unix(jwtPayload.Expire, 0) + expiryTime := time.Unix(jwtTokenPayload.Expire, 0) currentTime := time.Now() return expiryTime.After(currentTime) } // GetOIDCTokenByValidation returns the token if token is expired then get a new token else return old token -func (v Client) GetOIDCTokenByValidation(roleID string) (string, error) { +func (c *Client) GetOIDCTokenByValidation(roleID string) (string, error) { token := os.Getenv("PIPER_OIDCIdentityToken") if oidcTokenIsValid(token) { return token, nil } - token, err := v.getOIDCToken(roleID) - if token == "" || err != nil { + log.Entry().Debug("obtaining new OIDC token") + token, err := c.getOIDCToken(roleID) + if err != nil { return "", err } diff --git a/pkg/vault/oidc_test.go b/pkg/vault/oidc_test.go index 2e1a973305..7547a54ee8 100644 --- a/pkg/vault/oidc_test.go +++ b/pkg/vault/oidc_test.go @@ -28,7 +28,7 @@ func TestOIDC(t *testing.T) { // init vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", oidcPath).Return(mockJwt, nil) // run @@ -50,7 +50,7 @@ func TestOIDC(t *testing.T) { t.Setenv("PIPER_OIDCIdentityToken", token) vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", oidcPath).Return(mockJwt, nil) // run @@ -72,7 +72,7 @@ func TestOIDC(t *testing.T) { t.Setenv("PIPER_OIDCIdentityToken", token) vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", oidcPath).Return(mockJwt, nil) // run diff --git a/pkg/vault/vault.go b/pkg/vault/vault.go new file mode 100644 index 0000000000..5943b31c19 --- /dev/null +++ b/pkg/vault/vault.go @@ -0,0 +1,286 @@ +package vault + +import ( + "encoding/json" + "fmt" + "github.com/SAP/jenkins-library/pkg/log" + "github.com/hashicorp/vault/api" + "path" + "strconv" + "time" +) + +// GetSecret uses the given path to fetch a secret from vault +func (c *Client) GetSecret(path string) (*api.Secret, error) { + return c.logical.Read(sanitizePath(path)) +} + +// GetKvSecret reads secret from the KV engine. +// It Automatically transforms the logical path to the HTTP API Path for the corresponding KV Engine version +func (c *Client) GetKvSecret(path string) (map[string]string, error) { + path = sanitizePath(path) + mountPath, version, err := c.getKvInfo(path) + if err != nil { + return nil, err + } + if version == 2 { + path = addPrefixToKvPath(path, mountPath, "data") + } else if version != 1 { + return nil, fmt.Errorf("KV Engine in version %d is currently not supported", version) + } + + secret, err := c.GetSecret(path) + if secret == nil || err != nil { + return nil, err + + } + var rawData interface{} + switch version { + case 1: + rawData = secret.Data + case 2: + var ok bool + rawData, ok = secret.Data["data"] + if !ok { + return nil, fmt.Errorf("missing 'data' field in response: %v", rawData) + } + } + + data, ok := rawData.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("excpected 'data' field to be a map[string]interface{} but got %T instead", rawData) + } + + secretData := make(map[string]string, len(data)) + for k, v := range data { + switch t := v.(type) { + case string: + secretData[k] = t + case int: + secretData[k] = fmt.Sprintf("%d", t) + default: + jsonBytes, err := json.Marshal(t) + if err != nil { + log.Entry().Warnf("failed to parse Vault secret key %q, error: %s", k, err.Error()) + continue + } + + secretData[k] = string(jsonBytes) + } + } + return secretData, nil +} + +// WriteKvSecret writes secret to kv engine +func (c *Client) WriteKvSecret(path string, newSecret map[string]string) error { + oldSecret, err := c.GetKvSecret(path) + if err != nil { + return err + } + secret := make(map[string]interface{}, len(oldSecret)) + for k, v := range oldSecret { + secret[k] = v + } + for k, v := range newSecret { + secret[k] = v + } + path = sanitizePath(path) + mountPath, version, err := c.getKvInfo(path) + if err != nil { + return err + } + if version == 2 { + path = addPrefixToKvPath(path, mountPath, "data") + secret = map[string]interface{}{"data": secret} + } else if version != 1 { + return fmt.Errorf("KV Engine in version %d is currently not supported", version) + } + + _, err = c.logical.Write(path, secret) + return err +} + +// GenerateNewAppRoleSecret creates a new secret-id +func (c *Client) GenerateNewAppRoleSecret(secretID, appRoleName string) (string, error) { + appRolePath := c.getAppRolePath(appRoleName) + secretIDData, err := c.lookupSecretID(secretID, appRolePath) + if err != nil { + return "", err + } + + reqPath := sanitizePath(path.Join(appRolePath, "/secret-id")) + + // we preserve metadata which was attached to the secret-id + jsonBytes, err := json.Marshal(secretIDData["metadata"]) + if err != nil { + return "", err + } + + secret, err := c.logical.Write(reqPath, map[string]interface{}{ + "metadata": string(jsonBytes), + }) + if err != nil { + return "", err + } + + if secret == nil || secret.Data == nil { + return "", fmt.Errorf("could not generate new approle secret-id for approle path %s", reqPath) + } + + secretIDRaw, ok := secret.Data["secret_id"] + if !ok { + return "", fmt.Errorf("Vault response for path %s did not contain a new secret-id", reqPath) + } + + newSecretID, ok := secretIDRaw.(string) + if !ok { + return "", fmt.Errorf("new secret-id from approle path %s has an unexpected type %T expected 'string'", reqPath, secretIDRaw) + } + + return newSecretID, nil +} + +// GetAppRoleSecretIDTtl returns the remaining time until the given secret-id expires +func (c *Client) GetAppRoleSecretIDTtl(secretID, roleName string) (time.Duration, error) { + appRolePath := c.getAppRolePath(roleName) + data, err := c.lookupSecretID(secretID, appRolePath) + if err != nil { + return 0, err + } + + if data == nil || data["expiration_time"] == nil { + return 0, fmt.Errorf("could not load secret-id information from path %s", appRolePath) + } + + expiration, ok := data["expiration_time"].(string) + if !ok || expiration == "" { + return 0, fmt.Errorf("could not handle get expiration time for secret-id from path %s", appRolePath) + } + + expirationDate, err := time.Parse(time.RFC3339, expiration) + + if err != nil { + return 0, err + } + + ttl := expirationDate.Sub(time.Now()) + if ttl < 0 { + return 0, nil + } + + return ttl, nil +} + +// RevokeToken revokes the token which is currently used. +// The client can't be used anymore after this function was called. +func (c *Client) RevokeToken() error { + _, err := c.logical.Write("auth/token/revoke-self", map[string]interface{}{}) + return err +} + +// MustRevokeToken same as RevokeToken but the program is terminated with an error if this fails. +// Should be used in defer statements only. +func (c *Client) MustRevokeToken() { + if err := c.RevokeToken(); err != nil { + log.Entry().WithError(err).Fatal("Could not revoke token") + } +} + +// GetAppRoleName returns the AppRole role name which was used to authenticate. +// Returns "" when AppRole authentication wasn't used +func (c *Client) GetAppRoleName() (string, error) { + const lookupPath = "auth/token/lookup-self" + secret, err := c.GetSecret(lookupPath) + if err != nil { + return "", err + } + + if secret.Data == nil { + return "", fmt.Errorf("could not lookup token information: %s", lookupPath) + } + + meta, ok := secret.Data["meta"] + + if !ok { + return "", fmt.Errorf("token info did not contain metadata %s", lookupPath) + } + + metaMap, ok := meta.(map[string]interface{}) + + if !ok { + return "", fmt.Errorf("token info field 'meta' is not a map: %s", lookupPath) + } + + roleName := metaMap["role_name"] + + if roleName == nil { + return "", nil + } + + roleNameStr, ok := roleName.(string) + if !ok { + // when AppRole authentication is not used vault admins can use the role_name field with other type + // so no error in this case + return "", nil + } + + return roleNameStr, nil +} + +func (c *Client) getAppRolePath(roleName string) string { + appRoleMountPoint := c.cfg.AppRoleMountPoint + if appRoleMountPoint == "" { + appRoleMountPoint = "auth/approle" + } + return path.Join(appRoleMountPoint, "role", roleName) +} + +func (c *Client) getKvInfo(path string) (string, int, error) { + secret, err := c.GetSecret("sys/internal/ui/mounts/" + path) + if err != nil { + return "", 0, err + } + + if secret == nil { + return "", 0, fmt.Errorf("failed to get version and engine mountpoint for path: %s", path) + } + + var mountPath string + if mountPathRaw, ok := secret.Data["path"]; ok { + mountPath = mountPathRaw.(string) + } + + options := secret.Data["options"] + if options == nil { + return mountPath, 1, nil + } + + versionRaw := options.(map[string]interface{})["version"] + if versionRaw == nil { + return mountPath, 1, nil + } + + version := versionRaw.(string) + if version == "" { + return mountPath, 1, nil + } + + vNumber, err := strconv.Atoi(version) + if err != nil { + return mountPath, 0, err + } + + return mountPath, vNumber, nil +} + +func (c *Client) lookupSecretID(secretID, appRolePath string) (map[string]interface{}, error) { + reqPath := sanitizePath(path.Join(appRolePath, "/secret-id/lookup")) + secret, err := c.logical.Write(reqPath, map[string]interface{}{ + "secret_id": secretID, + }) + if err != nil { + return nil, err + } + + return secret.Data, nil +} diff --git a/pkg/vault/client_test.go b/pkg/vault/vault_test.go similarity index 89% rename from pkg/vault/client_test.go rename to pkg/vault/vault_test.go index 90a86b35bb..a71bfe9ea0 100644 --- a/pkg/vault/client_test.go +++ b/pkg/vault/vault_test.go @@ -30,7 +30,7 @@ const ( func TestGetKV2Secret(t *testing.T) { t.Run("Test missing secret", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} setupMockKvV2(vaultMock) vaultMock.On("Read", "secret/data/notexist").Return(nil, nil) secret, err := client.GetKvSecret("secret/notexist") @@ -45,7 +45,7 @@ func TestGetKV2Secret(t *testing.T) { t.Run("Getting secret from KV engine (v2)", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV2(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", secretAPIPath).Return(kv2Secret(SecretData{"key1": "value1", "key2": map[string]any{"subkey1": "subvalue2"}, "key3": 3, "key4": []string{"foo", "bar"}}), nil) secret, err := client.GetKvSecret(secretName) assert.NoError(t, err, "Expect GetKvSecret to succeed") @@ -58,7 +58,7 @@ func TestGetKV2Secret(t *testing.T) { t.Run("error is thrown when data field is missing", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV2(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", secretAPIPath).Return(kv1Secret(SecretData{"key1": "value1"}), nil) secret, err := client.GetKvSecret(secretName) assert.Error(t, err, "Expected to fail since 'data' field is missing") @@ -74,7 +74,7 @@ func TestGetKV1Secret(t *testing.T) { t.Run("Test missing secret", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV1(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", mock.AnythingOfType("string")).Return(nil, nil) secret, err := client.GetKvSecret("secret/notexist") @@ -85,7 +85,7 @@ func TestGetKV1Secret(t *testing.T) { t.Run("Test parsing KV1 secrets", func(t *testing.T) { vaultMock := &mocks.VaultMock{} setupMockKvV1(vaultMock) - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", secretName).Return(kv1Secret(SecretData{"key1": "value1", "key2": 5}), nil) secret, err := client.GetKvSecret(secretName) @@ -103,7 +103,7 @@ func TestSecretIDGeneration(t *testing.T) { t.Run("Test generating new secret-id", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) metadata := map[string]interface{}{ @@ -128,7 +128,7 @@ func TestSecretIDGeneration(t *testing.T) { t.Run("Test with no secret-id returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) metadata := map[string]interface{}{ @@ -151,7 +151,7 @@ func TestSecretIDGeneration(t *testing.T) { t.Run("Test with no new secret-id returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) metadata := map[string]interface{}{ @@ -168,7 +168,7 @@ func TestSecretIDGeneration(t *testing.T) { vaultMock.On("Write", path.Join(appRolePath, "/secret-id"), mapWith("metadata", string(metadataJSON))).Return(kv1Secret(nil), nil) newSecretID, err := client.GenerateNewAppRoleSecret(secretID, appRoleName) - assert.EqualError(t, err, fmt.Sprintf("Could not generate new approle secret-id for approle path %s", path.Join(appRolePath, "secret-id"))) + assert.EqualError(t, err, fmt.Sprintf("could not generate new approle secret-id for approle path %s", path.Join(appRolePath, "secret-id"))) assert.Equal(t, newSecretID, "") }) } @@ -181,7 +181,7 @@ func TestSecretIDTtl(t *testing.T) { t.Run("Test fetching secreID TTL", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(5 * time.Hour).Format(time.RFC3339) vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{ @@ -195,16 +195,16 @@ func TestSecretIDTtl(t *testing.T) { t.Run("Test with no expiration time", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{}), nil) ttl, err := client.GetAppRoleSecretIDTtl(secretID, appRoleName) - assert.EqualError(t, err, fmt.Sprintf("Could not load secret-id information from path %s", appRolePath)) + assert.EqualError(t, err, fmt.Sprintf("could not load secret-id information from path %s", appRolePath)) assert.Equal(t, time.Duration(0), ttl) }) t.Run("Test with wrong date format", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{ "expiration_time": time.Now().String(), }), nil) @@ -215,7 +215,7 @@ func TestSecretIDTtl(t *testing.T) { t.Run("Test with expired secret-id", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} now := time.Now() expiry := now.Add(-5 * time.Hour).Format(time.RFC3339) vaultMock.On("Write", path.Join(appRolePath, "secret-id/lookup"), mapWith("secret_id", secretID)).Return(kv1Secret(SecretData{ @@ -234,7 +234,7 @@ func TestGetAppRoleName(t *testing.T) { t.Run("Test that correct role name is returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{ "meta": SecretData{ "role_name": "test", @@ -248,27 +248,27 @@ func TestGetAppRoleName(t *testing.T) { t.Run("Test without secret data", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(nil), nil) appRoleName, err := client.GetAppRoleName() - assert.EqualError(t, err, "Could not lookup token information: auth/token/lookup-self") + assert.EqualError(t, err, "could not lookup token information: auth/token/lookup-self") assert.Empty(t, appRoleName) }) t.Run("Test without metadata data", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{}), nil) appRoleName, err := client.GetAppRoleName() - assert.EqualError(t, err, "Token info did not contain metadata auth/token/lookup-self") + assert.EqualError(t, err, "token info did not contain metadata auth/token/lookup-self") assert.Empty(t, appRoleName) }) t.Run("Test without role name in metadata", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{ "meta": SecretData{}, }), nil) @@ -280,7 +280,7 @@ func TestGetAppRoleName(t *testing.T) { t.Run("Test that different role_name types are ignored", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "auth/token/lookup-self").Return(kv1Secret(SecretData{ "meta": SecretData{ "role_name": 5, @@ -297,7 +297,7 @@ func TestTokenRevocation(t *testing.T) { t.Parallel() t.Run("Test that revocation error is returned", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", "auth/token/revoke-self", mock.IsType(map[string]interface{}{})).Return(nil, errors.New("Test")) @@ -308,7 +308,7 @@ func TestTokenRevocation(t *testing.T) { t.Run("Test that revocation endpoint is called", func(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Write", "auth/token/revoke-self", mock.IsType(map[string]interface{}{})).Return(nil, nil) @@ -319,7 +319,7 @@ func TestTokenRevocation(t *testing.T) { func TestUnknownKvVersion(t *testing.T) { vaultMock := &mocks.VaultMock{} - client := Client{vaultMock, &Config{}} + client := Client{nil, vaultMock, &ClientConfig{}} vaultMock.On("Read", "sys/internal/ui/mounts/secret/secret").Return(&api.Secret{ Data: map[string]interface{}{ @@ -335,15 +335,6 @@ func TestUnknownKvVersion(t *testing.T) { } -func TestSetAppRoleMountPont(t *testing.T) { - client := Client{nil, &Config{}} - const newMountpoint = "auth/test" - - client.SetAppRoleMountPoint("auth/test") - - assert.Equal(t, newMountpoint, client.config.AppRoleMountPoint) -} - func setupMockKvV2(vaultMock *mocks.VaultMock) { vaultMock.On("Read", mock.MatchedBy(func(path string) bool { return strings.HasPrefix(path, sysLookupPath) From 3ad2628095bf5e162807e428bc84a3601c8a300c Mon Sep 17 00:00:00 2001 From: Anil Keshav Date: Tue, 22 Oct 2024 10:50:32 +0200 Subject: [PATCH 25/43] feat(vault): not allowing batch token revoke (#4918) * not allowing batch token revoke * chaging values to hold variable name * error message when identifying service token * refactor --------- Co-authored-by: Googlom --- pkg/vault/vault.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/vault/vault.go b/pkg/vault/vault.go index 5943b31c19..4d916230b3 100644 --- a/pkg/vault/vault.go +++ b/pkg/vault/vault.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/vault/api" "path" "strconv" + "strings" "time" ) @@ -181,7 +182,27 @@ func (c *Client) RevokeToken() error { // MustRevokeToken same as RevokeToken but the program is terminated with an error if this fails. // Should be used in defer statements only. func (c *Client) MustRevokeToken() { - if err := c.RevokeToken(); err != nil { + lookupPath := "auth/token/lookup-self" + const serviceTokenPrefix = "hvs." + + secret, err := c.GetSecret(lookupPath) + if err != nil { + log.Entry().Warnf("Could not lookup token at %s, not continuing to revoke: %v", lookupPath, err) + return + } + + tokenID, ok := secret.Data["id"].(string) + if !ok { + log.Entry().Warnf("Could not lookup token.Data.id at %s, not continuing to revoke", lookupPath) + return + } + + if !strings.HasPrefix(tokenID, serviceTokenPrefix) { + log.Entry().Warnf("Service token not identified at %s, not continuing to revoke", lookupPath) + return + } + + if err = c.RevokeToken(); err != nil { log.Entry().WithError(err).Fatal("Could not revoke token") } } From 7b08d47bcb3be7b1e95b3ffee0c1d81b61367703 Mon Sep 17 00:00:00 2001 From: Akramdzhon Azamov <900658008.akram@gmail.com> Date: Wed, 23 Oct 2024 14:09:15 +0500 Subject: [PATCH 26/43] Detect script version 9 (#5154) * detect script v9 as default and detect script v8 as optional for blackduck * unit test fix --------- Co-authored-by: Dmitrii Pavlukhin --- cmd/detectExecuteScan.go | 7 ++++--- cmd/detectExecuteScan_generated.go | 8 ++++---- cmd/detectExecuteScan_test.go | 2 +- resources/metadata/detectExecuteScan.yaml | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/detectExecuteScan.go b/cmd/detectExecuteScan.go index e0f6a68444..5fc5561a10 100644 --- a/cmd/detectExecuteScan.go +++ b/cmd/detectExecuteScan.go @@ -427,10 +427,11 @@ func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error { log.Entry().Infof("Downloading Detect Script") downloadScript := func() error { - if config.UseDetect9 { - return utils.DownloadFile("https://detect.synopsys.com/detect9.sh", "detect.sh", nil, nil) + if config.UseDetect8 { + return utils.DownloadFile("https://detect.synopsys.com/detect8.sh", "detect.sh", nil, nil) } - return utils.DownloadFile("https://detect.synopsys.com/detect8.sh", "detect.sh", nil, nil) + return utils.DownloadFile("https://detect.synopsys.com/detect9.sh", "detect.sh", nil, nil) + } if err := downloadScript(); err != nil { diff --git a/cmd/detectExecuteScan_generated.go b/cmd/detectExecuteScan_generated.go index 2da821f465..bd771a332c 100644 --- a/cmd/detectExecuteScan_generated.go +++ b/cmd/detectExecuteScan_generated.go @@ -77,7 +77,7 @@ type detectExecuteScanOptions struct { RegistryURL string `json:"registryUrl,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` RepositoryUsername string `json:"repositoryUsername,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` RepositoryPassword string `json:"repositoryPassword,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` - UseDetect9 bool `json:"useDetect9,omitempty"` + UseDetect8 bool `json:"useDetect8,omitempty"` } type detectExecuteScanInflux struct { @@ -354,7 +354,7 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan cmd.Flags().StringVar(&stepConfig.RegistryURL, "registryUrl", os.Getenv("PIPER_registryUrl"), "Used accessing for the images to be scanned (typically filled by CPE)") cmd.Flags().StringVar(&stepConfig.RepositoryUsername, "repositoryUsername", os.Getenv("PIPER_repositoryUsername"), "Used accessing for the images to be scanned (typically filled by CPE)") cmd.Flags().StringVar(&stepConfig.RepositoryPassword, "repositoryPassword", os.Getenv("PIPER_repositoryPassword"), "Used accessing for the images to be scanned (typically filled by CPE)") - cmd.Flags().BoolVar(&stepConfig.UseDetect9, "useDetect9", false, "This flag enables the use of the supported version 9 of the Detect Script instead of v8") + cmd.Flags().BoolVar(&stepConfig.UseDetect8, "useDetect8", false, "This flag enables the use of the supported version 8 of the Detect Script instead of v9") cmd.MarkFlagRequired("token") cmd.MarkFlagRequired("projectName") @@ -947,12 +947,12 @@ func detectExecuteScanMetadata() config.StepData { Default: os.Getenv("PIPER_repositoryPassword"), }, { - Name: "useDetect9", + Name: "useDetect8", ResourceRef: []config.ResourceReference{}, Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "bool", Mandatory: false, - Aliases: []config.Alias{{Name: "detect/useDetect9"}}, + Aliases: []config.Alias{{Name: "detect/useDetect8"}}, Default: false, }, }, diff --git a/cmd/detectExecuteScan_test.go b/cmd/detectExecuteScan_test.go index 7f02817729..8b4ac8d1f8 100644 --- a/cmd/detectExecuteScan_test.go +++ b/cmd/detectExecuteScan_test.go @@ -310,7 +310,7 @@ func TestRunDetect(t *testing.T) { utilsMock.AddFile("detect.sh", []byte("")) err := runDetect(ctx, detectExecuteScanOptions{}, utilsMock, &detectExecuteScanInflux{}) - assert.Equal(t, utilsMock.downloadedFiles["https://detect.synopsys.com/detect8.sh"], "detect.sh") + assert.Equal(t, utilsMock.downloadedFiles["https://detect.synopsys.com/detect9.sh"], "detect.sh") assert.True(t, utilsMock.HasRemovedFile("detect.sh")) assert.NoError(t, err) assert.Equal(t, ".", utilsMock.Dir, "Wrong execution directory used") diff --git a/resources/metadata/detectExecuteScan.yaml b/resources/metadata/detectExecuteScan.yaml index e098c32c40..138a1e5e19 100644 --- a/resources/metadata/detectExecuteScan.yaml +++ b/resources/metadata/detectExecuteScan.yaml @@ -653,11 +653,11 @@ spec: resourceRef: - name: commonPipelineEnvironment param: container/repositoryPassword - - name: useDetect9 + - name: useDetect8 description: - "This flag enables the use of the supported version 9 of the Detect Script instead of v8" + "This flag enables the use of the supported version 8 of the Detect Script instead of v9" aliases: - - name: detect/useDetect9 + - name: detect/useDetect8 type: bool scope: - PARAMETERS From 7b7ba7743607e1840f3e80057374b0f94b22a74c Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:32:37 +0500 Subject: [PATCH 27/43] modify logging (#5158) --- pkg/vault/client.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/vault/client.go b/pkg/vault/client.go index 659d94c205..4e1fc842ab 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -90,9 +90,13 @@ func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { initialLoginSucceed := false for i := 0; i < c.vaultApiClient.MaxRetries(); i++ { + if i != 0 { + log.Entry().Infof("Retrying Vault login. Attempt %d of %d", i, c.vaultApiClient.MaxRetries()) + } + vaultLoginResp, err := c.login() if err != nil { - log.Entry().Errorf("unable to authenticate to Vault: %v", err) + log.Entry().Warnf("unable to authenticate to Vault: %v", err) continue } if !initialLoginSucceed { @@ -103,7 +107,7 @@ func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { tokenErr := c.manageTokenLifecycle(vaultLoginResp) if tokenErr != nil { - log.Entry().Errorf("unable to start managing token lifecycle: %v", err) + log.Entry().Warnf("unable to start managing token lifecycle: %v", err) continue } } From 4990b2d0ba9670dc54ac7d08ce701da4221ff87d Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:47:32 +0500 Subject: [PATCH 28/43] fix(sonar): allign groovy part of Sonar step with common piperExecuteBin (#5157) * add some logging to Vault login * allign groovy part of the sonar step * Revert "add some logging to Vault login" This reverts commit d1738c124d2c1fbfb5becaad2a28dafcef4574fc. --- vars/sonarExecuteScan.groovy | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/vars/sonarExecuteScan.groovy b/vars/sonarExecuteScan.groovy index 7265f4bbac..94d4c4fff6 100644 --- a/vars/sonarExecuteScan.groovy +++ b/vars/sonarExecuteScan.groovy @@ -65,17 +65,19 @@ void call(Map parameters = [:]) { writePipelineEnv(script: script, piperGoPath: piperGoPath) withEnv(environment) { influxWrapper(script) { - piperExecuteBin.credentialWrapper(config, credentialInfo) { - if (stepConfig.instance) { - withSonarQubeEnv(stepConfig.instance) { - echo "Instance is deprecated - please use serverUrl parameter to set URL to the Sonar backend." + try { + piperExecuteBin.credentialWrapper(config, credentialInfo) { + if (stepConfig.instance) { + withSonarQubeEnv(stepConfig.instance) { + echo "Instance is deprecated - please use serverUrl parameter to set URL to the Sonar backend." + sh "${piperGoPath} ${STEP_NAME}${customDefaultConfig}${customConfigArg}" + } + } else { sh "${piperGoPath} ${STEP_NAME}${customDefaultConfig}${customConfigArg}" - jenkinsUtils.handleStepResults(STEP_NAME, false, false) - readPipelineEnv(script: script, piperGoPath: piperGoPath) } - } else { - sh "${piperGoPath} ${STEP_NAME}${customDefaultConfig}${customConfigArg}" } + } finally { + jenkinsUtils.handleStepResults(STEP_NAME, false, false) } } } From 0c41f9c14156a263d9f94f538796515a50182038 Mon Sep 17 00:00:00 2001 From: Googlom <36107508+Googlom@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:08:09 +0500 Subject: [PATCH 29/43] fix deadlock and add more logging (#5160) --- pkg/vault/client.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/vault/client.go b/pkg/vault/client.go index 4e1fc842ab..4d5875be9b 100644 --- a/pkg/vault/client.go +++ b/pkg/vault/client.go @@ -82,16 +82,18 @@ func NewClientWithToken(cfg *ClientConfig, token string) (*Client, error) { func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { defer func() { // make sure to close channel to avoid blocking of the caller - _, open := <-initialLoginDone - if open { - close(initialLoginDone) - } + log.Entry().Debugf("exiting Vault token lifecycle manager") + initialLoginDone <- struct{}{} + close(initialLoginDone) }() initialLoginSucceed := false - for i := 0; i < c.vaultApiClient.MaxRetries(); i++ { + retryAttemptDuration := c.vaultApiClient.MinRetryWait() + for i := 0; i <= c.vaultApiClient.MaxRetries(); i++ { if i != 0 { - log.Entry().Infof("Retrying Vault login. Attempt %d of %d", i, c.vaultApiClient.MaxRetries()) + log.Entry().Infof("Retrying Vault login in %.0f seconds. Attempt %d of %d", + retryAttemptDuration.Seconds(), i, c.vaultApiClient.MaxRetries()) + time.Sleep(retryAttemptDuration) } vaultLoginResp, err := c.login() @@ -101,7 +103,6 @@ func (c *Client) startTokenLifecycleManager(initialLoginDone chan struct{}) { } if !initialLoginSucceed { initialLoginDone <- struct{}{} - close(initialLoginDone) initialLoginSucceed = true } From 8b4109bf85db2807f7eceb347df3e9c4a90e558d Mon Sep 17 00:00:00 2001 From: Holger Partsch Date: Thu, 24 Oct 2024 17:29:40 +0200 Subject: [PATCH 30/43] fix: command injection vulnerability (#5161) due to missing quoting, command injection was possible via pipeline configuration. This is now fixed using a quoting and escaping utility. Co-authored-by: Oliver Feldmann --- test/groovy/TestsPublishResultsTest.groovy | 17 +++++++++++++++++ vars/testsPublishResults.groovy | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/groovy/TestsPublishResultsTest.groovy b/test/groovy/TestsPublishResultsTest.groovy index f851577880..94dd1c21c5 100644 --- a/test/groovy/TestsPublishResultsTest.groovy +++ b/test/groovy/TestsPublishResultsTest.groovy @@ -1,3 +1,4 @@ +import com.sap.piper.BashUtils import org.junit.After import org.junit.Before import org.junit.Ignore @@ -9,8 +10,12 @@ import org.junit.rules.ExpectedException import util.BasePiperTest import util.JenkinsReadYamlRule import util.JenkinsStepRule + +import static org.hamcrest.Matchers.not import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertThat import static org.junit.Assert.assertTrue +import static org.hamcrest.Matchers.containsString import com.sap.piper.Utils @@ -230,4 +235,16 @@ class TestsPublishResultsTest extends BasePiperTest { stepRule.step.testsPublishResults(script: nullScript, failOnError: true) } + + @Test + void testPublishUnitTestsWithUpdateResultsDoesNotAllowCommandExecution() throws Exception { + def injectString = "' -exec touch {} ; rm -rf / # –" + helper.registerAllowedMethod('sh', [String], { String cmd -> + assertThat(cmd, containsString(BashUtils.quoteAndEscape(injectString))) + }) + + stepRule.step.testsPublishResults(script: nullScript, junit: [pattern: injectString, archive: true, active: true, updateResults: true]) + + + } } diff --git a/vars/testsPublishResults.groovy b/vars/testsPublishResults.groovy index 00a0685e55..e86521726d 100644 --- a/vars/testsPublishResults.groovy +++ b/vars/testsPublishResults.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper @@ -193,7 +194,7 @@ void touchFiles(pattern){ echo "[${STEP_NAME}] update test results" def patternArray = pattern.split(',') for(def i = 0; i < patternArray.length; i++){ - sh "find . -wholename '${patternArray[i].trim()}' -exec touch {} \\;" + sh "find . -wholename ${q(patternArray[i].trim())} -exec touch {} \\;" } } From f9dc47e47e278991725a10c46ff70049edc78f5b Mon Sep 17 00:00:00 2001 From: Srinikitha Kondreddy Date: Fri, 25 Oct 2024 09:53:55 +0200 Subject: [PATCH 31/43] Fix: validate app name (#5155) --- pkg/transportrequest/cts/upload.go | 4 ++-- pkg/transportrequest/cts/upload_test.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/transportrequest/cts/upload.go b/pkg/transportrequest/cts/upload.go index 6f959d7463..2cf141c378 100644 --- a/pkg/transportrequest/cts/upload.go +++ b/pkg/transportrequest/cts/upload.go @@ -62,7 +62,7 @@ const ( abapUserKey = "ABAP_USER" abapPasswordKey = "ABAP_PASSWORD" defaultConfigFileName = "ui5-deploy.yaml" - pattern = "^[a-zA-Z0-9_]+$" + pattern = "^(/[A-Za-z0-9_]{3,8}/)?[A-Za-z0-9_]+$" ) // WithConnection ... @@ -194,7 +194,7 @@ func getFioriDeployStatement( if len(app.Name) > 0 { re := regexp.MustCompile(pattern) if !re.MatchString(app.Name) { - return "", fmt.Errorf("application name '%s' contains spaces or special characters and is not according to the regex '%s'.", app.Name, pattern) + return "", fmt.Errorf("application name '%s' contains spaces or special characters or invalid namespace prefix and is not according to the regex '%s'.", app.Name, pattern) } log.Entry().Debugf("application name '%s' used from piper config", app.Name) cmd = append(cmd, "--name", app.Name) diff --git a/pkg/transportrequest/cts/upload_test.go b/pkg/transportrequest/cts/upload_test.go index 852d4908f2..f48b7da945 100644 --- a/pkg/transportrequest/cts/upload_test.go +++ b/pkg/transportrequest/cts/upload_test.go @@ -52,7 +52,7 @@ func TestUploadCTS(t *testing.T) { cmd := mock.ShellMockRunner{} action := UploadAction{ Connection: Connection{Endpoint: "https://example.org:8080/cts", Client: "001", User: "me", Password: "******"}, - Application: Application{Pack: "abapPackage", Name: "appName", Desc: "the Desc"}, + Application: Application{Pack: "abapPackage", Name: "/0ABCD/appName", Desc: "the Desc"}, Node: Node{ DeployDependencies: []string{}, InstallOpts: []string{}, @@ -66,7 +66,7 @@ func TestUploadCTS(t *testing.T) { if assert.NoError(t, err) { assert.Regexp( t, - "(?m)^fiori deploy --failfast --yes --username ABAP_USER --password ABAP_PASSWORD --description \"the Desc\" --noConfig --url https://example.org:8080/cts --client 001 --transport 12345678 --package abapPackage --name appName$", + "(?m)^fiori deploy --failfast --yes --username ABAP_USER --password ABAP_PASSWORD --description \"the Desc\" --noConfig --url https://example.org:8080/cts --client 001 --transport 12345678 --package abapPackage --name /0ABCD/appName", cmd.Calls[0], "Expected fiori deploy command not found", ) @@ -101,11 +101,11 @@ func TestUploadCTS(t *testing.T) { } }) - t.Run("fail in case of invalid app name", func(t *testing.T) { + t.Run("fail in case of invalid app name", func(t *testing.T) { cmd := mock.ShellMockRunner{} action := UploadAction{ Connection: Connection{Endpoint: "https://example.org:8080/cts", Client: "001", User: "me", Password: "******"}, - Application: Application{Pack: "abapPackage", Name: "app Name", Desc: "the Desc"}, + Application: Application{Pack: "abapPackage", Name: "/AB/app1", Desc: "the Desc"}, Node: Node{ DeployDependencies: []string{}, InstallOpts: []string{}, @@ -116,7 +116,7 @@ func TestUploadCTS(t *testing.T) { } err := action.Perform(&cmd) - expectedErrorMessge := "application name 'app Name' contains spaces or special characters and is not according to the regex '^[a-zA-Z0-9_]+$'." + expectedErrorMessge := "application name '/AB/app1' contains spaces or special characters or invalid namespace prefix and is not according to the regex '^(/[A-Za-z0-9_]{3,8}/)?[A-Za-z0-9_]+$'." assert.EqualErrorf(t, err, expectedErrorMessge, "invalid app name") }) From d6aaf43fae2621413e1cd01604c0d6c9325475ed Mon Sep 17 00:00:00 2001 From: Vyacheslav Starostin <32613074+vstarostin@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:07:28 +0500 Subject: [PATCH 32/43] Update version of org.cyclonedx:cyclonedx-maven-plugin (#5156) * Update version of org.cyclonedx:cyclonedx-maven-plugin * Update version of org.cyclonedx:cyclonedx-maven-plugin --------- Co-authored-by: Googlom <36107508+Googlom@users.noreply.github.com> --- cmd/mavenBuild.go | 6 +++--- cmd/mavenBuild_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index 559e5c2188..a3cb22d86d 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -66,7 +66,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { } defines = append(defines, createBOMConfig...) - goals := []string{"org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeBom"} + goals := []string{"org.cyclonedx:cyclonedx-maven-plugin:2.7.9:makeBom"} if config.Flatten { goals = append(goals, "flatten:flatten") @@ -88,7 +88,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { return err } -func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { +func runMavenBuild(config *mavenBuildOptions, _ *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { var flags = []string{"-update-snapshots", "--batch-mode"} @@ -111,7 +111,7 @@ func runMavenBuild(config *mavenBuildOptions, telemetryData *telemetry.CustomDat if config.CreateBOM { // Append the makeAggregateBOM goal to the rest of the goals - goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + goals = append(goals, "org.cyclonedx:cyclonedx-maven-plugin:2.7.9:makeAggregateBom") createBOMConfig := []string{ "-DschemaVersion=1.4", "-DincludeBomSerialNumber=true", diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index ea5e9abdbc..36647bb3ba 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -55,7 +55,7 @@ func TestMavenBuild(t *testing.T) { assert.Nil(t, err) if assert.Equal(t, 2, len(mockedUtils.Calls), "Expected two Maven invocations (default + makeAggregateBom)") { assert.Equal(t, "mvn", mockedUtils.Calls[1].Exec) - assert.Contains(t, mockedUtils.Calls[0].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.8:makeAggregateBom") + assert.Contains(t, mockedUtils.Calls[0].Params, "org.cyclonedx:cyclonedx-maven-plugin:2.7.9:makeAggregateBom") assert.Contains(t, mockedUtils.Calls[0].Params, "-DoutputName=bom-maven") } }) From 183004a80d2b240b642fa23bdd0c7631058d2a58 Mon Sep 17 00:00:00 2001 From: Manjunath Date: Tue, 29 Oct 2024 08:39:34 +0100 Subject: [PATCH 33/43] Add identifier data to create uuid in events (#5165) --- cmd/gcpPublishEvent.go | 2 +- pkg/events/events.go | 16 ++++++++++++++-- pkg/events/events_test.go | 21 ++++++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/cmd/gcpPublishEvent.go b/cmd/gcpPublishEvent.go index 83baab8cf2..95a5493e93 100644 --- a/cmd/gcpPublishEvent.go +++ b/cmd/gcpPublishEvent.go @@ -68,7 +68,7 @@ func runGcpPublishEvent(utils gcpPublishEventUtils) error { } func createNewEvent(config *gcpPublishEventOptions) ([]byte, error) { - event, err := events.NewEvent(config.EventType, config.EventSource).CreateWithJSONData(config.EventData) + event, err := events.NewEvent(config.EventType, config.EventSource, "").CreateWithJSONData(config.EventData) if err != nil { return []byte{}, errors.Wrap(err, "failed to create new event") } diff --git a/pkg/events/events.go b/pkg/events/events.go index 700337e9f4..ed95f10268 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -21,12 +21,14 @@ type Event struct { cloudEvent cloudevents.Event eventType string eventSource string + uuidData string } -func NewEvent(eventType, eventSource string) Event { +func NewEvent(eventType, eventSource string, uuidString string) Event { return Event{ eventType: eventType, eventSource: eventSource, + uuidData: uuidString, } } @@ -45,8 +47,14 @@ func (e Event) CreateWithJSONData(data string, opts ...Option) (Event, error) { func (e Event) Create(data any, opts ...Option) Event { e.cloudEvent = cloudevents.NewEvent("1.0") + + if e.uuidData != "" { + e.cloudEvent.SetID(GetUUID(e.uuidData)) + } else { + e.cloudEvent.SetID(uuid.New().String()) + } + // set default values - e.cloudEvent.SetID(uuid.New().String()) e.cloudEvent.SetType(e.eventType) e.cloudEvent.SetTime(time.Now()) e.cloudEvent.SetSource(e.eventSource) @@ -58,6 +66,10 @@ func (e Event) Create(data any, opts ...Option) Event { return e } +func GetUUID(pipelineIdentifier string) string { + return uuid.NewMD5(uuid.NameSpaceOID, []byte(pipelineIdentifier)).String() +} + func (e Event) ToBytes() ([]byte, error) { data, err := json.Marshal(e.cloudEvent) if err != nil { diff --git a/pkg/events/events_test.go b/pkg/events/events_test.go index 87e4742ef8..4ab9a40bbe 100644 --- a/pkg/events/events_test.go +++ b/pkg/events/events_test.go @@ -11,7 +11,7 @@ func TestEventCreation(t *testing.T) { t.Run("success", func(t *testing.T) { // init // test - event := NewEvent(mock.Anything, mock.Anything).Create(nil) + event := NewEvent(mock.Anything, mock.Anything, "").Create(nil) // asserts assert.Equal(t, mock.Anything, event.cloudEvent.Type()) assert.Equal(t, mock.Anything, event.cloudEvent.Source()) @@ -21,7 +21,7 @@ func TestEventCreation(t *testing.T) { // init testData := `{"testKey":"testValue"}` // test - event, err := NewEvent(mock.Anything, mock.Anything).CreateWithJSONData(testData) + event, err := NewEvent(mock.Anything, mock.Anything, "").CreateWithJSONData(testData) // asserts assert.NoError(t, err) assert.Equal(t, string(event.cloudEvent.Data()), testData) @@ -32,10 +32,25 @@ func TestEventCreation(t *testing.T) { testData := `{"testKey": "testValue"}` additionalData := `{"additionalKey": "additionalValue"}` // test - event, err := NewEvent(mock.Anything, mock.Anything).CreateWithJSONData(testData) + event, err := NewEvent(mock.Anything, mock.Anything, "").CreateWithJSONData(testData) event.AddToCloudEventData(additionalData) // asserts assert.NoError(t, err) assert.Equal(t, string(event.cloudEvent.Data()), `{"additionalKey":"additionalValue","testKey":"testValue"}`) }) } + +func TestGetUUID(t *testing.T) { + pipelineIdentifier := "pipelineIdentifier" + uuid := GetUUID(pipelineIdentifier) + + if uuid == "" { + t.Fatalf("expected a UUID but got none") + } + + uuid2 := GetUUID(pipelineIdentifier) + if uuid != uuid2 { + t.Fatalf("expected the same UUID but got different ones") + } + +} From da609e1536e3ed1faf4bebd200f17ccb03681fdb Mon Sep 17 00:00:00 2001 From: Holger Partsch Date: Wed, 30 Oct 2024 11:59:47 +0100 Subject: [PATCH 34/43] Fix more potential command injection via quoting (#5164) * fix: make quoting null safe * fix: apply quoting in artifact set version * fix: add quoting to more shell step * refactor: use import alias * fix: further quoting --------- Co-authored-by: Oliver Feldmann --- src/com/sap/piper/BashUtils.groovy | 3 ++ .../piper/tools/neo/NeoCommandHelper.groovy | 28 ++++++++++--------- test/groovy/ArtifactSetVersionTest.groovy | 18 ++++++------ .../groovy/ContainerPushToRegistryTest.groovy | 8 +++--- test/groovy/HealthExecuteCheckTest.groovy | 4 +-- test/groovy/NeoDeployTest.groovy | 23 ++++++++------- vars/artifactSetVersion.groovy | 15 +++++----- vars/containerPushToRegistry.groovy | 3 +- vars/dockerExecute.groovy | 3 +- vars/healthExecuteCheck.groovy | 3 +- vars/mailSendNotification.groovy | 9 +++--- vars/neoDeploy.groovy | 7 +++-- 12 files changed, 67 insertions(+), 57 deletions(-) diff --git a/src/com/sap/piper/BashUtils.groovy b/src/com/sap/piper/BashUtils.groovy index 2387898e02..15817b89dc 100644 --- a/src/com/sap/piper/BashUtils.groovy +++ b/src/com/sap/piper/BashUtils.groovy @@ -8,6 +8,9 @@ class BashUtils implements Serializable { * Put string in single quotes and escape contained single quotes by putting them into a double quoted string */ static String quoteAndEscape(String str) { + if(str == null) { + return 'null' + } def escapedString = str.replace("'", ESCAPED_SINGLE_QUOTE) return "'${escapedString}'" } diff --git a/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy b/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy index d81a3e385e..56250344de 100644 --- a/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy +++ b/src/com/sap/piper/tools/neo/NeoCommandHelper.groovy @@ -1,8 +1,10 @@ package com.sap.piper.tools.neo -import com.sap.piper.BashUtils +import static com.sap.piper.BashUtils.quoteAndEscape as q + import com.sap.piper.StepAssertions + class NeoCommandHelper { private Script step @@ -87,7 +89,7 @@ class NeoCommandHelper { private String source() { StepAssertions.assertFileExists(step, source) - return "--source ${BashUtils.quoteAndEscape(source)}" + return "--source ${q(source)}" } private String extensions() { @@ -96,19 +98,19 @@ class NeoCommandHelper { } private String mainArgs() { - String usernamePassword = "--user ${BashUtils.quoteAndEscape(user)} --password ${BashUtils.quoteAndEscape(password)}" + String usernamePassword = "--user ${q(user)} --password ${q(password)}" if (deployMode == DeployMode.WAR_PROPERTIES_FILE) { StepAssertions.assertFileIsConfiguredAndExists(step, deploymentConfiguration, 'propertiesFile') return "${deploymentConfiguration.propertiesFile} ${usernamePassword}" } - String targetArgs = "--host ${BashUtils.quoteAndEscape(deploymentConfiguration.host)}" - targetArgs += " --account ${BashUtils.quoteAndEscape(deploymentConfiguration.account)}" + String targetArgs = "--host ${q(deploymentConfiguration.host)}" + targetArgs += " --account ${q(deploymentConfiguration.account)}" if (deployMode == DeployMode.WAR_PARAMS) { - targetArgs += " --application ${BashUtils.quoteAndEscape(deploymentConfiguration.application)}" + targetArgs += " --application ${q(deploymentConfiguration.application)}" } return "${targetArgs} ${usernamePassword}" @@ -120,11 +122,11 @@ class NeoCommandHelper { } String args = "" - args += " --runtime ${BashUtils.quoteAndEscape(deploymentConfiguration.runtime)}" - args += " --runtime-version ${BashUtils.quoteAndEscape(deploymentConfiguration.runtimeVersion)}" + args += " --runtime ${q(deploymentConfiguration.runtime)}" + args += " --runtime-version ${q(deploymentConfiguration.runtimeVersion)}" if (deploymentConfiguration.size) { - args += " --size ${BashUtils.quoteAndEscape(deploymentConfiguration.size)}" + args += " --size ${q(deploymentConfiguration.size)}" } if (deploymentConfiguration.containsKey('environment')) { @@ -139,17 +141,17 @@ class NeoCommandHelper { for (int i = 0; i < keys.size(); i++) { def key = keys[i] def value = environment.get(keys[i]) - args += " --ev ${BashUtils.quoteAndEscape(key)}=${BashUtils.quoteAndEscape(value)}" + args += " --ev ${q(key)}=${q(value)}" } } if (deploymentConfiguration.containsKey('vmArguments')) { - args += " --vm-arguments ${BashUtils.quoteAndEscape(deploymentConfiguration.vmArguments)}" + args += " --vm-arguments ${q(deploymentConfiguration.vmArguments)}" } - + if (deploymentConfiguration.containsKey('azDistribution')) { - args += " --az-distribution ${BashUtils.quoteAndEscape(deploymentConfiguration.azDistribution)}" + args += " --az-distribution ${q(deploymentConfiguration.azDistribution)}" } return args diff --git a/test/groovy/ArtifactSetVersionTest.groovy b/test/groovy/ArtifactSetVersionTest.groovy index f9b145337c..edba0169f6 100644 --- a/test/groovy/ArtifactSetVersionTest.groovy +++ b/test/groovy/ArtifactSetVersionTest.groovy @@ -130,8 +130,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(shellRule.shell.join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - 'git push myGitSshUrl build_1.2.3-20180101010203_testCommitId', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "git push 'myGitSshUrl' 'build_1.2.3-20180101010203_testCommitId'", ] )) } @@ -173,8 +173,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(((Iterable)shellRule.shell).join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - 'git push https://me:topSecret@example.org/myGitRepo build_1.2.3-20180101010203_testCommitId', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "git push https://me:topSecret@example.org/myGitRepo 'build_1.2.3-20180101010203_testCommitId'", ] )) } @@ -246,8 +246,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(((Iterable)shellRule.shell).join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - '#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo build_1.2.3-20180101010203_testCommitId &>/dev/null', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo 'build_1.2.3-20180101010203_testCommitId' &>/dev/null", ] )) } @@ -278,8 +278,8 @@ class ArtifactSetVersionTest extends BasePiperTest { assertThat(((Iterable)shellRule.shell).join(), stringContainsInOrder([ "git add .", "git commit -m 'update version 1.2.3-20180101010203_testCommitId'", - 'git tag build_1.2.3-20180101010203_testCommitId', - '#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo build_1.2.3-20180101010203_testCommitId &>/dev/null', + "git tag 'build_1.2.3-20180101010203_testCommitId'", + "#!/bin/bash -e git push --quiet https://me:top%40Secret@example.org/myGitRepo 'build_1.2.3-20180101010203_testCommitId' &>/dev/null", ] )) } @@ -301,7 +301,7 @@ class ArtifactSetVersionTest extends BasePiperTest { void testVersioningCustomGitUserAndEMail() { stepRule.step.artifactSetVersion(script: stepRule.step, juStabGitUtils: gitUtils, buildTool: 'maven', gitSshUrl: 'myGitSshUrl', gitUserEMail: 'test@test.com', gitUserName: 'test') - assertThat(shellRule.shell, hasItem(containsString("git -c user.email=\"test@test.com\" -c user.name=\"test\" commit -m 'update version 1.2.3-20180101010203_testCommitId'"))) + assertThat(shellRule.shell, hasItem(containsString("git -c user.email='test@test.com' -c user.name='test' commit -m 'update version 1.2.3-20180101010203_testCommitId'"))) } @Test diff --git a/test/groovy/ContainerPushToRegistryTest.groovy b/test/groovy/ContainerPushToRegistryTest.groovy index 474ac86990..65fe14f973 100644 --- a/test/groovy/ContainerPushToRegistryTest.groovy +++ b/test/groovy/ContainerPushToRegistryTest.groovy @@ -221,7 +221,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.targetRegistry.credentials, is('testCredentialsId')) assertThat(dockerMock.targetRegistry.isAnonymous, is(false)) assertThat(dockerMock.image, is('path/testImage:tag')) - assertThat(shellCallRule.shell, hasItem('docker tag testRegistry:55555/path/testImage:tag path/testImage:tag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testRegistry:55555'/'path/testImage:tag' 'path/testImage:tag'")) assertThat(dockerMockPull, is(true)) } @@ -238,7 +238,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.sourceRegistry.url, is('http://testSourceRegistry')) assertThat(dockerMock.image, is('testSourceName:testSourceTag')) assertThat(dockerMock.sourceRegistry.isAnonymous, is(true)) - assertThat(shellCallRule.shell, hasItem('docker tag testSourceRegistry/testSourceName:testSourceTag testSourceName:testSourceTag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testSourceRegistry'/'testSourceName:testSourceTag' 'testSourceName:testSourceTag'")) assertThat(dockerMockPull, is(true)) } @@ -256,7 +256,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.sourceRegistry.url, is('http://testSourceRegistry')) assertThat(dockerMock.sourceRegistry.isAnonymous, is(true)) assertThat(dockerMock.image, is('testSourceName:testSourceTag')) - assertThat(shellCallRule.shell, hasItem('docker tag testSourceRegistry/testSourceName:testSourceTag testImage:tag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testSourceRegistry'/'testSourceName:testSourceTag' 'testImage:tag'")) assertThat(dockerMockPull, is(true)) } @@ -277,7 +277,7 @@ class ContainerPushToRegistryTest extends BasePiperTest { assertThat(dockerMock.targetRegistry.url, is('https://testRegistry')) assertThat(dockerMock.targetRegistry.credentials, is('testCredentialsId')) assertThat(dockerMock.image, is('testSourceName:testSourceTag')) - assertThat(shellCallRule.shell, hasItem('docker tag testSourceRegistry/testSourceName:testSourceTag testImage:tag')) + assertThat(shellCallRule.shell, hasItem("docker tag 'testSourceRegistry'/'testSourceName:testSourceTag' 'testImage:tag'")) assertThat(dockerMockPull, is(true)) } diff --git a/test/groovy/HealthExecuteCheckTest.groovy b/test/groovy/HealthExecuteCheckTest.groovy index 0c0762012d..5cb4c1bd2c 100644 --- a/test/groovy/HealthExecuteCheckTest.groovy +++ b/test/groovy/HealthExecuteCheckTest.groovy @@ -32,8 +32,8 @@ class HealthExecuteCheckTest extends BasePiperTest { @Before void init() throws Exception { // register Jenkins commands with mock values - def command1 = "curl -so /dev/null -w '%{response_code}' http://testserver" - def command2 = "curl -so /dev/null -w '%{response_code}' http://testserver/endpoint" + def command1 = "curl -so /dev/null -w '%{response_code}' 'http://testserver'" + def command2 = "curl -so /dev/null -w '%{response_code}' 'http://testserver/endpoint'" helper.registerAllowedMethod('sh', [Map.class], {map -> return map.script == command1 || map.script == command2 ? "200" : "404" }) diff --git a/test/groovy/NeoDeployTest.groovy b/test/groovy/NeoDeployTest.groovy index 72993b6de9..d4f6d15123 100644 --- a/test/groovy/NeoDeployTest.groovy +++ b/test/groovy/NeoDeployTest.groovy @@ -298,25 +298,24 @@ class NeoDeployTest extends BasePiperTest { nullScript.commonPipelineEnvironment.setMtarFilePath('archive.mtar') - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "https:\\/\\/api\\.test\\.com\\/oauth2\\/apitoken\\/v1", "{\"access_token\":\"xxx\"}") - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "https:\\/\\/slservice\\.test\\.host\\.com\\/slservice\\/v1\\/oauth\\/accounts\\/testUser123\\/mtars", "{\"id\":123}") - shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "https:\\/\\/slservice\\.test\\.host\\.com\\/slservice\\/v1\\/oauth\\/accounts\\/testUser123\\/mtars", "{\"state\":\"DONE\"}") - + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "-XPOST.*/apitoken", "{\"access_token\":\"xxx\"}") + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "-XPOST.*https://slservice", "{\"id\":123}") + shellRule.setReturnValue(JenkinsShellCallRule.Type.REGEX, "-XGET.*https://slservice", "{\"state\":\"DONE\"}") stepRule.step.neoDeploy( script: nullScript, source: archiveName, deployMode: 'mta', - neo: [ - host: 'test.host.com', - account: 'testUser123', - credentialsId: 'OauthDataFileId', - credentialType: 'SecretFile' - ], + neo: [ + host : 'test.host.com', + account : 'testUser123', + credentialsId : 'OauthDataFileId', + credentialType: 'SecretFile' + ], ) - Assert.assertThat(shellRule.shell[0], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -u \"abc123:testclientsecret123\" \"https://api.test.com/oauth2/apitoken/v1?grant_type=client_credentials\"")) - Assert.assertThat(shellRule.shell[1], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -H \"Authorization: Bearer xxx\" -F file=@\"archive.mtar\" \"https://slservice.test.host.com/slservice/v1/oauth/accounts/testUser123/mtars\"")) + Assert.assertThat(shellRule.shell[0], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -u 'abc123':'testclientsecret123' 'https://api.test.com/oauth2'/apitoken/v1?grant_type=client_credentials")) + Assert.assertThat(shellRule.shell[1], containsString("#!/bin/bash curl --fail --silent --show-error --retry 12 -XPOST -H \"Authorization: Bearer xxx\" -F file=@'archive.mtar' https://slservice.'test.host.com'/slservice/v1/oauth/accounts/'testUser123'/mtars")) } @Test diff --git a/vars/artifactSetVersion.groovy b/vars/artifactSetVersion.groovy index e46ea77845..2faf9390a2 100644 --- a/vars/artifactSetVersion.groovy +++ b/vars/artifactSetVersion.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper @@ -175,7 +176,7 @@ void call(Map parameters = [:], Closure body = null) { def gitConfig = [] if(config.gitUserEMail) { - gitConfig.add("-c user.email=\"${config.gitUserEMail}\"") + gitConfig.add("-c user.email=${q(config.gitUserEMail)}") } else { // in case there is no user.email configured on project level we might still // be able to work in case there is a configuration available on plain git level. @@ -184,7 +185,7 @@ void call(Map parameters = [:], Closure body = null) { } } if(config.gitUserName) { - gitConfig.add("-c user.name=\"${config.gitUserName}\"") + gitConfig.add("-c user.name=${q(config.gitUserName)}") } else { // in case there is no user.name configured on project level we might still // be able to work in case there is a configuration available on plain git level. @@ -199,7 +200,7 @@ void call(Map parameters = [:], Closure body = null) { set -e git add . --update git ${gitConfig} commit -m 'update version ${newVersion}' - git tag ${config.tagPrefix}${newVersion}""" + git tag ${q(config.tagPrefix+newVersion)}""" config.gitCommitId = gitUtils.getGitCommitIdOrNull() } catch (e) { error "[${STEP_NAME}]git commit and tag failed: ${e}" @@ -215,7 +216,7 @@ void call(Map parameters = [:], Closure body = null) { .use() sshagent([config.gitSshKeyCredentialsId]) { - sh "git push ${config.gitSshUrl} ${config.tagPrefix}${newVersion}" + sh "git push ${q(config.gitSshUrl)} ${q(config.tagPrefix+newVersion)}" } } else if(gitPushMode == GitPushMode.HTTPS) { @@ -259,7 +260,7 @@ void call(Map parameters = [:], Closure body = null) { gitConfig = [] if(config.gitHttpProxy) { - gitConfig.add("-c http.proxy=\"${config.gitHttpProxy}\"") + gitConfig.add("-c http.proxy=${q(config.gitHttpProxy)}") } if(config.gitDisableSslVerification) { @@ -288,7 +289,7 @@ void call(Map parameters = [:], Closure body = null) { gitPushFlags = gitPushFlags.join(' ') sh script: """|#!/bin/bash ${hashbangFlags} - |${gitDebug}git ${gitConfig} push ${gitPushFlags} ${gitUrlWithCredentials} ${config.tagPrefix}${newVersion} ${streamhandling}""".stripMargin() + |${gitDebug}git ${gitConfig} push ${gitPushFlags} ${gitUrlWithCredentials} ${q(config.tagPrefix+newVersion)} ${streamhandling}""".stripMargin() } } else { echo "Git push mode: ${gitPushMode.toString()}. Git push to remote has been skipped." @@ -313,5 +314,5 @@ def isAppContainer(config){ } def getTimestamp(pattern){ - return sh(returnStdout: true, script: "date --utc +'${pattern}'").trim() + return sh(returnStdout: true, script: "date --utc +${q(pattern)}").trim() } diff --git a/vars/containerPushToRegistry.groovy b/vars/containerPushToRegistry.groovy index ca29e6694e..3707eafd8c 100644 --- a/vars/containerPushToRegistry.groovy +++ b/vars/containerPushToRegistry.groovy @@ -5,6 +5,7 @@ import com.sap.piper.DockerUtils import groovy.transform.Field import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q @Field String STEP_NAME = getClass().getName() @Field Set GENERAL_CONFIG_KEYS = [ @@ -101,7 +102,7 @@ void call(Map parameters = [:]) { ) { sourceBuildImage.pull() } - sh "docker tag ${config.sourceRegistry}/${config.sourceImage} ${config.dockerImage}" + sh "docker tag ${q(config.sourceRegistry)}/${q(config.sourceImage)} ${q(config.dockerImage)}" } docker.withRegistry( diff --git a/vars/dockerExecute.groovy b/vars/dockerExecute.groovy index cb9034a711..9ce9901cc7 100644 --- a/vars/dockerExecute.groovy +++ b/vars/dockerExecute.groovy @@ -1,6 +1,7 @@ import com.sap.piper.SidecarUtils import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.cloudbees.groovy.cps.NonCPS import com.sap.piper.ConfigurationHelper @@ -242,7 +243,7 @@ void call(Map parameters = [:], body) { } } else { def networkName = "sidecar-${UUID.randomUUID()}" - sh "docker network create ${networkName}" + sh "docker network create ${q(networkName)}" try { def sidecarImage = docker.image(config.sidecarImage) pullWrapper(config.sidecarPullImage, sidecarImage, config.sidecarRegistryUrl, config.sidecarRegistryCredentialsId) { diff --git a/vars/healthExecuteCheck.groovy b/vars/healthExecuteCheck.groovy index 05b2997c0f..2992959475 100644 --- a/vars/healthExecuteCheck.groovy +++ b/vars/healthExecuteCheck.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper @@ -70,6 +71,6 @@ void call(Map parameters = [:]) { def curl(url){ return sh( returnStdout: true, - script: "curl -so /dev/null -w '%{response_code}' ${url}" + script: "curl -so /dev/null -w '%{response_code}' ${q(url)}" ).trim() } diff --git a/vars/mailSendNotification.groovy b/vars/mailSendNotification.groovy index df39091662..b53425853d 100644 --- a/vars/mailSendNotification.groovy +++ b/vars/mailSendNotification.groovy @@ -1,4 +1,5 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q import com.sap.piper.ConfigurationHelper import com.sap.piper.GenerateDocumentation @@ -198,8 +199,8 @@ def getCulprits(config, branch, numberOfCommits) { def pullRequestID = branch.replaceAll('PR-', '') def localBranchName = "pr" + pullRequestID sh """git init - git fetch ${config.gitUrl} pull/${pullRequestID}/head:${localBranchName} > /dev/null 2>&1 - git checkout -f ${localBranchName} > /dev/null 2>&1 + git fetch ${q(config.gitUrl)} pull/${q(pullRequestID)}/head:${q(localBranchName)} > /dev/null 2>&1 + git checkout -f ${q(localBranchName)} > /dev/null 2>&1 """ } } else { @@ -210,8 +211,8 @@ def getCulprits(config, branch, numberOfCommits) { credentials: [config.gitSshKeyCredentialsId], ignoreMissing: true ) { - sh """git clone ${config.gitUrl} . - git checkout ${config.gitCommitId} > /dev/null 2>&1""" + sh """git clone ${q(config.gitUrl)} . + git checkout ${q(config.gitCommitId)} > /dev/null 2>&1""" } } else { def retCode = sh(returnStatus: true, script: 'git log > /dev/null 2>&1') diff --git a/vars/neoDeploy.groovy b/vars/neoDeploy.groovy index b83b28f8b8..baaeb79111 100644 --- a/vars/neoDeploy.groovy +++ b/vars/neoDeploy.groovy @@ -10,6 +10,7 @@ import com.sap.piper.tools.neo.WarAction import groovy.transform.Field import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q @Field String STEP_NAME = getClass().getName() @@ -413,7 +414,7 @@ private deployWithBearerToken(def credentialFilePath, Map configuration, Script def myCurl = "curl --fail --silent --show-error --retry 12" def token_json = sh( script: """#!/bin/bash - ${myCurl} -XPOST -u \"${oauthClientId}:${oauthClientSecret}\" \"${oauthUrl}/apitoken/v1?grant_type=client_credentials" + ${myCurl} -XPOST -u ${q(oauthClientId)}:${q(oauthClientSecret)} ${q(oauthUrl)}/apitoken/v1?grant_type=client_credentials """, returnStdout: true ) @@ -424,7 +425,7 @@ private deployWithBearerToken(def credentialFilePath, Map configuration, Script def deploymentContentResponse = sh( script: """#!/bin/bash - ${myCurl} -XPOST -H \"Authorization: Bearer ${token}\" -F file=@\"${deployArchive}\" \"https://slservice.${host}/slservice/v1/oauth/accounts/${account}/mtars\" + ${myCurl} -XPOST -H "Authorization: Bearer ${token}" -F file=@${q(deployArchive)} https://slservice.${q(host)}/slservice/v1/oauth/accounts/${q(account)}/mtars """, returnStdout: true ) @@ -434,7 +435,7 @@ private deployWithBearerToken(def credentialFilePath, Map configuration, Script echo "[${STEP_NAME}] Deployment Id is '${deploymentId}'." def statusPollScript = """#!/bin/bash - ${myCurl} -XGET -H \"Authorization: Bearer ${token}\" \"https://slservice.${host}/slservice/v1/oauth/accounts/${account}/mtars/${deploymentId}\" + ${myCurl} -XGET -H "Authorization: Bearer ${token}" https://slservice.${q(host)}/slservice/v1/oauth/accounts/${q(account)}/mtars/${deploymentId} """ def statusResponse = sh(script: statusPollScript, returnStdout: true) def statusJson = readJSON text: statusResponse From 6988f43f7f244d69a0d971e7dd1e7b10d7098dc9 Mon Sep 17 00:00:00 2001 From: phgermanov Date: Mon, 4 Nov 2024 12:30:39 +0200 Subject: [PATCH 35/43] feat: add build artifacts metadata for mtaBuild (#5166) --- cmd/mavenBuild.go | 29 +--- cmd/mavenBuild_test.go | 54 ------- cmd/mtaBuild.go | 194 +++++++++++++---------- cmd/mtaBuild_generated.go | 14 ++ cmd/mtaBuild_test.go | 235 +++++++++++++++++++++------- pkg/npm/publish.go | 18 +-- pkg/npm/publish_test.go | 49 +----- pkg/piperutils/cyclonedxBom.go | 12 +- pkg/piperutils/cyclonedxbom_test.go | 81 ++++++++-- resources/metadata/mtaBuild.yaml | 9 ++ 10 files changed, 395 insertions(+), 300 deletions(-) diff --git a/cmd/mavenBuild.go b/cmd/mavenBuild.go index a3cb22d86d..5f8a4b0a96 100644 --- a/cmd/mavenBuild.go +++ b/cmd/mavenBuild.go @@ -41,7 +41,7 @@ func mavenBuild(config mavenBuildOptions, telemetryData *telemetry.CustomData, c } func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { - var flags = []string{"-update-snapshots", "--batch-mode"} + flags := []string{"-update-snapshots", "--batch-mode"} if len(config.Profiles) > 0 { flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) } @@ -89,8 +89,7 @@ func runMakeBOMGoal(config *mavenBuildOptions, utils maven.Utils) error { } func runMavenBuild(config *mavenBuildOptions, _ *telemetry.CustomData, utils maven.Utils, commonPipelineEnvironment *mavenBuildCommonPipelineEnvironment) error { - - var flags = []string{"-update-snapshots", "--batch-mode"} + flags := []string{"-update-snapshots", "--batch-mode"} if len(config.Profiles) > 0 { flags = append(flags, "--activate-profiles", strings.Join(config.Profiles, ",")) @@ -255,7 +254,7 @@ func createBuildArtifactsMetadata(config *mavenBuildOptions, commonPipelineEnvir } else { coordinate.BuildPath = filepath.Dir(match) coordinate.URL = config.AltDeploymentRepositoryURL - coordinate.PURL = getPurlForThePom(match) + coordinate.PURL = piperutils.GetPurl(filepath.Join(filepath.Dir(match), "/target/"+mvnSimpleBomFilename+".xml")) buildCoordinates = append(buildCoordinates, coordinate) } } @@ -274,25 +273,6 @@ func createBuildArtifactsMetadata(config *mavenBuildOptions, commonPipelineEnvir return nil, false } -func getPurlForThePom(pomFilePath string) string { - bomPath := filepath.Join(filepath.Dir(pomFilePath) + "/target/" + mvnSimpleBomFilename + ".xml") - exists, _ := piperutils.FileExists(bomPath) - if !exists { - log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", bomPath) - return "" - } - bom, err := piperutils.GetBom(bomPath) - if err != nil { - log.Entry().Warnf("failed to get bom file %s: %v", bomPath, err) - return "" - } - - log.Entry().Debugf("Found purl: %s for the bomPath: %s", bom.Metadata.Component.Purl, bomPath) - purl := bom.Metadata.Component.Purl - - return purl -} - func createOrUpdateProjectSettingsXML(projectSettingsFile string, altDeploymentRepositoryID string, altDeploymentRepositoryUser string, altDeploymentRepositoryPassword string, utils maven.Utils) (string, error) { if len(projectSettingsFile) > 0 { projectSettingsFilePath, err := maven.UpdateProjectSettingsXML(projectSettingsFile, altDeploymentRepositoryID, altDeploymentRepositoryUser, altDeploymentRepositoryPassword, utils) @@ -310,7 +290,7 @@ func createOrUpdateProjectSettingsXML(projectSettingsFile string, altDeploymentR } func loadRemoteRepoCertificates(certificateList []string, client piperhttp.Downloader, flags *[]string, runner command.ExecRunner, fileUtils piperutils.FileUtils, javaCaCertFilePath string) error { - //TODO: make use of java/keytool package + // TODO: make use of java/keytool package existingJavaCaCerts := filepath.Join(os.Getenv("JAVA_HOME"), "jre", "lib", "security", "cacerts") if len(javaCaCertFilePath) > 0 { @@ -318,7 +298,6 @@ func loadRemoteRepoCertificates(certificateList []string, client piperhttp.Downl } exists, err := fileUtils.FileExists(existingJavaCaCerts) - if err != nil { return errors.Wrap(err, "Could not find the existing java cacerts") } diff --git a/cmd/mavenBuild_test.go b/cmd/mavenBuild_test.go index 36647bb3ba..eddd3097b3 100644 --- a/cmd/mavenBuild_test.go +++ b/cmd/mavenBuild_test.go @@ -4,18 +4,14 @@ package cmd import ( - "os" - "path/filepath" "testing" - "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) var cpe mavenBuildCommonPipelineEnvironment func TestMavenBuild(t *testing.T) { - t.Run("mavenBuild should install the artifact", func(t *testing.T) { mockedUtils := newMavenMockUtils() @@ -123,54 +119,4 @@ func TestMavenBuild(t *testing.T) { assert.Equal(t, mockedUtils.Calls[0].Exec, "mvn") assert.Empty(t, cpe.custom.mavenBuildArtifacts) }) - -} - -func createTempFile(t *testing.T, dir string, filename string, content string) string { - filePath := filepath.Join(dir, filename) - err := os.WriteFile(filePath, []byte(content), 0666) - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } - return filePath -} - -func TestGetPurlForThePomAndDeleteIndividualBom(t *testing.T) { - - t.Run("valid BOM file, aggregated BOM", func(t *testing.T) { - tempDir, err := piperutils.Files{}.TempDir("", "test") - if err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - - bomContent := ` - - - pkg:maven/com.example/aggregatecomponent@1.0.0 - - - - - - ` - pomFilePath := createTempFile(t, tempDir, "pom.xml", "") - bomDir := filepath.Join(tempDir, "target") - if err := os.MkdirAll(bomDir, 0777); err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - bomFilePath := createTempFile(t, bomDir, mvnSimpleBomFilename+".xml", bomContent) - - purl := getPurlForThePom(pomFilePath) - assert.Equal(t, "pkg:maven/com.example/aggregatecomponent@1.0.0", purl) - _, err = os.Stat(bomFilePath) - assert.False(t, os.IsNotExist(err)) // File should not be deleted - }) - - t.Run("BOM file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - pomFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file - - purl := getPurlForThePom(pomFilePath) - assert.Equal(t, "", purl) - }) } diff --git a/cmd/mtaBuild.go b/cmd/mtaBuild.go index 4f1ef4c64c..0806d9ad61 100644 --- a/cmd/mtaBuild.go +++ b/cmd/mtaBuild.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "net/http" "os" "path" @@ -14,8 +15,10 @@ import ( "text/template" "time" + "github.com/SAP/jenkins-library/pkg/build" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/npm" + "github.com/SAP/jenkins-library/pkg/versioning" "github.com/SAP/jenkins-library/pkg/command" piperhttp "github.com/SAP/jenkins-library/pkg/http" @@ -53,7 +56,7 @@ const ( NEO MTABuildTarget = iota // CF ... CF MTABuildTarget = iota - //XSA ... + // XSA ... XSA MTABuildTarget = iota ) @@ -67,7 +70,7 @@ func ValueOfBuildTarget(str string) (MTABuildTarget, error) { case "XSA": return XSA, nil default: - return -1, fmt.Errorf("Unknown Platform: '%s'", str) + return -1, fmt.Errorf("unknown platform: '%s'", str) } } @@ -94,6 +97,9 @@ type mtaBuildUtils interface { SetNpmRegistries(defaultNpmRegistry string) error InstallAllDependencies(defaultNpmRegistry string) error + + Open(name string) (io.ReadWriteCloser, error) + SendRequest(method, url string, body io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) } type mtaBuildUtilsBundle struct { @@ -131,9 +137,7 @@ func newMtaBuildUtilsBundle() mtaBuildUtils { return &utils } -func mtaBuild(config mtaBuildOptions, - telemetryData *telemetry.CustomData, - commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment) { +func mtaBuild(config mtaBuildOptions, _ *telemetry.CustomData, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment) { log.Entry().Debugf("Launching mta build") utils := newMtaBuildUtilsBundle() @@ -145,39 +149,31 @@ func mtaBuild(config mtaBuildOptions, } } -func runMtaBuild(config mtaBuildOptions, - commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, - utils mtaBuildUtils) error { - - var err error - - err = handleSettingsFiles(config, utils) - if err != nil { +func runMtaBuild(config mtaBuildOptions, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, utils mtaBuildUtils) error { + if err := handleSettingsFiles(config, utils); err != nil { return err } - err = handleActiveProfileUpdate(config, utils) - if err != nil { + if err := handleActiveProfileUpdate(config, utils); err != nil { return err } - err = utils.SetNpmRegistries(config.DefaultNpmRegistry) + if err := utils.SetNpmRegistries(config.DefaultNpmRegistry); err != nil { + return err + } mtaYamlFile := filepath.Join(getSourcePath(config), "mta.yaml") mtaYamlFileExists, err := utils.FileExists(mtaYamlFile) - if err != nil { return err } if !mtaYamlFileExists { - if err = createMtaYamlFile(mtaYamlFile, config.ApplicationName, utils); err != nil { return err } - } else { - log.Entry().Infof("\"%s\" file found in project sources", mtaYamlFile) + log.Entry().Infof(`"%s" file found in project sources`, mtaYamlFile) } if config.EnableSetTimestamp { @@ -187,20 +183,17 @@ func runMtaBuild(config mtaBuildOptions, } mtarName, isMtarNativelySuffixed, err := getMtarName(config, mtaYamlFile, utils) - if err != nil { return err } - var call []string - platform, err := ValueOfBuildTarget(config.Platform) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err } - call = append(call, "mbt", "build", "--mtar", mtarName, "--platform", platform.String()) + call := []string{"mbt", "build", "--mtar", mtarName, "--platform", platform.String()} if len(config.Extensions) != 0 { call = append(call, fmt.Sprintf("--extensions=%s", config.Extensions)) } @@ -229,7 +222,7 @@ func runMtaBuild(config mtaBuildOptions, utils.AppendEnv([]string{"MAVEN_OPTS=-Dmaven.repo.local=" + absolutePath}) } - log.Entry().Infof("Executing mta build call: \"%s\"", strings.Join(call, " ")) + log.Entry().Infof(`Executing mta build call: "%s"`, strings.Join(call, " ")) if err := utils.RunExecutable(call[0], call[1:]...); err != nil { log.SetErrorCategory(log.ErrorBuild) @@ -261,65 +254,97 @@ func runMtaBuild(config mtaBuildOptions, commonPipelineEnvironment.custom.mtaBuildToolDesc = filepath.ToSlash(mtaYamlFile) if config.InstallArtifacts { - // install maven artifacts in local maven repo because `mbt build` executes `mvn package -B` - err = installMavenArtifacts(utils, config) - if err != nil { + if err = installMavenArtifacts(utils, config); err != nil { return err } - // mta-builder executes 'npm install --production', therefore we need 'npm ci/install' to install the dev-dependencies - err = utils.InstallAllDependencies(config.DefaultNpmRegistry) - if err != nil { + if err = utils.InstallAllDependencies(config.DefaultNpmRegistry); err != nil { return err } } if config.Publish { - log.Entry().Infof("publish detected") - if (len(config.MtaDeploymentRepositoryPassword) > 0) && (len(config.MtaDeploymentRepositoryUser) > 0) && - (len(config.MtaDeploymentRepositoryURL) > 0) { - if (len(config.MtarGroup) > 0) && (len(config.Version) > 0) { - httpClient := &piperhttp.Client{} + if err = handlePublish(config, commonPipelineEnvironment, utils, mtarName, isMtarNativelySuffixed); err != nil { + return err + } + } else { + log.Entry().Infof("no publish detected, skipping upload of mtar artifact") + } - credentialsEncoded := "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", config.MtaDeploymentRepositoryUser, config.MtaDeploymentRepositoryPassword))) - headers := http.Header{} - headers.Add("Authorization", credentialsEncoded) + return nil +} - config.MtarGroup = strings.ReplaceAll(config.MtarGroup, ".", "/") +func handlePublish(config mtaBuildOptions, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, utils mtaBuildUtils, mtarName string, isMtarNativelySuffixed bool) error { + log.Entry().Infof("publish detected") - mtarArtifactName := mtarName + if len(config.MtaDeploymentRepositoryPassword) == 0 || + len(config.MtaDeploymentRepositoryUser) == 0 || + len(config.MtaDeploymentRepositoryURL) == 0 { + return errors.New("mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found, must be present") + } - // only trim the .mtar suffix from the mtarName - if !isMtarNativelySuffixed { - mtarArtifactName = strings.TrimSuffix(mtarArtifactName, ".mtar") - } + if len(config.MtarGroup) == 0 || len(config.Version) == 0 { + return errors.New("mtarGroup, version not found and must be present") + } - config.MtaDeploymentRepositoryURL += config.MtarGroup + "/" + mtarArtifactName + "/" + config.Version + "/" + fmt.Sprintf("%v-%v.%v", mtarArtifactName, config.Version, "mtar") + credentialsEncoded := "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", config.MtaDeploymentRepositoryUser, config.MtaDeploymentRepositoryPassword))) + headers := http.Header{} + headers.Add("Authorization", credentialsEncoded) - commonPipelineEnvironment.custom.mtarPublishedURL = config.MtaDeploymentRepositoryURL + config.MtarGroup = strings.ReplaceAll(config.MtarGroup, ".", "/") + mtarArtifactName := mtarName + if !isMtarNativelySuffixed { + mtarArtifactName = strings.TrimSuffix(mtarArtifactName, ".mtar") + } - log.Entry().Infof("pushing mtar artifact to repository : %s", config.MtaDeploymentRepositoryURL) + config.MtaDeploymentRepositoryURL += config.MtarGroup + "/" + mtarArtifactName + "/" + config.Version + "/" + fmt.Sprintf("%v-%v.%v", mtarArtifactName, config.Version, "mtar") + commonPipelineEnvironment.custom.mtarPublishedURL = config.MtaDeploymentRepositoryURL - data, err := os.Open(getMtarFilePath(config, mtarName)) - if err != nil { - return errors.Wrap(err, "failed to open mtar archive for upload") - } - _, httpErr := httpClient.SendRequest("PUT", config.MtaDeploymentRepositoryURL, data, headers, nil) + log.Entry().Infof("pushing mtar artifact to repository : %s", config.MtaDeploymentRepositoryURL) - if httpErr != nil { - return errors.Wrap(err, "failed to upload mtar to repository") - } - } else { - return errors.New("mtarGroup, version not found and must be present") + mtarPath := getMtarFilePath(config, mtarName) + data, err := utils.Open(mtarPath) + if err != nil { + return errors.Wrap(err, "failed to open mtar archive for upload") + } + defer data.Close() - } + if _, httpErr := utils.SendRequest("PUT", config.MtaDeploymentRepositoryURL, data, headers, nil); httpErr != nil { + return errors.Wrap(httpErr, "failed to upload mtar to repository") + } - } else { - return errors.New("mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found , must be present") + if config.CreateBuildArtifactsMetadata { + if err := buildArtifactsMetadata(config, commonPipelineEnvironment, mtarPath); err != nil { + log.Entry().Warnf("unable to create build artifacts metadata: %v", err) + return nil } - } else { - log.Entry().Infof("no publish detected, skipping upload of mtar artifact") } - return err + + return nil +} + +func buildArtifactsMetadata(config mtaBuildOptions, commonPipelineEnvironment *mtaBuildCommonPipelineEnvironment, mtarPath string) error { + mtarDir := filepath.Dir(mtarPath) + buildArtifacts := build.BuildArtifacts{ + Coordinates: []versioning.Coordinates{ + { + GroupID: config.MtarGroup, + ArtifactID: config.MtarName, + Version: config.Version, + Packaging: "mtar", + BuildPath: getSourcePath(config), + URL: config.MtaDeploymentRepositoryURL, + PURL: piperutils.GetPurl(filepath.Join(mtarDir, "sbom-gen/bom-mta.xml")), + }, + }, + } + + jsonResult, err := json.Marshal(buildArtifacts) + if err != nil { + return fmt.Errorf("failed to marshal build artifacts: %v", err) + } + + commonPipelineEnvironment.custom.mtaBuildArtifacts = string(jsonResult) + return nil } func handleActiveProfileUpdate(config mtaBuildOptions, utils mtaBuildUtils) error { @@ -355,15 +380,12 @@ func addNpmBinToPath(utils mtaBuildUtils) error { } func getMtarName(config mtaBuildOptions, mtaYamlFile string, utils mtaBuildUtils) (string, bool, error) { - mtarName := config.MtarName isMtarNativelySuffixed := false if len(mtarName) == 0 { - - log.Entry().Debugf("mtar name not provided via config. Extracting from file \"%s\"", mtaYamlFile) + log.Entry().Debugf(`mtar name not provided via config. Extracting from file "%s"`, mtaYamlFile) mtaID, err := getMtaID(mtaYamlFile, utils) - if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return "", isMtarNativelySuffixed, err @@ -371,10 +393,10 @@ func getMtarName(config mtaBuildOptions, mtaYamlFile string, utils mtaBuildUtils if len(mtaID) == 0 { log.SetErrorCategory(log.ErrorConfiguration) - return "", isMtarNativelySuffixed, fmt.Errorf("Invalid mtar ID. Was empty") + return "", isMtarNativelySuffixed, fmt.Errorf("invalid mtar ID. Was empty") } - log.Entry().Debugf("mtar name extracted from file \"%s\": \"%s\"", mtaYamlFile, mtaID) + log.Entry().Debugf(`mtar name extracted from file "%s": "%s"`, mtaYamlFile, mtaID) // there can be cases where the mtaId itself has the value com.myComapany.mtar , adding an extra .mtar causes .mtar.mtar if !strings.HasSuffix(mtaID, ".mtar") { @@ -387,11 +409,9 @@ func getMtarName(config mtaBuildOptions, mtaYamlFile string, utils mtaBuildUtils } return mtarName, isMtarNativelySuffixed, nil - } func setTimeStamp(mtaYamlFile string, utils mtaBuildUtils) error { - mtaYaml, err := utils.FileRead(mtaYamlFile) if err != nil { return err @@ -406,9 +426,9 @@ func setTimeStamp(mtaYamlFile string, utils mtaBuildUtils) error { log.SetErrorCategory(log.ErrorConfiguration) return err } - log.Entry().Infof("Timestamp replaced in \"%s\"", mtaYamlFile) + log.Entry().Infof(`Timestamp replaced in "%s"`, mtaYamlFile) } else { - log.Entry().Infof("No timestamp contained in \"%s\". File has not been modified.", mtaYamlFile) + log.Entry().Infof(`No timestamp contained in "%s". File has not been modified.`, mtaYamlFile) } return nil @@ -420,14 +440,16 @@ func getTimestamp() string { } func createMtaYamlFile(mtaYamlFile, applicationName string, utils mtaBuildUtils) error { - - log.Entry().Infof("\"%s\" file not found in project sources", mtaYamlFile) + log.Entry().Infof(`"%s" file not found in project sources`, mtaYamlFile) if len(applicationName) == 0 { return fmt.Errorf("'%[1]s' not found in project sources and 'applicationName' not provided as parameter - cannot generate '%[1]s' file", mtaYamlFile) } packageFileExists, err := utils.FileExists("package.json") + if err != nil { + return err + } if !packageFileExists { return fmt.Errorf("package.json file does not exist") } @@ -437,16 +459,18 @@ func createMtaYamlFile(mtaYamlFile, applicationName string, utils mtaBuildUtils) if err != nil { return err } - json.Unmarshal(pContent, &result) + if err := json.Unmarshal(pContent, &result); err != nil { + return fmt.Errorf("failed to unmarshal package.json: %w", err) + } version, ok := result["version"].(string) if !ok { - return fmt.Errorf("Version not found in \"package.json\" (or wrong type)") + return fmt.Errorf(`version not found in "package.json" (or wrong type)`) } name, ok := result["name"].(string) if !ok { - return fmt.Errorf("Name not found in \"package.json\" (or wrong type)") + return fmt.Errorf(`name not found in "package.json" (or wrong type)`) } mtaConfig, err := generateMta(name, applicationName, version) @@ -457,7 +481,7 @@ func createMtaYamlFile(mtaYamlFile, applicationName string, utils mtaBuildUtils) if err := utils.FileWrite(mtaYamlFile, []byte(mtaConfig), 0644); err != nil { return fmt.Errorf("failed to write %v: %w", mtaYamlFile, err) } - log.Entry().Infof("\"%s\" created.", mtaYamlFile) + log.Entry().Infof(`"%s" created.`, mtaYamlFile) return nil } @@ -467,15 +491,14 @@ func handleSettingsFiles(config mtaBuildOptions, utils mtaBuildUtils) error { } func generateMta(id, applicationName, version string) (string, error) { - if len(id) == 0 { - return "", fmt.Errorf("Generating mta file: ID not provided") + return "", fmt.Errorf("generating mta file: ID not provided") } if len(applicationName) == 0 { - return "", fmt.Errorf("Generating mta file: ApplicationName not provided") + return "", fmt.Errorf("generating mta file: ApplicationName not provided") } if len(version) == 0 { - return "", fmt.Errorf("Generating mta file: Version not provided") + return "", fmt.Errorf("generating mta file: Version not provided") } tmpl, e := template.New("mta.yaml").Parse(templateMtaYml) @@ -499,7 +522,6 @@ func generateMta(id, applicationName, version string) (string, error) { } func getMtaID(mtaYamlFile string, utils mtaBuildUtils) (string, error) { - var result map[string]interface{} p, err := utils.FileRead(mtaYamlFile) if err != nil { @@ -512,7 +534,7 @@ func getMtaID(mtaYamlFile string, utils mtaBuildUtils) (string, error) { id, ok := result["ID"].(string) if !ok || len(id) == 0 { - return "", fmt.Errorf("Id not found in mta yaml file (or wrong type)") + return "", fmt.Errorf("id not found in mta yaml file (or wrong type)") } return id, nil diff --git a/cmd/mtaBuild_generated.go b/cmd/mtaBuild_generated.go index 5a15ee3b8e..3c03768c6a 100644 --- a/cmd/mtaBuild_generated.go +++ b/cmd/mtaBuild_generated.go @@ -45,6 +45,7 @@ type mtaBuildOptions struct { BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` CreateBOM bool `json:"createBOM,omitempty"` EnableSetTimestamp bool `json:"enableSetTimestamp,omitempty"` + CreateBuildArtifactsMetadata bool `json:"createBuildArtifactsMetadata,omitempty"` } type mtaBuildCommonPipelineEnvironment struct { @@ -53,6 +54,7 @@ type mtaBuildCommonPipelineEnvironment struct { mtaBuildToolDesc string mtarPublishedURL string buildSettingsInfo string + mtaBuildArtifacts string } } @@ -66,6 +68,7 @@ func (p *mtaBuildCommonPipelineEnvironment) persist(path, resourceName string) { {category: "custom", name: "mtaBuildToolDesc", value: p.custom.mtaBuildToolDesc}, {category: "custom", name: "mtarPublishedUrl", value: p.custom.mtarPublishedURL}, {category: "custom", name: "buildSettingsInfo", value: p.custom.buildSettingsInfo}, + {category: "custom", name: "mtaBuildArtifacts", value: p.custom.mtaBuildArtifacts}, } errCount := 0 @@ -266,6 +269,7 @@ func addMtaBuildFlags(cmd *cobra.Command, stepConfig *mtaBuildOptions) { cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the mta build . This information is typically used for compliance related processes.") cmd.Flags().BoolVar(&stepConfig.CreateBOM, "createBOM", false, "Creates the bill of materials (BOM) using CycloneDX plugin.") cmd.Flags().BoolVar(&stepConfig.EnableSetTimestamp, "enableSetTimestamp", true, "Enables setting the timestamp in the `mta.yaml` when it contains `${timestamp}`. Disable this when you want the MTA Deploy Service to do this instead.") + cmd.Flags().BoolVar(&stepConfig.CreateBuildArtifactsMetadata, "createBuildArtifactsMetadata", false, "metadata about the artifacts that are build and published, this metadata is generally used by steps downstream in the pipeline") } @@ -529,6 +533,15 @@ func mtaBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: true, }, + { + Name: "createBuildArtifactsMetadata", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, }, }, Containers: []config.Container{ @@ -544,6 +557,7 @@ func mtaBuildMetadata() config.StepData { {"name": "custom/mtaBuildToolDesc"}, {"name": "custom/mtarPublishedUrl"}, {"name": "custom/buildSettingsInfo"}, + {"name": "custom/mtaBuildArtifacts"}, }, }, { diff --git a/cmd/mtaBuild_test.go b/cmd/mtaBuild_test.go index 3f127c4944..273ff3af91 100644 --- a/cmd/mtaBuild_test.go +++ b/cmd/mtaBuild_test.go @@ -1,12 +1,15 @@ package cmd import ( + "bytes" "errors" + "io" "net/http" "os" "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/mock" "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" @@ -18,6 +21,8 @@ type mtaBuildTestUtilsBundle struct { projectSettingsFile string globalSettingsFile string registryUsedInSetNpmRegistries string + openReturns string + sendRequestReturns func() (*http.Response, error) } func (m *mtaBuildTestUtilsBundle) SetNpmRegistries(defaultNpmRegistry string) error { @@ -26,7 +31,7 @@ func (m *mtaBuildTestUtilsBundle) SetNpmRegistries(defaultNpmRegistry string) er } func (m *mtaBuildTestUtilsBundle) InstallAllDependencies(defaultNpmRegistry string) error { - return errors.New("Test should not install dependencies.") //TODO implement test + return errors.New("Test should not install dependencies.") // TODO implement test } func (m *mtaBuildTestUtilsBundle) DownloadAndCopySettingsFiles(globalSettingsFile string, projectSettingsFile string) error { @@ -39,6 +44,36 @@ func (m *mtaBuildTestUtilsBundle) DownloadFile(url, filename string, header http return errors.New("Test should not download files.") } +func (m *mtaBuildTestUtilsBundle) Open(name string) (io.ReadWriteCloser, error) { + if m.openReturns != "" { + return NewMockReadCloser(m.openReturns), nil + } + return nil, errors.New("Test should not open files.") +} + +// MockReadCloser is a struct that implements io.ReadCloser +type MockReadWriteCloser struct { + io.Reader + io.Writer +} + +// Close is a no-op method to satisfy the io.Closer interface +func (m *MockReadWriteCloser) Close() error { + return nil +} + +// NewMockReadCloser returns a new MockReadCloser with the given data +func NewMockReadCloser(data string) io.ReadWriteCloser { + return &MockReadWriteCloser{Reader: bytes.NewBufferString(data)} +} + +func (m *mtaBuildTestUtilsBundle) SendRequest(method, url string, body io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) { + if m.sendRequestReturns != nil { + return m.sendRequestReturns() + } + return nil, errors.New("Test should not send requests.") +} + func newMtaBuildTestUtilsBundle() *mtaBuildTestUtilsBundle { utilsBundle := mtaBuildTestUtilsBundle{ ExecMockRunner: &mock.ExecMockRunner{}, @@ -48,11 +83,11 @@ func newMtaBuildTestUtilsBundle() *mtaBuildTestUtilsBundle { } func TestMtaBuild(t *testing.T) { - cpe := mtaBuildCommonPipelineEnvironment{} - + SetConfigOptions(ConfigCommandOptions{ + OpenFile: config.OpenPiperFile, + }) t.Run("Application name not set", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{} @@ -60,15 +95,20 @@ func TestMtaBuild(t *testing.T) { assert.NotNil(t, err) assert.Equal(t, "'mta.yaml' not found in project sources and 'applicationName' not provided as parameter - cannot generate 'mta.yaml' file", err.Error()) - }) t.Run("Provide default npm registry", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", DefaultNpmRegistry: "https://example.org/npm", MtarName: "myName", Source: "./", Target: "./"} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + DefaultNpmRegistry: "https://example.org/npm", + MtarName: "myName", + Source: "./", + Target: "./", + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -78,7 +118,6 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Package json does not exist", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{ApplicationName: "myApp"} @@ -88,16 +127,21 @@ func TestMtaBuild(t *testing.T) { assert.NotNil(t, err) assert.Equal(t, "package.json file does not exist", err.Error()) - }) t.Run("Write yaml file", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName", Source: "./", Target: "./", EnableSetTimestamp: true} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + EnableSetTimestamp: true, + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -125,16 +169,14 @@ func TestMtaBuild(t *testing.T) { assert.Equal(t, "myApp", result.Modules[0].Name) assert.Regexp(t, "^1\\.2\\.3-[\\d]{14}$", result.Modules[0].Parameters["version"]) assert.Equal(t, "myApp", result.Modules[0].Parameters["name"]) - }) t.Run("Dont write mta yaml file when already present no timestamp placeholder", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{ApplicationName: "myApp"} - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) utilsMock.AddFile("mta.yaml", []byte("already there")) _ = runMtaBuild(options, &cpe, utilsMock) @@ -143,12 +185,14 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Write mta yaml file when already present with timestamp placeholder", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - options := mtaBuildOptions{ApplicationName: "myApp", EnableSetTimestamp: true} + options := mtaBuildOptions{ + ApplicationName: "myApp", + EnableSetTimestamp: true, + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) utilsMock.AddFile("mta.yaml", []byte("already there with-${timestamp}")) _ = runMtaBuild(options, &cpe, utilsMock) @@ -157,14 +201,13 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Mta build mbt toolset", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() cpe.mtarFilePath = "" options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName.mtar", Source: "./", Target: "./"} - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -179,14 +222,19 @@ func TestMtaBuild(t *testing.T) { t.Run("Source and target related tests", func(t *testing.T) { t.Run("Mta build mbt toolset with custom source and target paths", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() cpe.mtarFilePath = "" - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName.mtar", Source: "mySourcePath/", Target: "myTargetPath/"} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + MtarName: "myName.mtar", + Source: "mySourcePath/", + Target: "myTargetPath/", + } - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -194,9 +242,11 @@ func TestMtaBuild(t *testing.T) { if assert.Len(t, utilsMock.Calls, 1) { assert.Equal(t, "mbt", utilsMock.Calls[0].Exec) - assert.Equal(t, []string{"build", "--mtar", "myName.mtar", "--platform", "CF", + assert.Equal(t, []string{ + "build", "--mtar", "myName.mtar", "--platform", "CF", "--source", filepath.FromSlash("mySourcePath/"), - "--target", filepath.Join(_ignoreError(os.Getwd()), filepath.FromSlash("mySourcePath/myTargetPath/"))}, + "--target", filepath.Join(_ignoreError(os.Getwd()), filepath.FromSlash("mySourcePath/myTargetPath/")), + }, utilsMock.Calls[0].Params) } assert.Equal(t, "mySourcePath/myTargetPath/myName.mtar", cpe.mtarFilePath) @@ -206,14 +256,20 @@ func TestMtaBuild(t *testing.T) { t.Run("M2Path related tests", func(t *testing.T) { t.Run("Mta build mbt toolset with m2Path", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() utilsMock.CurrentDir = "root_folder/workspace" cpe.mtarFilePath = "" - options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", MtarName: "myName.mtar", Source: "./", Target: "./", M2Path: ".pipeline/local_repo"} + options := mtaBuildOptions{ + ApplicationName: "myApp", + Platform: "CF", + MtarName: "myName.mtar", + Source: "./", + Target: "./", + M2Path: ".pipeline/local_repo", + } - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) err := runMtaBuild(options, &cpe, utilsMock) @@ -223,13 +279,18 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Settings file releatd tests", func(t *testing.T) { - t.Run("Copy global settings file", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) - - options := mtaBuildOptions{ApplicationName: "myApp", GlobalSettingsFile: "/opt/maven/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./"} + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + } err := runMtaBuild(options, &cpe, utilsMock) @@ -240,9 +301,8 @@ func TestMtaBuild(t *testing.T) { }) t.Run("Copy project settings file", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) options := mtaBuildOptions{ApplicationName: "myApp", ProjectSettingsFile: "/my/project/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./"} @@ -256,36 +316,102 @@ func TestMtaBuild(t *testing.T) { }) t.Run("publish related tests", func(t *testing.T) { - t.Run("error when no repository url", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) - - options := mtaBuildOptions{ApplicationName: "myApp", GlobalSettingsFile: "/opt/maven/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./", Publish: true} + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + Publish: true, + } err := runMtaBuild(options, &cpe, utilsMock) - assert.Equal(t, "mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found , must be present", err.Error()) + assert.Equal(t, "mtaDeploymentRepositoryUser, mtaDeploymentRepositoryPassword and mtaDeploymentRepositoryURL not found, must be present", err.Error()) }) t.Run("error when no mtar group", func(t *testing.T) { - utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"myNameFromMtar\"")) - - options := mtaBuildOptions{ApplicationName: "myApp", GlobalSettingsFile: "/opt/maven/settings.xml", Platform: "CF", MtarName: "myName", Source: "./", Target: "./", Publish: true, - MtaDeploymentRepositoryURL: "dummy", MtaDeploymentRepositoryPassword: "dummy", MtaDeploymentRepositoryUser: "dummy"} + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "myName", + Source: "./", + Target: "./", + Publish: true, + MtaDeploymentRepositoryURL: "dummy", + MtaDeploymentRepositoryPassword: "dummy", + MtaDeploymentRepositoryUser: "dummy", + } err := runMtaBuild(options, &cpe, utilsMock) assert.Equal(t, "mtarGroup, version not found and must be present", err.Error()) }) + + t.Run("successful publish", func(t *testing.T) { + utilsMock := newMtaBuildTestUtilsBundle() + utilsMock.sendRequestReturns = func() (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil + } + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + utilsMock.openReturns = `{"version":"1.2.3"}` + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "test", + Source: "./", + Target: "./", + Publish: true, + MtaDeploymentRepositoryURL: "dummy", + MtaDeploymentRepositoryPassword: "dummy", + MtaDeploymentRepositoryUser: "dummy", + MtarGroup: "dummy", + Version: "dummy", + } + err := runMtaBuild(options, &cpe, utilsMock) + assert.Nil(t, err) + }) + + t.Run("succesful build artifact", func(t *testing.T) { + utilsMock := newMtaBuildTestUtilsBundle() + utilsMock.AddFile("mta.yaml", []byte(`ID: "myNameFromMtar"`)) + utilsMock.openReturns = `{"version":"1.2.3"}` + utilsMock.sendRequestReturns = func() (*http.Response, error) { + return &http.Response{StatusCode: 200}, nil + } + options := mtaBuildOptions{ + ApplicationName: "myApp", + GlobalSettingsFile: "/opt/maven/settings.xml", + Platform: "CF", + MtarName: "test", + Source: "./", + Target: "./", + Publish: true, + MtaDeploymentRepositoryURL: "dummy", + MtaDeploymentRepositoryPassword: "dummy", + MtaDeploymentRepositoryUser: "dummy", + MtarGroup: "dummy", + Version: "dummy", + CreateBuildArtifactsMetadata: true, + } + err := runMtaBuild(options, &cpe, utilsMock) + assert.Nil(t, err) + assert.Equal(t, cpe.custom.mtaBuildArtifacts, `{"Coordinates":[{"groupId":"dummy","artifactId":"test","version":"dummy","packaging":"mtar","buildPath":"./","url":"dummydummy/test/dummy/test-dummy.mtar","purl":""}]}`) + }) }) } func TestMtaBuildSourceDir(t *testing.T) { - cpe := mtaBuildCommonPipelineEnvironment{} t.Run("getSourcePath", func(t *testing.T) { t.Parallel() @@ -338,7 +464,7 @@ func TestMtaBuildSourceDir(t *testing.T) { t.Run("create mta.yaml from config.source", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) _ = runMtaBuild(mtaBuildOptions{ApplicationName: "myApp", Source: "create"}, &cpe, utilsMock) @@ -359,35 +485,33 @@ func TestMtaBuildSourceDir(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() options := mtaBuildOptions{ApplicationName: "myApp", Platform: "CF", DefaultNpmRegistry: "https://example.org/npm", MtarName: "myName", Source: "./", Target: "./", CreateBOM: true} - utilsMock.AddFile("package.json", []byte("{\"name\": \"myName\", \"version\": \"1.2.3\"}")) + utilsMock.AddFile("package.json", []byte(`{"name": "myName", "version": "1.2.3"}`)) err := runMtaBuild(options, &cpe, utilsMock) assert.Nil(t, err) assert.Contains(t, utilsMock.Calls[0].Params, "--sbom-file-path") - }) } func TestMtaBuildMtar(t *testing.T) { - t.Run("getMtarName", func(t *testing.T) { t.Parallel() t.Run("mtar name from yaml", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"nameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "nameFromMtar"`)) assert.Equal(t, filepath.FromSlash("nameFromMtar.mtar"), _ignoreErrorForGetMtarName(getMtarName(mtaBuildOptions{MtarName: ""}, "mta.yaml", utilsMock))) }) t.Run("mtar name from yaml with suffixed value", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"nameFromMtar.mtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "nameFromMtar.mtar"`)) assert.Equal(t, filepath.FromSlash("nameFromMtar.mtar"), _ignoreErrorForGetMtarName(getMtarName(mtaBuildOptions{MtarName: ""}, "mta.yaml", utilsMock))) }) t.Run("mtar name from config", func(t *testing.T) { utilsMock := newMtaBuildTestUtilsBundle() - utilsMock.AddFile("mta.yaml", []byte("ID: \"nameFromMtar\"")) + utilsMock.AddFile("mta.yaml", []byte(`ID: "nameFromMtar"`)) assert.Equal(t, filepath.FromSlash("nameFromConfig.mtar"), _ignoreErrorForGetMtarName(getMtarName(mtaBuildOptions{MtarName: "nameFromConfig.mtar"}, "mta.yaml", utilsMock))) }) @@ -412,7 +536,6 @@ func TestMtaBuildMtar(t *testing.T) { assert.Equal(t, filepath.FromSlash("source/target/mta.mtar"), getMtarFilePath(mtaBuildOptions{Source: "source", Target: "target"}, "mta.mtar")) }) }) - } func _ignoreError(s string, e error) string { diff --git a/pkg/npm/publish.go b/pkg/npm/publish.go index 3466e89aea..a5626281b3 100644 --- a/pkg/npm/publish.go +++ b/pkg/npm/publish.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/SAP/jenkins-library/pkg/log" + "github.com/SAP/jenkins-library/pkg/piperutils" CredentialUtils "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/versioning" ) @@ -217,7 +218,7 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p coordinate.BuildPath = filepath.Dir(packageJSON) coordinate.URL = registry coordinate.Packaging = "tgz" - coordinate.PURL = getPurl(packageJSON) + coordinate.PURL = piperutils.GetPurl(filepath.Join(filepath.Dir(packageJSON), npmBomFilename)) *buildCoordinates = append(*buildCoordinates, coordinate) } @@ -226,21 +227,6 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p return nil } -func getPurl(packageJSON string) string { - expectedBomFilePath := filepath.Join(filepath.Dir(packageJSON) + "/" + npmBomFilename) - exists, _ := CredentialUtils.FileExists(expectedBomFilePath) - if !exists { - log.Entry().Debugf("bom file doesn't exist and hence no pURL info: %v", expectedBomFilePath) - return "" - } - bom, err := CredentialUtils.GetBom(expectedBomFilePath) - if err != nil { - log.Entry().Warnf("unable to get bom metdata : %v", err) - return "" - } - return bom.Metadata.Component.Purl -} - func (exec *Execute) readPackageScope(packageJSON string) (string, error) { b, err := exec.Utils.FileRead(packageJSON) if err != nil { diff --git a/pkg/npm/publish_test.go b/pkg/npm/publish_test.go index 4c7b24f95d..bdbdf8b655 100644 --- a/pkg/npm/publish_test.go +++ b/pkg/npm/publish_test.go @@ -4,15 +4,15 @@ package npm import ( - "github.com/SAP/jenkins-library/pkg/mock" "io" "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/versioning" "github.com/stretchr/testify/assert" - "os" ) type npmMockUtilsBundleRelativeGlob struct { @@ -531,7 +531,7 @@ func TestNpmPublish(t *testing.T) { // This stub simulates the behavior of npm pack and puts a tgz into the requested utils.execRunner.Stub = func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error { - //tgzTargetPath := filepath.Dir(test.packageDescriptors[0]) + // tgzTargetPath := filepath.Dir(test.packageDescriptors[0]) utils.AddFile(filepath.Join(".", "package.tgz"), []byte("this is a tgz file")) return nil } @@ -574,46 +574,3 @@ func TestNpmPublish(t *testing.T) { }) } } - -func createTempFile(t *testing.T, dir string, filename string, content string) string { - filePath := filepath.Join(dir, filename) - err := os.WriteFile(filePath, []byte(content), 0666) - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } - return filePath -} - -func TestGetPurl(t *testing.T) { - t.Run("valid BOM file", func(t *testing.T) { - tempDir, err := piperutils.Files{}.TempDir("", "test") - if err != nil { - t.Fatalf("Failed to create temp directory: %s", err) - } - - bomContent := ` - - - pkg:npm/com.example/mycomponent@1.0.0 - - - - - - ` - packageJsonFilePath := createTempFile(t, tempDir, "package.json", "") - bomFilePath := createTempFile(t, tempDir, npmBomFilename, bomContent) - defer os.Remove(bomFilePath) - - purl := getPurl(packageJsonFilePath) - assert.Equal(t, "pkg:npm/com.example/mycomponent@1.0.0", purl) - }) - - t.Run("BOM file does not exist", func(t *testing.T) { - tempDir := t.TempDir() - packageJsonFilePath := createTempFile(t, tempDir, "pom.xml", "") // Create a temp pom file - - purl := getPurl(packageJsonFilePath) - assert.Equal(t, "", purl) - }) -} diff --git a/pkg/piperutils/cyclonedxBom.go b/pkg/piperutils/cyclonedxBom.go index fddebae8a6..51c8685235 100644 --- a/pkg/piperutils/cyclonedxBom.go +++ b/pkg/piperutils/cyclonedxBom.go @@ -2,9 +2,10 @@ package piperutils import ( "encoding/xml" - "github.com/SAP/jenkins-library/pkg/log" "io" "os" + + "github.com/SAP/jenkins-library/pkg/log" ) // To serialize the cyclonedx BOM file @@ -46,3 +47,12 @@ func GetBom(absoluteBomPath string) (Bom, error) { } return bom, nil } + +func GetPurl(bomFilePath string) string { + bom, err := GetBom(bomFilePath) + if err != nil { + log.Entry().Warnf("unable to get bom metadata: %v", err) + return "" + } + return bom.Metadata.Component.Purl +} diff --git a/pkg/piperutils/cyclonedxbom_test.go b/pkg/piperutils/cyclonedxbom_test.go index d9d25e6256..d4f6824293 100644 --- a/pkg/piperutils/cyclonedxbom_test.go +++ b/pkg/piperutils/cyclonedxbom_test.go @@ -18,6 +18,18 @@ func createTempFile(t *testing.T, content string) (string, func()) { } } +const validBom = ` + + + pkg:maven/com.example/mycomponent@1.0.0 + + + + + + + ` + func TestGetBom(t *testing.T) { tests := []struct { name string @@ -27,18 +39,8 @@ func TestGetBom(t *testing.T) { expectedError string }{ { - name: "valid file", - xmlContent: ` - - - pkg:maven/com.example/mycomponent@1.0.0 - - - - - - - `, + name: "valid file", + xmlContent: validBom, expectedBom: Bom{ Metadata: Metadata{ Component: BomComponent{ @@ -73,12 +75,8 @@ func TestGetBom(t *testing.T) { var fileName string var cleanup func() if tt.xmlContent != "" { - var err error fileName, cleanup = createTempFile(t, tt.xmlContent) defer cleanup() - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } } else { // Use a non-existent file path fileName = "nonexistent.xml" @@ -102,6 +100,57 @@ func TestGetBom(t *testing.T) { } } +func TestGetPurl(t *testing.T) { + tests := []struct { + name string + filePath string + bomFilename string + xmlContent string + expectedPurl string + expectError bool + expectedError string + }{ + { + name: "valid BOM file", + xmlContent: validBom, + expectedPurl: "pkg:maven/com.example/mycomponent@1.0.0", + }, + { + name: "BOM file not found", + xmlContent: "", + expectedPurl: "", + expectError: true, + expectedError: "no such file or directory", + }, + { + name: "invalid BOM file", + xmlContent: "invalid xml", + expectedPurl: "", + expectError: true, + expectedError: "XML syntax error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var filePath string + var cleanup func() + if tt.xmlContent != "" { + filePath, cleanup = createTempFile(t, tt.xmlContent) + defer cleanup() + } else { + // Use a non-existent file path + filePath = "nonexistent.xml" + } + + purl := GetPurl(filePath) + if purl != tt.expectedPurl { + t.Errorf("Expected PURL: %v, got: %v", tt.expectedPurl, purl) + } + }) + } +} + func bomEquals(a, b Bom) bool { // compare a and b manually since reflect.DeepEqual can be problematic with slices and nil values return a.Metadata.Component.Purl == b.Metadata.Component.Purl && diff --git a/resources/metadata/mtaBuild.yaml b/resources/metadata/mtaBuild.yaml index acbe5c06a7..67d8fcdb3b 100644 --- a/resources/metadata/mtaBuild.yaml +++ b/resources/metadata/mtaBuild.yaml @@ -244,6 +244,14 @@ spec: - STAGES - PARAMETERS default: true + - name: createBuildArtifactsMetadata + type: bool + default: false + description: metadata about the artifacts that are build and published, this metadata is generally used by steps downstream in the pipeline + scope: + - STEPS + - STAGES + - PARAMETERS outputs: resources: - name: commonPipelineEnvironment @@ -253,6 +261,7 @@ spec: - name: custom/mtaBuildToolDesc - name: custom/mtarPublishedUrl - name: custom/buildSettingsInfo + - name: custom/mtaBuildArtifacts - name: reports type: reports params: From d4e298464e978a5505e3673ef5db08b6dec94b03 Mon Sep 17 00:00:00 2001 From: Holger Partsch Date: Mon, 4 Nov 2024 14:05:47 +0100 Subject: [PATCH 36/43] Final round of adding quoting to prevent command injection (#5167) * refactor: use import alias * fix: add quoting to further shell steps --- vars/npmExecute.groovy | 4 +++- vars/piperExecuteBin.groovy | 5 +++-- vars/sonarExecuteScan.groovy | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/vars/npmExecute.groovy b/vars/npmExecute.groovy index 75eab2b6d0..f2dd21f3bc 100644 --- a/vars/npmExecute.groovy +++ b/vars/npmExecute.groovy @@ -1,4 +1,6 @@ import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q + import com.sap.piper.GenerateDocumentation import com.sap.piper.ConfigurationHelper import com.sap.piper.Utils @@ -65,7 +67,7 @@ void call(Map parameters = [:], body = null) { npm --version """ if (configuration.defaultNpmRegistry) { - sh "npm config set registry ${configuration.defaultNpmRegistry}" + sh "npm config set registry ${q(configuration.defaultNpmRegistry)}" } if (configuration.npmCommand) { sh "npm ${configuration.npmCommand}" diff --git a/vars/piperExecuteBin.groovy b/vars/piperExecuteBin.groovy index 87eec381a5..311242c81a 100644 --- a/vars/piperExecuteBin.groovy +++ b/vars/piperExecuteBin.groovy @@ -9,6 +9,7 @@ import com.sap.piper.analytics.InfluxData import groovy.transform.Field import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q @Field String STEP_NAME = getClass().getName() @@ -132,7 +133,7 @@ static String getCustomDefaultConfigs() { // resources by setupCommonPipelineEnvironment.groovy into .pipeline/. List customDefaults = DefaultValueCache.getInstance().getCustomDefaults() for (int i = 0; i < customDefaults.size(); i++) { - customDefaults[i] = BashUtils.quoteAndEscape(".pipeline/${customDefaults[i]}") + customDefaults[i] = q(".pipeline/${customDefaults[i]}") } return customDefaults.join(',') } @@ -151,7 +152,7 @@ static String getCustomConfigArg(def script) { if (script?.commonPipelineEnvironment?.configurationFile && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yml' && script.commonPipelineEnvironment.configurationFile != '.pipeline/config.yaml') { - return " --customConfig ${BashUtils.quoteAndEscape(script.commonPipelineEnvironment.configurationFile)}" + return " --customConfig ${q(script.commonPipelineEnvironment.configurationFile)}" } return '' } diff --git a/vars/sonarExecuteScan.groovy b/vars/sonarExecuteScan.groovy index 94d4c4fff6..dcb59e5cb8 100644 --- a/vars/sonarExecuteScan.groovy +++ b/vars/sonarExecuteScan.groovy @@ -1,7 +1,9 @@ +import static com.sap.piper.Prerequisites.checkScript +import static com.sap.piper.BashUtils.quoteAndEscape as q + import com.sap.piper.JenkinsUtils import com.sap.piper.Utils import com.sap.piper.analytics.InfluxData -import static com.sap.piper.Prerequisites.checkScript import groovy.transform.Field import java.nio.charset.StandardCharsets @@ -40,7 +42,7 @@ void call(Map parameters = [:]) { // & `legacyPRHandling` & `inferBranchName` // writePipelineEnv needs to be called here as owner and repository may come from the pipeline environment writePipelineEnv(script: script, piperGoPath: piperGoPath) - Map stepConfig = readJSON(text: sh(returnStdout: true, script: "${piperGoPath} getConfig --stepMetadata '.pipeline/tmp/${METADATA_FILE}'${customDefaultConfig}${customConfigArg}")) + Map stepConfig = readJSON(text: sh(returnStdout: true, script: "${piperGoPath} getConfig --stepMetadata ${q('.pipeline/tmp/' + METADATA_FILE)}${customDefaultConfig}${customConfigArg}")) echo "Step Config: ${stepConfig}" List environment = [] @@ -135,7 +137,7 @@ private void loadCertificates(Map config) { def filename = new File(url).getName() filename = URLDecoder.decode(filename, StandardCharsets.UTF_8.name()) sh "wget ${wgetOptions.join(' ')} ${url}" - sh "keytool ${keytoolOptions.join(' ')} -alias '${filename}' -file '${certificateFolder}${filename}'" + sh "keytool ${keytoolOptions.join(' ')} -alias ${q(filename)} -file '${certificateFolder}${filename}'" } } } From 364238f1541540bb2e135b56a87ad9962acd649c Mon Sep 17 00:00:00 2001 From: Manjunath Date: Mon, 4 Nov 2024 14:20:37 +0100 Subject: [PATCH 37/43] handle error while fetching working directory (#5168) --- cmd/abapAddonAssemblyKitCheckCVs_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitCheckPV_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitCheck_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitCreateTargetVector_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitPublishTargetVector_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitRegisterPackages_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitReleasePackages_generated.go | 7 +++++-- cmd/abapAddonAssemblyKitReserveNextPackages_generated.go | 7 +++++-- cmd/abapEnvironmentAssembleConfirm_generated.go | 7 +++++-- cmd/abapEnvironmentAssemblePackages_generated.go | 7 +++++-- cmd/abapEnvironmentBuild_generated.go | 7 +++++-- cmd/abapEnvironmentCheckoutBranch_generated.go | 7 +++++-- cmd/abapEnvironmentCloneGitRepo_generated.go | 7 +++++-- cmd/abapEnvironmentCreateSystem_generated.go | 7 +++++-- cmd/abapEnvironmentCreateTag_generated.go | 7 +++++-- cmd/abapEnvironmentPullGitRepo_generated.go | 7 +++++-- cmd/abapEnvironmentPushATCSystemConfig_generated.go | 7 +++++-- cmd/abapEnvironmentRunATCCheck_generated.go | 7 +++++-- cmd/abapEnvironmentRunAUnitTest_generated.go | 7 +++++-- cmd/abapLandscapePortalUpdateAddOnProduct_generated.go | 7 +++++-- cmd/ansSendEvent_generated.go | 7 +++++-- cmd/apiKeyValueMapDownload_generated.go | 7 +++++-- cmd/apiKeyValueMapUpload_generated.go | 7 +++++-- cmd/apiProviderDownload_generated.go | 7 +++++-- cmd/apiProviderList_generated.go | 7 +++++-- cmd/apiProviderUpload_generated.go | 7 +++++-- cmd/apiProxyDownload_generated.go | 7 +++++-- cmd/apiProxyList_generated.go | 7 +++++-- cmd/apiProxyUpload_generated.go | 7 +++++-- cmd/artifactPrepareVersion_generated.go | 7 +++++-- cmd/ascAppUpload_generated.go | 7 +++++-- cmd/awsS3Upload_generated.go | 7 +++++-- cmd/azureBlobUpload_generated.go | 7 +++++-- cmd/batsExecuteTests_generated.go | 7 +++++-- cmd/checkmarxExecuteScan_generated.go | 7 +++++-- cmd/checkmarxOneExecuteScan_generated.go | 7 +++++-- cmd/cloudFoundryCreateServiceKey_generated.go | 7 +++++-- cmd/cloudFoundryCreateService_generated.go | 7 +++++-- cmd/cloudFoundryCreateSpace_generated.go | 7 +++++-- cmd/cloudFoundryDeleteService_generated.go | 7 +++++-- cmd/cloudFoundryDeleteSpace_generated.go | 7 +++++-- cmd/cloudFoundryDeploy_generated.go | 7 +++++-- cmd/cnbBuild_generated.go | 7 +++++-- cmd/codeqlExecuteScan_generated.go | 7 +++++-- cmd/containerExecuteStructureTests_generated.go | 7 +++++-- cmd/containerSaveImage_generated.go | 7 +++++-- cmd/contrastExecuteScan_generated.go | 7 +++++-- cmd/credentialdiggerScan_generated.go | 7 +++++-- cmd/detectExecuteScan_generated.go | 7 +++++-- cmd/fortifyExecuteScan_generated.go | 7 +++++-- cmd/gaugeExecuteTests_generated.go | 7 +++++-- cmd/gcpPublishEvent_generated.go | 7 +++++-- cmd/gctsCloneRepository_generated.go | 7 +++++-- cmd/gctsCreateRepository_generated.go | 7 +++++-- cmd/gctsDeploy_generated.go | 7 +++++-- cmd/gctsExecuteABAPQualityChecks_generated.go | 7 +++++-- cmd/gctsExecuteABAPUnitTests_generated.go | 7 +++++-- cmd/gctsRollback_generated.go | 7 +++++-- cmd/githubCheckBranchProtection_generated.go | 7 +++++-- cmd/githubCommentIssue_generated.go | 7 +++++-- cmd/githubCreateIssue_generated.go | 7 +++++-- cmd/githubCreatePullRequest_generated.go | 7 +++++-- cmd/githubPublishRelease_generated.go | 7 +++++-- cmd/githubSetCommitStatus_generated.go | 7 +++++-- cmd/gitopsUpdateDeployment_generated.go | 7 +++++-- cmd/golangBuild_generated.go | 7 +++++-- cmd/gradleExecuteBuild_generated.go | 7 +++++-- cmd/hadolintExecute_generated.go | 7 +++++-- cmd/helmExecute_generated.go | 7 +++++-- cmd/imagePushToRegistry_generated.go | 7 +++++-- cmd/influxWriteData_generated.go | 7 +++++-- cmd/integrationArtifactDeploy_generated.go | 7 +++++-- cmd/integrationArtifactDownload_generated.go | 7 +++++-- cmd/integrationArtifactGetMplStatus_generated.go | 7 +++++-- cmd/integrationArtifactGetServiceEndpoint_generated.go | 7 +++++-- cmd/integrationArtifactResource_generated.go | 7 +++++-- cmd/integrationArtifactTransport_generated.go | 7 +++++-- cmd/integrationArtifactTriggerIntegrationTest_generated.go | 7 +++++-- cmd/integrationArtifactUnDeploy_generated.go | 7 +++++-- cmd/integrationArtifactUpdateConfiguration_generated.go | 7 +++++-- cmd/integrationArtifactUpload_generated.go | 7 +++++-- cmd/isChangeInDevelopment_generated.go | 7 +++++-- cmd/jsonApplyPatch_generated.go | 7 +++++-- cmd/kanikoExecute_generated.go | 7 +++++-- cmd/karmaExecuteTests_generated.go | 7 +++++-- cmd/kubernetesDeploy_generated.go | 7 +++++-- cmd/malwareExecuteScan_generated.go | 7 +++++-- cmd/mavenBuild_generated.go | 7 +++++-- cmd/mavenExecuteIntegration_generated.go | 7 +++++-- cmd/mavenExecuteStaticCodeChecks_generated.go | 7 +++++-- cmd/mavenExecute_generated.go | 7 +++++-- cmd/mtaBuild_generated.go | 7 +++++-- cmd/newmanExecute_generated.go | 7 +++++-- cmd/nexusUpload_generated.go | 7 +++++-- cmd/npmExecuteLint_generated.go | 7 +++++-- cmd/npmExecuteScripts_generated.go | 7 +++++-- cmd/pipelineCreateScanSummary_generated.go | 7 +++++-- cmd/protecodeExecuteScan_generated.go | 7 +++++-- cmd/pythonBuild_generated.go | 7 +++++-- cmd/shellExecute_generated.go | 7 +++++-- cmd/sonarExecuteScan_generated.go | 7 +++++-- cmd/terraformExecute_generated.go | 7 +++++-- cmd/tmsExport_generated.go | 7 +++++-- cmd/tmsUpload_generated.go | 7 +++++-- cmd/transportRequestDocIDFromGit_generated.go | 7 +++++-- cmd/transportRequestReqIDFromGit_generated.go | 7 +++++-- cmd/transportRequestUploadCTS_generated.go | 7 +++++-- cmd/transportRequestUploadRFC_generated.go | 7 +++++-- cmd/transportRequestUploadSOLMAN_generated.go | 7 +++++-- cmd/uiVeri5ExecuteTests_generated.go | 7 +++++-- cmd/vaultRotateSecretId_generated.go | 7 +++++-- cmd/whitesourceExecuteScan_generated.go | 7 +++++-- cmd/xsDeploy_generated.go | 7 +++++-- pkg/generator/helper/helper.go | 7 +++++-- .../helper/testdata/TestProcessMetaFiles/README.md | 1 + .../TestProcessMetaFiles/custom_step_code_generated.golden | 7 +++++-- .../TestProcessMetaFiles/step_code_generated.golden | 7 +++++-- 117 files changed, 581 insertions(+), 232 deletions(-) create mode 100644 pkg/generator/helper/testdata/TestProcessMetaFiles/README.md diff --git a/cmd/abapAddonAssemblyKitCheckCVs_generated.go b/cmd/abapAddonAssemblyKitCheckCVs_generated.go index 651d25e833..9f1af748f1 100644 --- a/cmd/abapAddonAssemblyKitCheckCVs_generated.go +++ b/cmd/abapAddonAssemblyKitCheckCVs_generated.go @@ -84,11 +84,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitCheckPV_generated.go b/cmd/abapAddonAssemblyKitCheckPV_generated.go index c3d4ac2206..68cbc03785 100644 --- a/cmd/abapAddonAssemblyKitCheckPV_generated.go +++ b/cmd/abapAddonAssemblyKitCheckPV_generated.go @@ -84,11 +84,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitCheck_generated.go b/cmd/abapAddonAssemblyKitCheck_generated.go index 0dec1e978e..6bf2fc877c 100644 --- a/cmd/abapAddonAssemblyKitCheck_generated.go +++ b/cmd/abapAddonAssemblyKitCheck_generated.go @@ -86,11 +86,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go index 2d237e7764..92dfb268f2 100644 --- a/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitCreateTargetVector_generated.go @@ -85,11 +85,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go b/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go index 87de4808af..3b99470eb7 100644 --- a/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go +++ b/cmd/abapAddonAssemblyKitPublishTargetVector_generated.go @@ -56,11 +56,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go index 3d2eeffa73..71cdfd42bb 100644 --- a/cmd/abapAddonAssemblyKitRegisterPackages_generated.go +++ b/cmd/abapAddonAssemblyKitRegisterPackages_generated.go @@ -86,11 +86,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitReleasePackages_generated.go b/cmd/abapAddonAssemblyKitReleasePackages_generated.go index ca9f78c1fc..2a363fc4e9 100644 --- a/cmd/abapAddonAssemblyKitReleasePackages_generated.go +++ b/cmd/abapAddonAssemblyKitReleasePackages_generated.go @@ -86,11 +86,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go index 58f6e627b4..7d48f492b1 100644 --- a/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go +++ b/cmd/abapAddonAssemblyKitReserveNextPackages_generated.go @@ -90,11 +90,14 @@ For Terminology refer to the [Scenario Description](https://www.project-piper.io GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentAssembleConfirm_generated.go b/cmd/abapEnvironmentAssembleConfirm_generated.go index 9b7bfafdc2..b7f6880a97 100644 --- a/cmd/abapEnvironmentAssembleConfirm_generated.go +++ b/cmd/abapEnvironmentAssembleConfirm_generated.go @@ -83,11 +83,14 @@ func AbapEnvironmentAssembleConfirmCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentAssemblePackages_generated.go b/cmd/abapEnvironmentAssemblePackages_generated.go index e9ac906aaf..adfb53fea4 100644 --- a/cmd/abapEnvironmentAssemblePackages_generated.go +++ b/cmd/abapEnvironmentAssemblePackages_generated.go @@ -85,11 +85,14 @@ Platform ABAP Environment system and saves the corresponding [SAR archive](https GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentBuild_generated.go b/cmd/abapEnvironmentBuild_generated.go index c87a380f8d..85a20bc2b5 100644 --- a/cmd/abapEnvironmentBuild_generated.go +++ b/cmd/abapEnvironmentBuild_generated.go @@ -98,11 +98,14 @@ func AbapEnvironmentBuildCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCheckoutBranch_generated.go b/cmd/abapEnvironmentCheckoutBranch_generated.go index 06fc95284f..342b11225e 100644 --- a/cmd/abapEnvironmentCheckoutBranch_generated.go +++ b/cmd/abapEnvironmentCheckoutBranch_generated.go @@ -59,11 +59,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCloneGitRepo_generated.go b/cmd/abapEnvironmentCloneGitRepo_generated.go index c8759f5b83..e1da421535 100644 --- a/cmd/abapEnvironmentCloneGitRepo_generated.go +++ b/cmd/abapEnvironmentCloneGitRepo_generated.go @@ -62,11 +62,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCreateSystem_generated.go b/cmd/abapEnvironmentCreateSystem_generated.go index 9928abc03c..7cf1d951f8 100644 --- a/cmd/abapEnvironmentCreateSystem_generated.go +++ b/cmd/abapEnvironmentCreateSystem_generated.go @@ -58,11 +58,14 @@ func AbapEnvironmentCreateSystemCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentCreateTag_generated.go b/cmd/abapEnvironmentCreateTag_generated.go index 22e9260669..2d0a2daa30 100644 --- a/cmd/abapEnvironmentCreateTag_generated.go +++ b/cmd/abapEnvironmentCreateTag_generated.go @@ -62,11 +62,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentPullGitRepo_generated.go b/cmd/abapEnvironmentPullGitRepo_generated.go index 05cc2a0c21..d763185c8d 100644 --- a/cmd/abapEnvironmentPullGitRepo_generated.go +++ b/cmd/abapEnvironmentPullGitRepo_generated.go @@ -61,11 +61,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentPushATCSystemConfig_generated.go b/cmd/abapEnvironmentPushATCSystemConfig_generated.go index c33c2d9eb9..ad1834feda 100644 --- a/cmd/abapEnvironmentPushATCSystemConfig_generated.go +++ b/cmd/abapEnvironmentPushATCSystemConfig_generated.go @@ -56,11 +56,14 @@ Please provide either of the following options: GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentRunATCCheck_generated.go b/cmd/abapEnvironmentRunATCCheck_generated.go index 04a52ad14d..43983a3115 100644 --- a/cmd/abapEnvironmentRunATCCheck_generated.go +++ b/cmd/abapEnvironmentRunATCCheck_generated.go @@ -61,11 +61,14 @@ Regardless of the option you chose, please make sure to provide the configuratio GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapEnvironmentRunAUnitTest_generated.go b/cmd/abapEnvironmentRunAUnitTest_generated.go index a3e865cd2d..19e35e7f5d 100644 --- a/cmd/abapEnvironmentRunAUnitTest_generated.go +++ b/cmd/abapEnvironmentRunAUnitTest_generated.go @@ -60,11 +60,14 @@ Regardless of the option you chose, please make sure to provide the object set c GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go b/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go index a44196072b..6be7825b22 100644 --- a/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go +++ b/cmd/abapLandscapePortalUpdateAddOnProduct_generated.go @@ -44,11 +44,14 @@ func AbapLandscapePortalUpdateAddOnProductCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/ansSendEvent_generated.go b/cmd/ansSendEvent_generated.go index 9993c7745a..3f4e002034 100644 --- a/cmd/ansSendEvent_generated.go +++ b/cmd/ansSendEvent_generated.go @@ -53,11 +53,14 @@ func AnsSendEventCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiKeyValueMapDownload_generated.go b/cmd/apiKeyValueMapDownload_generated.go index 2ced0e9124..d69d948a8e 100644 --- a/cmd/apiKeyValueMapDownload_generated.go +++ b/cmd/apiKeyValueMapDownload_generated.go @@ -45,11 +45,14 @@ Learn more about the SAP API Management API for downloading an Key Value Map art GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiKeyValueMapUpload_generated.go b/cmd/apiKeyValueMapUpload_generated.go index c62d559b3f..75e0456bf8 100644 --- a/cmd/apiKeyValueMapUpload_generated.go +++ b/cmd/apiKeyValueMapUpload_generated.go @@ -46,11 +46,14 @@ Learn more about the SAP API Management API for creating an API key value map ar GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProviderDownload_generated.go b/cmd/apiProviderDownload_generated.go index def4eb576c..b972acde58 100644 --- a/cmd/apiProviderDownload_generated.go +++ b/cmd/apiProviderDownload_generated.go @@ -44,11 +44,14 @@ func ApiProviderDownloadCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProviderList_generated.go b/cmd/apiProviderList_generated.go index 76ce80c0f1..cb9ee968b2 100644 --- a/cmd/apiProviderList_generated.go +++ b/cmd/apiProviderList_generated.go @@ -81,11 +81,14 @@ func ApiProviderListCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProviderUpload_generated.go b/cmd/apiProviderUpload_generated.go index 1b1728099f..daf844d013 100644 --- a/cmd/apiProviderUpload_generated.go +++ b/cmd/apiProviderUpload_generated.go @@ -44,11 +44,14 @@ Learn more about API Management api for creating an API provider artifact [here] GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProxyDownload_generated.go b/cmd/apiProxyDownload_generated.go index 7f0238202b..0cae8c928b 100644 --- a/cmd/apiProxyDownload_generated.go +++ b/cmd/apiProxyDownload_generated.go @@ -44,11 +44,14 @@ func ApiProxyDownloadCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProxyList_generated.go b/cmd/apiProxyList_generated.go index 53f8c86d7b..2f4a2e2e96 100644 --- a/cmd/apiProxyList_generated.go +++ b/cmd/apiProxyList_generated.go @@ -81,11 +81,14 @@ func ApiProxyListCommand() *cobra.Command { GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/apiProxyUpload_generated.go b/cmd/apiProxyUpload_generated.go index 51bf6c42a1..f8036b1407 100644 --- a/cmd/apiProxyUpload_generated.go +++ b/cmd/apiProxyUpload_generated.go @@ -44,11 +44,14 @@ Learn more about the SAP API Management API for uploading an api proxy artifact GeneralConfig.GitHubAccessTokens = ResolveAccessTokens(GeneralConfig.GitHubTokens) - path, _ := os.Getwd() + path, err := os.Getwd() + if err != nil { + return err + } fatalHook := &log.FatalHook{CorrelationID: GeneralConfig.CorrelationID, Path: path} log.RegisterHook(fatalHook) - err := PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) + err = PrepareConfig(cmd, &metadata, STEP_NAME, &stepConfig, config.OpenPiperFile) if err != nil { log.SetErrorCategory(log.ErrorConfiguration) return err diff --git a/cmd/artifactPrepareVersion_generated.go b/cmd/artifactPrepareVersion_generated.go index e39872d89b..24e29a34d1 100644 --- a/cmd/artifactPrepareVersion_generated.go +++ b/cmd/artifactPrepareVersion_generated.go @@ -172,11 +172,14 @@ Define ` + "`" + `buildTool: custom` + "`" + `, ` + "`" + `filePath: --install --force --namespace Date: Wed, 6 Nov 2024 12:51:14 +0200 Subject: [PATCH 38/43] docs: update pr template to include inner source update reminder (#5169) --- .github/PULL_REQUEST_TEMPLATE.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 74a197d2ed..2e60b49eb0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,7 @@ -# Changes +# Description + +# Checklist - [ ] Tests - [ ] Documentation +- [ ] Inner source library needs updating From 9626bfca0d830285917a218b837e14586039d5c4 Mon Sep 17 00:00:00 2001 From: Adrien <99400874+hubadr@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:14:04 +0100 Subject: [PATCH 39/43] checkmarxOneExecuteScan - Fix report generation in CxOne 3.20 (#5170) * Initial in progress * compiling but not yet functional * Missed file * updated checkmarxone step * Working up to fetching a project then breaks * Missed file * Breaks when retrieving projects+proxy set * Create project & run scan working, now polling * Fixed polling * added back the zipfile remove command * Fixed polling again * Generates and downloads PDF report * Updated and working, prep for refactor * Added compliance steps * Cleanup, reporting, added groovy connector * fixed groovy file * checkmarxone to checkmarxOne * checkmarxone to checkmarxOne * split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix * Fixed filenames & yaml * missed the metadata_generated.go * added json to sarif conversion * fix:type in new checkmarxone package * fix:type in new checkmarxone package * removed test logs, added temp error log for creds * extra debugging to fix crash * improved auth logging, fixed query parse issue * fixed bug with group fetch when using oauth user * CWE can be -1 if not defined, can't be uint * Query also had CweID * Disabled predicates-fetch in sarif generation * Removing leftover info log message * Better error handling * fixed default preset configuration * removing .bat files - sorry * Cleanup per initial review * refactoring per Gist, fixed project find, add apps * small fix - sorry for commit noise while testing * Fixing issues with incremental scans. * removing maxretries * Updated per PR feedback, further changes todo toda * JSON Report changes and reporting cleanup * removing .bat (again?) * adding docs, groovy unit test, linter fixes * Started adding tests maybe 15% covered * fix(checkmarxOne): test cases for pkg and reporting * fix(checkmarxOne):fix formatting * feat(checkmarxone): update interface with missing method * feat(checkmarxone):change runStep signature to be able to inject dependency * feat(checkmarxone): add tests for step (wip) * Adding a bit more coverage * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix integration test PR * adding scan-summary bug workaround, reportgen fail * enforceThresholds fix when no results passed in * fixed gap when preset empty in yaml & project conf * fixed another gap in preset selection * fix 0-result panic * fail when no preset is set anywhere * removed comment * initial project-under-app support * fixing sarif reportgen * some cleanup of error messages * post-merge test fixes * revert previous upstream merge * adding "incremental" to "full" triggers * wrong boolean * project-in-application api change prep * Fixing SARIF report without preset access * fix sarif deeplink * removing comments * fix(cxone):formatting * fix(cxone):formatting * small sarif fixes * fixed merge * attempt at pulling git source repo branch * fix(cxone):new endpoint for project creation * fix(cxOne): taxa is an array * fix(cxOne): get Git branch from commonPipelineEnvironment * fix(cxOne): add params to tag a scan and a project * fix(cxOne): unit test - update project * fix(cxOne): unit test - update project tags * fix(cxOne): improve logs * fix(cxOne): improve logs * adding RequestNewPDFReport function using v2 api * added version check * fix(cxone): JSON report using v2 API * update to set reportType in v2 reportgen --------- Co-authored-by: michael kubiaczyk Co-authored-by: thtri Co-authored-by: Thanh-Hai Trinh Co-authored-by: michaelkubiaczyk <48311127+michaelkubiaczyk@users.noreply.github.com> Co-authored-by: sumeet patil --- pkg/checkmarxone/checkmarxone.go | 95 ++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/pkg/checkmarxone/checkmarxone.go b/pkg/checkmarxone/checkmarxone.go index f28a87a3cc..705aea76da 100644 --- a/pkg/checkmarxone/checkmarxone.go +++ b/pkg/checkmarxone/checkmarxone.go @@ -8,12 +8,10 @@ import ( "net/http" "net/url" "os" - - //"strconv" + "strconv" "strings" "time" - //"encoding/xml" piperHttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" @@ -1314,6 +1312,19 @@ func (sys *SystemInstance) GetResultsPredicates(SimilarityID int64, ProjectID st // RequestNewReport triggers the generation of a report for a specific scan addressed by scanID func (sys *SystemInstance) RequestNewReport(scanID, projectID, branch, reportType string) (string, error) { + if strings.EqualFold("pdf", reportType) || strings.EqualFold("json", reportType) { + version, err := sys.GetVersion() + if err == nil { + if version.CheckCxOne("3.20.0") >= 0 && version.CheckCxOne("3.21.0") == -1 { + sys.logger.Debugf("Current version is %v - between 3.20.0 and 3.21.0 - using v2 %v report", reportType, version.CxOne) + return sys.RequestNewReportV2(scanID, reportType) + } + sys.logger.Debugf("Current version is %v - using v1 %v report", reportType, version.CxOne) + } else { + sys.logger.Errorf("Failed to get the CxOne version during report-gen request, will use v1 %v report. Error: %s", reportType, err) + } + } + jsonData := map[string]interface{}{ "fileFormat": reportType, "reportType": "ui", @@ -1352,13 +1363,52 @@ func (sys *SystemInstance) RequestNewReport(scanID, projectID, branch, reportTyp return reportResponse.ReportId, err } +// Use the new V2 Report API to generate a PDF report +func (sys *SystemInstance) RequestNewReportV2(scanID, reportType string) (string, error) { + jsonData := map[string]interface{}{ + "reportName": "improved-scan-report", + "entities": []map[string]interface{}{ + { + "entity": "scan", + "ids": []string{scanID}, + "tags": []string{}, + }, + }, + "filters": map[string][]string{ + "scanners": {"sast"}, + }, + "reportType": "ui", + "fileFormat": reportType, + } + + jsonValue, _ := json.Marshal(jsonData) + + header := http.Header{} + header.Set("cxOrigin", cxOrigin) + header.Set("Content-Type", "application/json") + data, err := sendRequest(sys, http.MethodPost, "/reports/v2", bytes.NewBuffer(jsonValue), header, []int{}) + if err != nil { + return "", errors.Wrapf(err, "Failed to trigger report generation for scan %v", scanID) + } else { + sys.logger.Infof("Generating report %v", string(data)) + } + + var reportResponse struct { + ReportId string + } + err = json.Unmarshal(data, &reportResponse) + + return reportResponse.ReportId, err + +} + // GetReportStatus returns the status of the report generation process func (sys *SystemInstance) GetReportStatus(reportID string) (ReportStatus, error) { var response ReportStatus header := http.Header{} header.Set("Accept", "application/json") - data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/%v", reportID), nil, header, []int{}) + data, err := sendRequest(sys, http.MethodGet, fmt.Sprintf("/reports/%v?returnUrl=true", reportID), nil, header, []int{}) if err != nil { sys.logger.Errorf("Failed to fetch report status for reportID %v: %s", reportID, err) return response, errors.Wrapf(err, "failed to fetch report status for reportID %v", reportID) @@ -1412,3 +1462,40 @@ func (sys *SystemInstance) GetVersion() (VersionInfo, error) { err = json.Unmarshal(data, &version) return version, err } + +func (v VersionInfo) CheckCxOne(version string) int { + check := versionStringToInts(version) + cx1 := versionStringToInts(v.CxOne) + + if check[0] < cx1[0] { + return 1 + } else if check[0] > cx1[0] { + return -1 + } else { + if check[1] < cx1[1] { + return 1 + } else if check[1] > cx1[1] { + return -1 + } else { + if check[2] < cx1[2] { + return 1 + } else if check[2] > cx1[2] { + return -1 + } else { + return 0 + } + } + } +} + +func versionStringToInts(version string) []int64 { + if version == "" { + return []int64{0, 0, 0} + } + str := strings.Split(version, ".") + ints := make([]int64, len(str)) + for id, val := range str { + ints[id], _ = strconv.ParseInt(val, 10, 64) + } + return ints +} From 1edf8299f3f3457536c38c3073b6e510c96183c7 Mon Sep 17 00:00:00 2001 From: christian <153755613+skateball@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:40:51 +0100 Subject: [PATCH 40/43] Update URL to current SapMachine (#5126) * Update URL to recent SapMachine 11.0.2 in from Jan2019 := 5 years old use the recent version => 11.0.24 * 11.0.24 to 25 meanwhile 11.0.25 is the latest version * update from 11.0.25 to stable 21 link now to use https://sap.github.io/SapMachine/latest/21/linux-x64/jre/ * applied generated go code * Update whitesourceExecuteScan.yaml * from 21-latest to 17.0.13 to speed this up * revert description * generate for new url --------- Co-authored-by: Harald Aamot Co-authored-by: Oliver Feldmann --- cmd/whitesourceExecuteScan_generated.go | 4 ++-- resources/metadata/whitesourceExecuteScan.yaml | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/whitesourceExecuteScan_generated.go b/cmd/whitesourceExecuteScan_generated.go index 19a1f2021a..c41928a315 100644 --- a/cmd/whitesourceExecuteScan_generated.go +++ b/cmd/whitesourceExecuteScan_generated.go @@ -369,7 +369,7 @@ func addWhitesourceExecuteScanFlags(cmd *cobra.Command, stepConfig *whitesourceE cmd.Flags().BoolVar(&stepConfig.FailOnSevereVulnerabilities, "failOnSevereVulnerabilities", true, "Whether to fail the step on severe vulnerabilties or not") cmd.Flags().StringSliceVar(&stepConfig.Includes, "includes", []string{}, "List of file path patterns to include in the scan.") cmd.Flags().StringVar(&stepConfig.InstallCommand, "installCommand", os.Getenv("PIPER_installCommand"), "Install command that can be used to populate the default docker image for some scenarios.") - cmd.Flags().StringVar(&stepConfig.JreDownloadURL, "jreDownloadUrl", `https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz`, "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent.") + cmd.Flags().StringVar(&stepConfig.JreDownloadURL, "jreDownloadUrl", `https://github.com/SAP/SapMachine/releases/download/sapmachine-17.0.13/sapmachine-jre-17.0.13_linux-x64_bin.tar.gz`, "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent.") cmd.Flags().BoolVar(&stepConfig.LicensingVulnerabilities, "licensingVulnerabilities", true, "[NOT IMPLEMENTED] Whether license compliance is considered and reported as part of the assessment.") cmd.Flags().StringVar(&stepConfig.OrgToken, "orgToken", os.Getenv("PIPER_orgToken"), "WhiteSource token identifying your organization.") cmd.Flags().StringVar(&stepConfig.ProductName, "productName", os.Getenv("PIPER_productName"), "Name of the WhiteSource product used for results aggregation. This parameter is mandatory if the parameter `createProductFromPipeline` is set to `true` and the WhiteSource product does not yet exist. It is also mandatory if the parameter `productToken` is not provided.") @@ -692,7 +692,7 @@ func whitesourceExecuteScanMetadata() config.StepData { Type: "string", Mandatory: false, Aliases: []config.Alias{{Name: "whitesource/jreDownloadUrl", Deprecated: true}}, - Default: `https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz`, + Default: `https://github.com/SAP/SapMachine/releases/download/sapmachine-17.0.13/sapmachine-jre-17.0.13_linux-x64_bin.tar.gz`, }, { Name: "licensingVulnerabilities", diff --git a/resources/metadata/whitesourceExecuteScan.yaml b/resources/metadata/whitesourceExecuteScan.yaml index 9b030ac5b4..85298d9c76 100644 --- a/resources/metadata/whitesourceExecuteScan.yaml +++ b/resources/metadata/whitesourceExecuteScan.yaml @@ -257,14 +257,13 @@ spec: deprecated: true type: string description: - "URL used for downloading the Java Runtime Environment (JRE) required to run the - WhiteSource Unified Agent." + "URL used for downloading the Java Runtime Environment (JRE) required to run the WhiteSource Unified Agent." scope: - GENERAL - PARAMETERS - STAGES - STEPS - default: "https://github.com/SAP/SapMachine/releases/download/sapmachine-11.0.2/sapmachine-jre-11.0.2_linux-x64_bin.tar.gz" + default: "https://github.com/SAP/SapMachine/releases/download/sapmachine-17.0.13/sapmachine-jre-17.0.13_linux-x64_bin.tar.gz" - name: licensingVulnerabilities type: bool description: "[NOT IMPLEMENTED] Whether license compliance is considered and reported as part of the assessment." From 758d10b06e3dee6673642d89e66ff77b00a76dd9 Mon Sep 17 00:00:00 2001 From: maxcask Date: Tue, 12 Nov 2024 15:33:38 +0400 Subject: [PATCH 41/43] handle new naming strategy for Central Build stage name (#5171) * config helper rename * rename in name provider --------- Co-authored-by: maxcask --- src/com/sap/piper/ConfigurationHelper.groovy | 3 +++ src/com/sap/piper/StageNameProvider.groovy | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/com/sap/piper/ConfigurationHelper.groovy b/src/com/sap/piper/ConfigurationHelper.groovy index 2eeaaf0e6f..b6b1558ddf 100644 --- a/src/com/sap/piper/ConfigurationHelper.groovy +++ b/src/com/sap/piper/ConfigurationHelper.groovy @@ -10,6 +10,9 @@ class ConfigurationHelper implements Serializable { } ConfigurationHelper loadStepDefaults(Map compatibleParameters = [:], String stageName = step.env.STAGE_NAME) { + if (stageName == 'Central Build'){ + stageName = 'Build' + } DefaultValueCache.prepare(step) this.config = ConfigurationLoader.defaultGeneralConfiguration() mixin(ConfigurationLoader.defaultGeneralConfiguration(), null, compatibleParameters) diff --git a/src/com/sap/piper/StageNameProvider.groovy b/src/com/sap/piper/StageNameProvider.groovy index 675387ca4c..cd2fa71f06 100644 --- a/src/com/sap/piper/StageNameProvider.groovy +++ b/src/com/sap/piper/StageNameProvider.groovy @@ -3,13 +3,18 @@ package com.sap.piper @Singleton class StageNameProvider implements Serializable { static final long serialVersionUID = 1L + static final String CENTRAL_BUILD = "Central Build"; + static final String BUILD = "Build"; /** Stores a feature toggle for defaulting to technical names in stages */ boolean useTechnicalStageNames = false String getStageName(Script script, Map parameters, Script step) { + String stageName = null if (parameters.stageName in CharSequence) { - return parameters.stageName + stageName = parameters.stageName + stageName = replaceCentralBuild(stageName); + return stageName } if (this.useTechnicalStageNames) { String technicalStageName = getTechnicalStageName(step) @@ -17,7 +22,15 @@ class StageNameProvider implements Serializable { return technicalStageName } } - return script.env.STAGE_NAME + if (stageName == null) { + stageName = script.env.STAGE_NAME + stageName = replaceCentralBuild(stageName); + } + return stageName + } + + private String replaceCentralBuild(String stageName) { + return CENTRAL_BUILD.equals(stageName) ? BUILD : stageName; } static String getTechnicalStageName(Script step) { From f54dbfd433a68a9b3f0f31b06f6ce8e2b5ab0c42 Mon Sep 17 00:00:00 2001 From: Ralf Pannemans Date: Mon, 18 Nov 2024 09:51:35 +0100 Subject: [PATCH 42/43] Remove commit message from project-metadata.toml (#5176) --- pkg/cnbutils/project/metadata/metadata.go | 3 +-- pkg/cnbutils/project/metadata/metadata_test.go | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/cnbutils/project/metadata/metadata.go b/pkg/cnbutils/project/metadata/metadata.go index ec6483d3d2..556e89b8ec 100644 --- a/pkg/cnbutils/project/metadata/metadata.go +++ b/pkg/cnbutils/project/metadata/metadata.go @@ -36,8 +36,7 @@ func extractMetadataFromCPE(piperEnvRoot string, utils cnbutils.BuildUtils) file Source: &files.ProjectSource{ Type: "git", Version: map[string]interface{}{ - "commit": piperenv.GetResourceParameter(cpePath, "git", "headCommitId"), - "describe": piperenv.GetResourceParameter(cpePath, "git", "commitMessage"), + "commit": piperenv.GetResourceParameter(cpePath, "git", "headCommitId"), }, Metadata: map[string]interface{}{ "refs": []string{ diff --git a/pkg/cnbutils/project/metadata/metadata_test.go b/pkg/cnbutils/project/metadata/metadata_test.go index cf36c9eaaa..95a693098e 100644 --- a/pkg/cnbutils/project/metadata/metadata_test.go +++ b/pkg/cnbutils/project/metadata/metadata_test.go @@ -19,7 +19,6 @@ func TestWriteProjectMetadata(t *testing.T) { type = "git" [source.version] commit = "012548" - describe = "test-commit" [source.metadata] refs = ["main"] ` @@ -31,9 +30,8 @@ func TestWriteProjectMetadata(t *testing.T) { fileutils := piperutils.Files{} cpeFiles := map[string]string{ - "headCommitId": "012548", - "commitMessage": "test-commit", - "branch": "main", + "headCommitId": "012548", + "branch": "main", } dir := t.TempDir() From 10f535c8fa2643981cd8672d1dafb87d88b1cba4 Mon Sep 17 00:00:00 2001 From: maxcask Date: Tue, 19 Nov 2024 11:58:55 +0400 Subject: [PATCH 43/43] fix handle new naming strategy for Central Build stage name if extension is used (#5178) * handle extension filename --------- Co-authored-by: maxcask Co-authored-by: Googlom --- vars/piperStageWrapper.groovy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/vars/piperStageWrapper.groovy b/vars/piperStageWrapper.groovy index 1d6f1a3195..142a89617a 100644 --- a/vars/piperStageWrapper.groovy +++ b/vars/piperStageWrapper.groovy @@ -92,6 +92,7 @@ private void stageLocking(Map config, Closure body) { private void executeStage(script, originalStage, stageName, config, utils, telemetryDisabled = false) { boolean projectExtensions boolean globalExtensions + def startTime = System.currentTimeMillis() try { @@ -103,6 +104,18 @@ private void executeStage(script, originalStage, stageName, config, utils, telem */ def projectInterceptorFile = "${config.projectExtensionsDirectory}${stageName}.groovy" def globalInterceptorFile = "${config.globalExtensionsDirectory}${stageName}.groovy" + /* due to renaming stage 'Central Build' to 'Build' need to define extension file name 'Central Build.groovy' + as stageName used to generate it, once all the users will 'Build' as a stageName + and extension filename, below renaming snippet should be removed + */ + if (stageName == 'Build'){ + if (!fileExists(projectInterceptorFile) || !fileExists(globalInterceptorFile)){ + def centralBuildExtensionFileName = "Central Build.groovy" + projectInterceptorFile = "${config.projectExtensionsDirectory}${centralBuildExtensionFileName}" + globalInterceptorFile = "${config.globalExtensionsDirectory}${centralBuildExtensionFileName}" + } + } + projectExtensions = fileExists(projectInterceptorFile) globalExtensions = fileExists(globalInterceptorFile) // Pre-defining the real originalStage in body variable, might be overwritten later if extensions exist