From 5fc0d2b4896d982739f4a01d26e51129477828c6 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Mon, 5 Jun 2023 14:10:38 +0200 Subject: [PATCH 01/13] Test --- pkg/npm/npm.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index 8a729e69e0..b768aaf20c 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -357,7 +357,7 @@ func (exec *Execute) checkIfLockFilesExist() (bool, bool, error) { func (exec *Execute) CreateBOM(packageJSONFiles []string) error { execRunner := exec.Utils.GetExecRunner() // Install CycloneDX Node.js module locally without saving in package.json - err := execRunner.RunExecutable("npm", "install", cycloneDxPackageVersion, "--no-save") + err := execRunner.RunExecutable("npm", "install", cycloneDxPackageVersion) if err != nil { return fmt.Errorf("failed to install CycloneDX package: %w", err) } @@ -372,6 +372,8 @@ func (exec *Execute) CreateBOM(packageJSONFiles []string) error { "--spec-version", cycloneDxSchemaVersion, "--output-file", filepath.Join(path, npmBomFilename), + "--package-lock-only", + //"--ignore-npm-errors", packageJSONFile, } err := execRunner.RunExecutable("npx", params...) From 08075634dd032db244368f6087d379c450f29580 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Mon, 12 Jun 2023 07:06:48 +0200 Subject: [PATCH 02/13] Try omit --- pkg/npm/npm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index b768aaf20c..520550c47e 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -367,13 +367,13 @@ func (exec *Execute) CreateBOM(packageJSONFiles []string) error { path := filepath.Dir(packageJSONFile) params := []string{ cycloneDxPackageVersion, + "--omit", + "optional", "--output-format", "XML", "--spec-version", cycloneDxSchemaVersion, "--output-file", filepath.Join(path, npmBomFilename), - "--package-lock-only", - //"--ignore-npm-errors", packageJSONFile, } err := execRunner.RunExecutable("npx", params...) From ba30ade698c6bc082f3d56a1fbc8a639dd8e5cc4 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Thu, 15 Jun 2023 16:47:13 +0200 Subject: [PATCH 03/13] Introduce global installation and fallback --- pkg/npm/npm.go | 68 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index 520550c47e..4993d1a3b1 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -14,9 +14,10 @@ import ( ) const ( - npmBomFilename = "bom-npm.xml" - cycloneDxPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" - cycloneDxSchemaVersion = "1.4" + npmBomFilename = "bom-npm.xml" + cycloneDxNewPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" + cycloneDxOldPackageVersion = "@cyclonedx/bom@^3.10.6" + cycloneDxSchemaVersion = "1.4" ) // Execute struct holds utils to enable mocking and common parameters @@ -356,29 +357,60 @@ func (exec *Execute) checkIfLockFilesExist() (bool, bool, error) { // CreateBOM generates BOM file using CycloneDX from all package.json files func (exec *Execute) CreateBOM(packageJSONFiles []string) error { execRunner := exec.Utils.GetExecRunner() - // Install CycloneDX Node.js module locally without saving in package.json - err := execRunner.RunExecutable("npm", "install", cycloneDxPackageVersion) + // Install CycloneDX node npm package globally and try to generate bBOM + // if not successful, as fallback use old package - cyclonedx bom + newPackageInstallParams := []string{"install", "--global", cycloneDxNewPackageVersion} + newPackageRunParams := []string{ + cycloneDxNewPackageVersion, + "--omit", + "optional", + "--output-format", + "XML", + "--spec-version", + cycloneDxSchemaVersion, + "--output-file", + } + + err := execRunner.RunExecutable("npm", newPackageInstallParams...) if err != nil { - return fmt.Errorf("failed to install CycloneDX package: %w", err) + //return fmt.Errorf("failed to install CycloneDX package: %w", err) + log.Entry().Infof("failed to install CycloneDX BOM with cyclonedx-npm: %w", err) + log.Entry().Infof("Fallback to cyclonedx/bom") + goto fallback } if len(packageJSONFiles) > 0 { for _, packageJSONFile := range packageJSONFiles { path := filepath.Dir(packageJSONFile) - params := []string{ - cycloneDxPackageVersion, - "--omit", - "optional", - "--output-format", - "XML", - "--spec-version", - cycloneDxSchemaVersion, - "--output-file", filepath.Join(path, npmBomFilename), - packageJSONFile, - } + params := append(newPackageRunParams, filepath.Join(path, npmBomFilename), + packageJSONFile) err := execRunner.RunExecutable("npx", params...) if err != nil { - return fmt.Errorf("failed to generate CycloneDX BOM: %w", err) + log.Entry().Infof("failed to generate CycloneDX BOM with yclonedx-npm: %w", err) + log.Entry().Infof("Fallback to cyclonedx/bom") + goto fallback + } + } + } + +fallback: + oldPackageInstallParams := []string{"install", "@cyclonedx/bom@^3.10.6", "--no-save"} + + err = execRunner.RunExecutable("npm", oldPackageInstallParams...) + if err != nil { + return fmt.Errorf("failed to install CycloneDX via cyclonedx/bom package: %w", err) + } + + if len(packageJSONFiles) > 0 { + for _, packageJSONFile := range packageJSONFiles { + path := filepath.Dir(packageJSONFile) + oldPackageRunParams := []string{ + "cyclonedx-bom", + path, + "--output", filepath.Join(path, npmBomFilename)} + err := execRunner.RunExecutable("npx", oldPackageRunParams...) + if err != nil { + return fmt.Errorf("failed to generate CycloneDX BOM via cyclonedx/bom package : %w", err) } } } From 2f11fa0f647813a25c8f9d9347a3f70863d473d0 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Mon, 19 Jun 2023 12:13:16 +0200 Subject: [PATCH 04/13] Extract to a separate function --- pkg/npm/npm.go | 78 ++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index 4993d1a3b1..d78930b4ea 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -15,8 +15,8 @@ import ( const ( npmBomFilename = "bom-npm.xml" - cycloneDxNewPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" - cycloneDxOldPackageVersion = "@cyclonedx/bom@^3.10.6" + cycloneDxNpmPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" + cycloneDxBomPackageVersion = "@cyclonedx/bom@^3.10.6" cycloneDxSchemaVersion = "1.4" ) @@ -356,14 +356,10 @@ func (exec *Execute) checkIfLockFilesExist() (bool, bool, error) { // CreateBOM generates BOM file using CycloneDX from all package.json files func (exec *Execute) CreateBOM(packageJSONFiles []string) error { - execRunner := exec.Utils.GetExecRunner() - // Install CycloneDX node npm package globally and try to generate bBOM - // if not successful, as fallback use old package - cyclonedx bom - newPackageInstallParams := []string{"install", "--global", cycloneDxNewPackageVersion} - newPackageRunParams := []string{ - cycloneDxNewPackageVersion, - "--omit", - "optional", + // Install cyclonedx-npm globally (to avoid extraneous errors) and generate BOM + cycloneDxNpmInstallParams := []string{"install", "--global", cycloneDxNpmPackageVersion} + cycloneDxNpmRunParams := []string{ + cycloneDxNpmPackageVersion, "--output-format", "XML", "--spec-version", @@ -371,48 +367,56 @@ func (exec *Execute) CreateBOM(packageJSONFiles []string) error { "--output-file", } - err := execRunner.RunExecutable("npm", newPackageInstallParams...) - if err != nil { - //return fmt.Errorf("failed to install CycloneDX package: %w", err) - log.Entry().Infof("failed to install CycloneDX BOM with cyclonedx-npm: %w", err) - log.Entry().Infof("Fallback to cyclonedx/bom") - goto fallback + // Install cyclonedx/bom with --nosave and generate BOM. + cycloneDxBomInstallParams := []string{"install", cycloneDxBomPackageVersion, "--no-save"} + cycloneDxBomRunParams := []string{ + "cyclonedx-bom", + "--output", } - if len(packageJSONFiles) > 0 { - for _, packageJSONFile := range packageJSONFiles { - path := filepath.Dir(packageJSONFile) - params := append(newPackageRunParams, filepath.Join(path, npmBomFilename), - packageJSONFile) - err := execRunner.RunExecutable("npx", params...) - if err != nil { - log.Entry().Infof("failed to generate CycloneDX BOM with yclonedx-npm: %w", err) - log.Entry().Infof("Fallback to cyclonedx/bom") - goto fallback - } + // Attempt#1, generate BOM via cyclonedx -npm + err := exec.createBOMWithParams(cycloneDxNpmInstallParams, cycloneDxNpmRunParams, packageJSONFiles, false) + if err != nil { + + log.Entry().Infof("Failed to generate BOM CycloneDX BOM with cyclonedx-npm ,fallback to cyclonedx/bom") + // Attempt #2, generate BOM via cyclonedx/bom@^3.10.6 + err = exec.createBOMWithParams(cycloneDxBomInstallParams, cycloneDxBomRunParams, packageJSONFiles, true) + if err != nil { + log.Entry().Infof("Failed to generate BOM CycloneDX BOM with fallback package cyclonedx/bom ") + return err } } + return nil +} + +func (exec *Execute) createBOMWithParams(packageInstallParams []string, packageRunParams []string, packageJSONFiles []string, fallback bool) error { + execRunner := exec.Utils.GetExecRunner() -fallback: - oldPackageInstallParams := []string{"install", "@cyclonedx/bom@^3.10.6", "--no-save"} + err := execRunner.RunExecutable("npm", packageInstallParams...) - err = execRunner.RunExecutable("npm", oldPackageInstallParams...) if err != nil { - return fmt.Errorf("failed to install CycloneDX via cyclonedx/bom package: %w", err) + return fmt.Errorf("failed to install CycloneDX BOM %w", err) } if len(packageJSONFiles) > 0 { for _, packageJSONFile := range packageJSONFiles { path := filepath.Dir(packageJSONFile) - oldPackageRunParams := []string{ - "cyclonedx-bom", - path, - "--output", filepath.Join(path, npmBomFilename)} - err := execRunner.RunExecutable("npx", oldPackageRunParams...) + params := append(packageRunParams, filepath.Join(path, npmBomFilename)) + + //Below code needed as cyclonedx-npm needs packageJson file itself, while + //cyclonedx/bom needs dir path of package json + if !fallback { + params = append(params, packageJSONFile) + } else { + params = append(params, path) + } + + err := execRunner.RunExecutable("npx", params...) if err != nil { - return fmt.Errorf("failed to generate CycloneDX BOM via cyclonedx/bom package : %w", err) + return fmt.Errorf("failed to generate CycloneDX BOM :%w", err) } } } + return nil } From b5ca914db224bb66d0592ce7cdaf1486dc6bcd04 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Wed, 21 Jun 2023 10:22:02 +0200 Subject: [PATCH 05/13] Fix unit tests --- pkg/npm/npm_test.go | 61 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/pkg/npm/npm_test.go b/pkg/npm/npm_test.go index 4decdcf2a6..4b06e45de8 100644 --- a/pkg/npm/npm_test.go +++ b/pkg/npm/npm_test.go @@ -4,6 +4,7 @@ package npm import ( + "fmt" "path/filepath" "testing" @@ -342,7 +343,7 @@ func TestNpm(t *testing.T) { } }) - t.Run("Create BOM", func(t *testing.T) { + t.Run("Create BOM with cyclonedx-npm", func(t *testing.T) { utils := newNpmMockUtilsBundle() utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }")) utils.AddFile("package-lock.json", []byte("{}")) @@ -357,20 +358,56 @@ func TestNpm(t *testing.T) { Options: options, } err := exec.CreateBOM([]string{"package.json", filepath.Join("src", "package.json")}) + cycloneDxNpmInstallParams := []string{"install", "--global", "@cyclonedx/cyclonedx-npm@1.11.0"} + cycloneDxNpmRunParams := []string{ + cycloneDxNpmPackageVersion, + "--output-format", + "XML", + "--spec-version", + cycloneDxSchemaVersion, + "--output-file", + } if assert.NoError(t, err) { if assert.Equal(t, 3, len(utils.execRunner.Calls)) { - assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"install", "@cyclonedx/cyclonedx-npm@1.11.0", "--no-save"}}, utils.execRunner.Calls[0]) - assert.Equal(t, mock.ExecCall{Exec: "npx", Params: []string{"@cyclonedx/cyclonedx-npm@1.11.0", "--output-format", - "XML", - "--spec-version", - "1.4", - "--output-file", "bom-npm.xml", "package.json"}}, utils.execRunner.Calls[1]) - assert.Equal(t, mock.ExecCall{Exec: "npx", Params: []string{"@cyclonedx/cyclonedx-npm@1.11.0", "--output-format", - "XML", - "--spec-version", - "1.4", - "--output-file", filepath.Join("src", "bom-npm.xml"), filepath.Join("src", "package.json")}}, utils.execRunner.Calls[2]) + assert.Equal(t, mock.ExecCall{Exec: "npm", Params: cycloneDxNpmInstallParams}, utils.execRunner.Calls[0]) + assert.Equal(t, mock.ExecCall{Exec: "npx", Params: append(cycloneDxNpmRunParams, "bom-npm.xml", "package.json")}, utils.execRunner.Calls[1]) + assert.Equal(t, mock.ExecCall{Exec: "npx", Params: append(cycloneDxNpmRunParams, filepath.Join("src", "bom-npm.xml"), filepath.Join("src", "package.json"))}, utils.execRunner.Calls[2]) + } + + } + }) + + t.Run("Create BOM with fallback cyclonedx/bom", func(t *testing.T) { + utils := newNpmMockUtilsBundle() + utils.AddFile("package.json", []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }")) + utils.AddFile("package-lock.json", []byte("{}")) + utils.AddFile(filepath.Join("src", "package.json"), []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }")) + utils.AddFile(filepath.Join("src", "package-lock.json"), []byte("{}")) + utils.execRunner.ShouldFailOnCommand = map[string]error{"npm install --global @cyclonedx/cyclonedx-npm@1.11.0": fmt.Errorf("failed to install CycloneDX BOM")} + + options := ExecutorOptions{} + options.DefaultNpmRegistry = "foo.bar" + + exec := &Execute{ + Utils: &utils, + Options: options, + } + err := exec.CreateBOM([]string{"package.json", filepath.Join("src", "package.json")}) + cycloneDxNpmInstallParams := []string{"install", "--global", "@cyclonedx/cyclonedx-npm@1.11.0"} + + cycloneDxBomInstallParams := []string{"install", cycloneDxBomPackageVersion, "--no-save"} + cycloneDxBomRunParams := []string{ + "cyclonedx-bom", + "--output", + } + + if assert.NoError(t, err) { + if assert.Equal(t, 4, len(utils.execRunner.Calls)) { + assert.Equal(t, mock.ExecCall{Exec: "npm", Params: cycloneDxNpmInstallParams}, utils.execRunner.Calls[0]) + assert.Equal(t, mock.ExecCall{Exec: "npm", Params: cycloneDxBomInstallParams}, utils.execRunner.Calls[1]) + assert.Equal(t, mock.ExecCall{Exec: "npx", Params: append(cycloneDxBomRunParams, "bom-npm.xml", ".")}, utils.execRunner.Calls[2]) + assert.Equal(t, mock.ExecCall{Exec: "npx", Params: append(cycloneDxBomRunParams, filepath.Join("src", "bom-npm.xml"), filepath.Join("src"))}, utils.execRunner.Calls[3]) } } From 58c27a8fb27b07775720fde40b2ea4901d6497db Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Wed, 21 Jun 2023 16:43:07 +0200 Subject: [PATCH 06/13] Add root permissions for docker image for Azure --- cmd/npmExecuteScripts_generated.go | 2 +- resources/metadata/npmExecuteScripts.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/npmExecuteScripts_generated.go b/cmd/npmExecuteScripts_generated.go index f77d4a6d39..1a6ef1a1cb 100644 --- a/cmd/npmExecuteScripts_generated.go +++ b/cmd/npmExecuteScripts_generated.go @@ -414,7 +414,7 @@ func npmExecuteScriptsMetadata() config.StepData { }, }, Containers: []config.Container{ - {Name: "node", Image: "node:lts-buster"}, + {Name: "node", Image: "node:lts-buster", Options: []config.Option{{Name: "-u", Value: "0"}}}, }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ diff --git a/resources/metadata/npmExecuteScripts.yaml b/resources/metadata/npmExecuteScripts.yaml index d76490da94..11107f5eb8 100644 --- a/resources/metadata/npmExecuteScripts.yaml +++ b/resources/metadata/npmExecuteScripts.yaml @@ -175,3 +175,6 @@ spec: containers: - name: node image: node:lts-buster + options: + - name: -u + value: "0" From c36eea52be44fff6b0713112080cdd84b623b3d0 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Mon, 26 Jun 2023 11:19:50 +0200 Subject: [PATCH 07/13] Install in another folder --- cmd/npmExecuteScripts_generated.go | 2 +- pkg/npm/npm.go | 22 ++++++++++++---------- resources/metadata/npmExecuteScripts.yaml | 3 --- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/npmExecuteScripts_generated.go b/cmd/npmExecuteScripts_generated.go index 1a6ef1a1cb..f77d4a6d39 100644 --- a/cmd/npmExecuteScripts_generated.go +++ b/cmd/npmExecuteScripts_generated.go @@ -414,7 +414,7 @@ func npmExecuteScriptsMetadata() config.StepData { }, }, Containers: []config.Container{ - {Name: "node", Image: "node:lts-buster", Options: []config.Option{{Name: "-u", Value: "0"}}}, + {Name: "node", Image: "node:lts-buster"}, }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index d78930b4ea..129cfdb791 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -14,10 +14,11 @@ import ( ) const ( - npmBomFilename = "bom-npm.xml" - cycloneDxNpmPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" - cycloneDxBomPackageVersion = "@cyclonedx/bom@^3.10.6" - cycloneDxSchemaVersion = "1.4" + npmBomFilename = "bom-npm.xml" + cycloneDxNpmPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" + cycloneDxNpmInstallationFolder = "./bomFolder" + cycloneDxBomPackageVersion = "@cyclonedx/bom@^3.10.6" + cycloneDxSchemaVersion = "1.4" ) // Execute struct holds utils to enable mocking and common parameters @@ -356,10 +357,9 @@ func (exec *Execute) checkIfLockFilesExist() (bool, bool, error) { // CreateBOM generates BOM file using CycloneDX from all package.json files func (exec *Execute) CreateBOM(packageJSONFiles []string) error { - // Install cyclonedx-npm globally (to avoid extraneous errors) and generate BOM - cycloneDxNpmInstallParams := []string{"install", "--global", cycloneDxNpmPackageVersion} + // Install cyclonedx-npm in a new folder (to avoid extraneous errors) and generate BOM + cycloneDxNpmInstallParams := []string{"install", cycloneDxNpmPackageVersion, "--prefix", cycloneDxNpmInstallationFolder} cycloneDxNpmRunParams := []string{ - cycloneDxNpmPackageVersion, "--output-format", "XML", "--spec-version", @@ -401,17 +401,19 @@ func (exec *Execute) createBOMWithParams(packageInstallParams []string, packageR if len(packageJSONFiles) > 0 { for _, packageJSONFile := range packageJSONFiles { path := filepath.Dir(packageJSONFile) + executable := "npx" params := append(packageRunParams, filepath.Join(path, npmBomFilename)) - //Below code needed as cyclonedx-npm needs packageJson file itself, while - //cyclonedx/bom needs dir path of package json + //Below code needed as to adjust according to needs of + // two packages if !fallback { params = append(params, packageJSONFile) + executable = cycloneDxNpmInstallationFolder + "/node_modules/.bin/cyclonedx-npm" } else { params = append(params, path) } - err := execRunner.RunExecutable("npx", params...) + err := execRunner.RunExecutable(executable, params...) if err != nil { return fmt.Errorf("failed to generate CycloneDX BOM :%w", err) } diff --git a/resources/metadata/npmExecuteScripts.yaml b/resources/metadata/npmExecuteScripts.yaml index 11107f5eb8..d76490da94 100644 --- a/resources/metadata/npmExecuteScripts.yaml +++ b/resources/metadata/npmExecuteScripts.yaml @@ -175,6 +175,3 @@ spec: containers: - name: node image: node:lts-buster - options: - - name: -u - value: "0" From 1efd1b576b8cfd4a601a4f88332a78a584ba4566 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Tue, 27 Jun 2023 10:49:57 +0200 Subject: [PATCH 08/13] fix unit tests --- pkg/npm/npm_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/npm/npm_test.go b/pkg/npm/npm_test.go index 4b06e45de8..d0a87aac95 100644 --- a/pkg/npm/npm_test.go +++ b/pkg/npm/npm_test.go @@ -358,9 +358,8 @@ func TestNpm(t *testing.T) { Options: options, } err := exec.CreateBOM([]string{"package.json", filepath.Join("src", "package.json")}) - cycloneDxNpmInstallParams := []string{"install", "--global", "@cyclonedx/cyclonedx-npm@1.11.0"} + cycloneDxNpmInstallParams := []string{"install", "@cyclonedx/cyclonedx-npm@1.11.0", "--prefix", "./bomFolder"} cycloneDxNpmRunParams := []string{ - cycloneDxNpmPackageVersion, "--output-format", "XML", "--spec-version", @@ -371,8 +370,8 @@ func TestNpm(t *testing.T) { if assert.NoError(t, err) { if assert.Equal(t, 3, len(utils.execRunner.Calls)) { assert.Equal(t, mock.ExecCall{Exec: "npm", Params: cycloneDxNpmInstallParams}, utils.execRunner.Calls[0]) - assert.Equal(t, mock.ExecCall{Exec: "npx", Params: append(cycloneDxNpmRunParams, "bom-npm.xml", "package.json")}, utils.execRunner.Calls[1]) - assert.Equal(t, mock.ExecCall{Exec: "npx", Params: append(cycloneDxNpmRunParams, filepath.Join("src", "bom-npm.xml"), filepath.Join("src", "package.json"))}, utils.execRunner.Calls[2]) + assert.Equal(t, mock.ExecCall{Exec: "./bomFolder/node_modules/.bin/cyclonedx-npm", Params: append(cycloneDxNpmRunParams, "bom-npm.xml", "package.json")}, utils.execRunner.Calls[1]) + assert.Equal(t, mock.ExecCall{Exec: "./bomFolder/node_modules/.bin/cyclonedx-npm", Params: append(cycloneDxNpmRunParams, filepath.Join("src", "bom-npm.xml"), filepath.Join("src", "package.json"))}, utils.execRunner.Calls[2]) } } @@ -384,7 +383,7 @@ func TestNpm(t *testing.T) { utils.AddFile("package-lock.json", []byte("{}")) utils.AddFile(filepath.Join("src", "package.json"), []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }")) utils.AddFile(filepath.Join("src", "package-lock.json"), []byte("{}")) - utils.execRunner.ShouldFailOnCommand = map[string]error{"npm install --global @cyclonedx/cyclonedx-npm@1.11.0": fmt.Errorf("failed to install CycloneDX BOM")} + utils.execRunner.ShouldFailOnCommand = map[string]error{"npm install @cyclonedx/cyclonedx-npm@1.11.0 --prefix ./bomFolder": fmt.Errorf("failed to install CycloneDX BOM")} options := ExecutorOptions{} options.DefaultNpmRegistry = "foo.bar" @@ -394,7 +393,7 @@ func TestNpm(t *testing.T) { Options: options, } err := exec.CreateBOM([]string{"package.json", filepath.Join("src", "package.json")}) - cycloneDxNpmInstallParams := []string{"install", "--global", "@cyclonedx/cyclonedx-npm@1.11.0"} + cycloneDxNpmInstallParams := []string{"install", "@cyclonedx/cyclonedx-npm@1.11.0", "--prefix", "./bomFolder"} cycloneDxBomInstallParams := []string{"install", cycloneDxBomPackageVersion, "--no-save"} cycloneDxBomRunParams := []string{ From bc5f1825dc22188f0c98ecedee8aa3ce58de2094 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Tue, 27 Jun 2023 10:58:58 +0200 Subject: [PATCH 09/13] Cleanup --- pkg/npm/npm.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index 129cfdb791..c695b3c545 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -359,26 +359,18 @@ func (exec *Execute) checkIfLockFilesExist() (bool, bool, error) { func (exec *Execute) CreateBOM(packageJSONFiles []string) error { // Install cyclonedx-npm in a new folder (to avoid extraneous errors) and generate BOM cycloneDxNpmInstallParams := []string{"install", cycloneDxNpmPackageVersion, "--prefix", cycloneDxNpmInstallationFolder} - cycloneDxNpmRunParams := []string{ - "--output-format", - "XML", - "--spec-version", - cycloneDxSchemaVersion, - "--output-file", - } + cycloneDxNpmRunParams := []string{"--output-format", "XML", "--spec-version", cycloneDxSchemaVersion, "--output-file"} // Install cyclonedx/bom with --nosave and generate BOM. cycloneDxBomInstallParams := []string{"install", cycloneDxBomPackageVersion, "--no-save"} - cycloneDxBomRunParams := []string{ - "cyclonedx-bom", - "--output", - } + cycloneDxBomRunParams := []string{"cyclonedx-bom", "--output"} - // Attempt#1, generate BOM via cyclonedx -npm + // Attempt#1, generate BOM via cyclonedx-npm err := exec.createBOMWithParams(cycloneDxNpmInstallParams, cycloneDxNpmRunParams, packageJSONFiles, false) if err != nil { log.Entry().Infof("Failed to generate BOM CycloneDX BOM with cyclonedx-npm ,fallback to cyclonedx/bom") + // Attempt #2, generate BOM via cyclonedx/bom@^3.10.6 err = exec.createBOMWithParams(cycloneDxBomInstallParams, cycloneDxBomRunParams, packageJSONFiles, true) if err != nil { @@ -389,23 +381,25 @@ func (exec *Execute) CreateBOM(packageJSONFiles []string) error { return nil } +// Facilitates BOM generation with different packages func (exec *Execute) createBOMWithParams(packageInstallParams []string, packageRunParams []string, packageJSONFiles []string, fallback bool) error { execRunner := exec.Utils.GetExecRunner() + // Install package err := execRunner.RunExecutable("npm", packageInstallParams...) if err != nil { return fmt.Errorf("failed to install CycloneDX BOM %w", err) } + // Run package for all package JSON files if len(packageJSONFiles) > 0 { for _, packageJSONFile := range packageJSONFiles { path := filepath.Dir(packageJSONFile) executable := "npx" params := append(packageRunParams, filepath.Join(path, npmBomFilename)) - //Below code needed as to adjust according to needs of - // two packages + //Below code needed as to adjust according to needs of cyclonedx-npm and fallback cyclonedx/bom@^3.10.6 if !fallback { params = append(params, packageJSONFile) executable = cycloneDxNpmInstallationFolder + "/node_modules/.bin/cyclonedx-npm" From e7741279a4aee51339ba36fea629a37d377a8904 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Mon, 3 Jul 2023 12:08:56 +0200 Subject: [PATCH 10/13] introduce back --no-save,change directory name, fix tests --- pkg/npm/npm.go | 4 ++-- pkg/npm/npm_test.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index c695b3c545..23ccbcf8fd 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -16,7 +16,7 @@ import ( const ( npmBomFilename = "bom-npm.xml" cycloneDxNpmPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" - cycloneDxNpmInstallationFolder = "./bomFolder" + cycloneDxNpmInstallationFolder = "./tmp" cycloneDxBomPackageVersion = "@cyclonedx/bom@^3.10.6" cycloneDxSchemaVersion = "1.4" ) @@ -358,7 +358,7 @@ func (exec *Execute) checkIfLockFilesExist() (bool, bool, error) { // CreateBOM generates BOM file using CycloneDX from all package.json files func (exec *Execute) CreateBOM(packageJSONFiles []string) error { // Install cyclonedx-npm in a new folder (to avoid extraneous errors) and generate BOM - cycloneDxNpmInstallParams := []string{"install", cycloneDxNpmPackageVersion, "--prefix", cycloneDxNpmInstallationFolder} + cycloneDxNpmInstallParams := []string{"install", "--no-save", cycloneDxNpmPackageVersion, "--prefix", cycloneDxNpmInstallationFolder} cycloneDxNpmRunParams := []string{"--output-format", "XML", "--spec-version", cycloneDxSchemaVersion, "--output-file"} // Install cyclonedx/bom with --nosave and generate BOM. diff --git a/pkg/npm/npm_test.go b/pkg/npm/npm_test.go index d0a87aac95..319fb6c1f0 100644 --- a/pkg/npm/npm_test.go +++ b/pkg/npm/npm_test.go @@ -358,7 +358,7 @@ func TestNpm(t *testing.T) { Options: options, } err := exec.CreateBOM([]string{"package.json", filepath.Join("src", "package.json")}) - cycloneDxNpmInstallParams := []string{"install", "@cyclonedx/cyclonedx-npm@1.11.0", "--prefix", "./bomFolder"} + cycloneDxNpmInstallParams := []string{"install", "--no-save", "@cyclonedx/cyclonedx-npm@1.11.0", "--prefix", "./tmp"} cycloneDxNpmRunParams := []string{ "--output-format", "XML", @@ -370,8 +370,8 @@ func TestNpm(t *testing.T) { if assert.NoError(t, err) { if assert.Equal(t, 3, len(utils.execRunner.Calls)) { assert.Equal(t, mock.ExecCall{Exec: "npm", Params: cycloneDxNpmInstallParams}, utils.execRunner.Calls[0]) - assert.Equal(t, mock.ExecCall{Exec: "./bomFolder/node_modules/.bin/cyclonedx-npm", Params: append(cycloneDxNpmRunParams, "bom-npm.xml", "package.json")}, utils.execRunner.Calls[1]) - assert.Equal(t, mock.ExecCall{Exec: "./bomFolder/node_modules/.bin/cyclonedx-npm", Params: append(cycloneDxNpmRunParams, filepath.Join("src", "bom-npm.xml"), filepath.Join("src", "package.json"))}, utils.execRunner.Calls[2]) + assert.Equal(t, mock.ExecCall{Exec: "./tmp/node_modules/.bin/cyclonedx-npm", Params: append(cycloneDxNpmRunParams, "bom-npm.xml", "package.json")}, utils.execRunner.Calls[1]) + assert.Equal(t, mock.ExecCall{Exec: "./tmp/node_modules/.bin/cyclonedx-npm", Params: append(cycloneDxNpmRunParams, filepath.Join("src", "bom-npm.xml"), filepath.Join("src", "package.json"))}, utils.execRunner.Calls[2]) } } @@ -383,7 +383,7 @@ func TestNpm(t *testing.T) { utils.AddFile("package-lock.json", []byte("{}")) utils.AddFile(filepath.Join("src", "package.json"), []byte("{\"scripts\": { \"ci-lint\": \"exit 0\" } }")) utils.AddFile(filepath.Join("src", "package-lock.json"), []byte("{}")) - utils.execRunner.ShouldFailOnCommand = map[string]error{"npm install @cyclonedx/cyclonedx-npm@1.11.0 --prefix ./bomFolder": fmt.Errorf("failed to install CycloneDX BOM")} + utils.execRunner.ShouldFailOnCommand = map[string]error{"npm install --no-save @cyclonedx/cyclonedx-npm@1.11.0 --prefix ./tmp": fmt.Errorf("failed to install CycloneDX BOM")} options := ExecutorOptions{} options.DefaultNpmRegistry = "foo.bar" @@ -393,7 +393,7 @@ func TestNpm(t *testing.T) { Options: options, } err := exec.CreateBOM([]string{"package.json", filepath.Join("src", "package.json")}) - cycloneDxNpmInstallParams := []string{"install", "@cyclonedx/cyclonedx-npm@1.11.0", "--prefix", "./bomFolder"} + cycloneDxNpmInstallParams := []string{"install", "--no-save", "@cyclonedx/cyclonedx-npm@1.11.0", "--prefix", "./tmp"} cycloneDxBomInstallParams := []string{"install", cycloneDxBomPackageVersion, "--no-save"} cycloneDxBomRunParams := []string{ From e06a3069edac55fbddb8c2c34b0f5c10e68763dc Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Tue, 4 Jul 2023 12:21:01 +0200 Subject: [PATCH 11/13] add tmp folder to npmignore --- pkg/npm/npm.go | 2 +- pkg/npm/publish.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go index 23ccbcf8fd..ce1bae46f2 100644 --- a/pkg/npm/npm.go +++ b/pkg/npm/npm.go @@ -16,8 +16,8 @@ import ( const ( npmBomFilename = "bom-npm.xml" cycloneDxNpmPackageVersion = "@cyclonedx/cyclonedx-npm@1.11.0" - cycloneDxNpmInstallationFolder = "./tmp" cycloneDxBomPackageVersion = "@cyclonedx/bom@^3.10.6" + cycloneDxNpmInstallationFolder = "./tmp" // This folder is also added to npmignore in publish.go.Any changes to this folder needs a change in publish.go publish() cycloneDxSchemaVersion = "1.4" ) diff --git a/pkg/npm/publish.go b/pkg/npm/publish.go index 333c8c5405..76cd4a62b0 100644 --- a/pkg/npm/publish.go +++ b/pkg/npm/publish.go @@ -77,6 +77,9 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p npmignore.Add("**/piper") log.Entry().Debug("adding **/sap-piper") npmignore.Add("**/sap-piper") + // temporary installation folder used to install BOM to be ignored + log.Entry().Debug("adding tmp to npmignore") + npmignore.Add("tmp/") npmrc := NewNPMRC(filepath.Dir(packageJSON)) From 45ac7ca9bc86ae98dbd4b6a146d84680228774d4 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Tue, 11 Jul 2023 12:37:47 +0200 Subject: [PATCH 12/13] change docker image for guage --- integration/integration_gauge_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/integration_gauge_test.go b/integration/integration_gauge_test.go index c0561c025e..5d9fbfc93f 100644 --- a/integration/integration_gauge_test.go +++ b/integration/integration_gauge_test.go @@ -47,7 +47,7 @@ cd /test ioutil.WriteFile(filepath.Join(tempDir, "runPiper.sh"), []byte(testScript), 0700) reqNode := testcontainers.ContainerRequest{ - Image: "getgauge/gocd-jdk-mvn-node", + Image: "node:lts-buster", Cmd: []string{"tail", "-f"}, BindMounts: map[string]string{ pwd: "/piperbin", From 732ddda2287922adc389687bfdb019af0cda77b1 Mon Sep 17 00:00:00 2001 From: Ashly Mathew Date: Tue, 11 Jul 2023 15:50:59 +0200 Subject: [PATCH 13/13] Revert "change docker image for guage" This reverts commit 45ac7ca9bc86ae98dbd4b6a146d84680228774d4. --- integration/integration_gauge_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/integration_gauge_test.go b/integration/integration_gauge_test.go index 5d9fbfc93f..c0561c025e 100644 --- a/integration/integration_gauge_test.go +++ b/integration/integration_gauge_test.go @@ -47,7 +47,7 @@ cd /test ioutil.WriteFile(filepath.Join(tempDir, "runPiper.sh"), []byte(testScript), 0700) reqNode := testcontainers.ContainerRequest{ - Image: "node:lts-buster", + Image: "getgauge/gocd-jdk-mvn-node", Cmd: []string{"tail", "-f"}, BindMounts: map[string]string{ pwd: "/piperbin",