Cxone updated release (#4723)

* Initial in progress

* compiling but not yet functional

* Missed file

* updated checkmarxone step

* Working up to fetching a project then breaks

* Missed file

* Breaks when retrieving projects+proxy set

* Create project & run scan working, now polling

* Fixed polling

* added back the zipfile remove command

* Fixed polling again

* Generates and downloads PDF report

* Updated and working, prep for refactor

* Added compliance steps

* Cleanup, reporting, added groovy connector

* fixed groovy file

* checkmarxone to checkmarxOne

* checkmarxone to checkmarxOne

* split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix

* Fixed filenames & yaml

* missed the metadata_generated.go

* added json to sarif conversion

* fix:type in new checkmarxone package

* fix:type in new checkmarxone package

* removed test logs, added temp error log for creds

* extra debugging to fix crash

* improved auth logging, fixed query parse issue

* fixed bug with group fetch when using oauth user

* CWE can be -1 if not defined, can't be uint

* Query also had CweID

* Disabled predicates-fetch in sarif generation

* Removing leftover info log message

* Better error handling

* fixed default preset configuration

* removing .bat files - sorry

* Cleanup per initial review

* refactoring per Gist, fixed project find, add apps

* small fix - sorry for commit noise while testing

* Fixing issues with incremental scans.

* removing maxretries

* Updated per PR feedback, further changes todo toda

* JSON Report changes and reporting cleanup

* removing .bat (again?)

* adding docs, groovy unit test, linter fixes

* Started adding tests maybe 15% covered

* fix(checkmarxOne): test cases for pkg and reporting

* fix(checkmarxOne):fix formatting

* feat(checkmarxone): update interface with missing method

* feat(checkmarxone):change runStep signature to be able to inject dependency

* feat(checkmarxone): add tests for step (wip)

* Adding a bit more coverage

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix code review

* feat(checkmarxOne): fix integration test PR

* adding scan-summary bug workaround, reportgen fail

* enforceThresholds fix when no results passed in

* fixed gap when preset empty in yaml & project conf

* fixed another gap in preset selection

* fix 0-result panic

* fail when no preset is set anywhere

* removed comment

* initial project-under-app support

* fixing sarif reportgen

* some cleanup of error messages

* post-merge test fixes

* revert previous upstream merge

* adding "incremental" to "full" triggers

* wrong boolean

* project-in-application api change prep

* Fixing SARIF report without preset access

* fix sarif deeplink

* removing comments

* fix(cxone):formatting

* fix(cxone):formatting

---------

Co-authored-by: thtri <[email protected]>
Co-authored-by: Thanh-Hai Trinh <[email protected]>

cmd/checkmarxOneExecuteScan.go

@@ -141,6 +141,10 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS
 		return fmt.Errorf("failed to determine incremental or full scan configuration: %s", err)
 	}
 
+	if config.Incremental {
+		log.Entry().Warnf("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.")
+	}
+
 	zipFile, err := cx1sh.ZipFiles()
 	if err != nil {
 		return fmt.Errorf("failed to create zip file: %s", err)
@@ -302,10 +306,31 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error {
 	}
 
 	currentPreset := ""
+	currentLanguageMode := "multi" // piper default
 	for _, conf := range projectConf {
 		if conf.Key == "scan.config.sast.presetName" {
 			currentPreset = conf.Value
-			break
+		}
+		if conf.Key == "scan.config.sast.languageMode" {
+			currentLanguageMode = conf.Value
+		}
+	}
+
+	if c.config.LanguageMode == "" || strings.EqualFold(c.config.LanguageMode, "multi") { // default multi if blank
+		if currentLanguageMode != "multi" {
+			log.Entry().Info("Pipeline yaml requests multi-language scan - updating project configuration")
+			c.sys.SetProjectLanguageMode(c.Project.ProjectID, "multi", true)
+
+			if c.config.Incremental {
+				log.Entry().Warn("Pipeline yaml requests incremental scan, but switching from 'primary' to 'multi' language mode requires a full scan - switching from incremental to full")
+				c.config.Incremental = false
+			}
+		}
+	} else { // primary language mode
+		if currentLanguageMode != "primary" {
+			log.Entry().Info("Pipeline yaml requests primary-language scan - updating project configuration")
+			c.sys.SetProjectLanguageMode(c.Project.ProjectID, "primary", true)
+			// no need to switch incremental to full here (multi-language scan includes single-language scan coverage)
 		}
 	}
 
@@ -319,6 +344,11 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error {
 	} else if currentPreset != c.config.Preset {
 		log.Entry().Infof("Project configured preset (%v) does not match pipeline yaml (%v) - updating project configuration.", currentPreset, c.config.Preset)
 		c.sys.SetProjectPreset(c.Project.ProjectID, c.config.Preset, true)
+
+		if c.config.Incremental {
+			log.Entry().Warn("Changing project settings requires a full scan to take effect - switching from incremental to full")
+			c.config.Incremental = false
+		}
 	} else {
 		log.Entry().Infof("Project is already configured to use pipeline preset %v", currentPreset)
 	}
@@ -717,7 +747,13 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc
 
 	resultMap["LinesOfCodeScanned"] = scanmeta.LOC
 	resultMap["FilesScanned"] = scanmeta.FileCount
-	resultMap["ToolVersion"] = "Cx1 Gap: No API for this"
+
+	version, err := c.sys.GetVersion()
+	if err != nil {
+		resultMap["ToolVersion"] = "Error fetching current version"
+	} else {
+		resultMap["ToolVersion"] = fmt.Sprintf("CxOne: %v, SAST: %v, KICS: %v", version.CxOne, version.SAST, version.KICS)
+	}
 
 	if scanmeta.IsIncremental {
 		resultMap["ScanType"] = "Incremental"

cmd/checkmarxOneExecuteScan_test.go

@@ -240,6 +240,10 @@ func (sys *checkmarxOneSystemMock) UpdateProjectConfiguration(projectID string,
 	return nil
 }
 
+func (sys *checkmarxOneSystemMock) GetVersion() (checkmarxOne.VersionInfo, error) {
+	return checkmarxOne.VersionInfo{}, nil
+}
+
 type checkmarxOneExecuteScanHelperMock struct {
 	ctx     context.Context
 	config  checkmarxOneExecuteScanOptions

pkg/checkmarxone/checkmarxone.go

@@ -259,6 +259,12 @@ type Status struct {
 	Details ScanStatusDetails `json:"details"`
 }
 
+type VersionInfo struct {
+	CxOne string `json:"CxOne"`
+	KICS  string `json:"KICS"`
+	SAST  string `json:"SAST"`
+}
+
 type WorkflowLog struct {
 	Source    string `json:"Source"`
 	Info      string `json:"Info"`
@@ -327,6 +333,8 @@ type System interface {
 
 	GetProjectConfiguration(projectID string) ([]ProjectConfigurationSetting, error)
 	UpdateProjectConfiguration(projectID string, settings []ProjectConfigurationSetting) error
+
+	GetVersion() (VersionInfo, error)
 }
 
 // NewSystemInstance returns a new Checkmarx client for communicating with the backend
@@ -833,6 +841,16 @@ func (sys *SystemInstance) CreateProjectInApplication(projectName, applicationID
 	header.Set("Content-Type", "application/json")
 
 	data, err := sendRequest(sys, http.MethodPost, fmt.Sprintf("/projects/application/%v", applicationID), bytes.NewBuffer(jsonValue), header, []int{})
+
+	if err != nil && err.Error()[0:8] == "HTTP 404" { // At some point, the api /projects/applications will be removed and instead the normal /projects API will do the job.
+		jsonData["applicationIds"] = []string{applicationID}
+		jsonValue, err = json.Marshal(data)
+		if err != nil {
+			return project, err
+		}
+		data, err = sendRequest(sys, http.MethodPost, "/projects", bytes.NewReader(jsonValue), header, []int{})
+	}
+
 	if err != nil {
 		return project, errors.Wrapf(err, "failed to create project %v under %v", projectName, applicationID)
 	}
@@ -973,10 +991,6 @@ func (sys *SystemInstance) ScanProject(projectID, sourceUrl, branch, scanType st
 	return Scan{}, errors.New("Invalid scanType provided, must be 'upload' or 'git'")
 }
 
-//func (sys *SystemInstance) UpdateProjectExcludeSettings(projectID string, excludeFolders string, excludeFiles string) error {
-// replaced by SetProjectFileFilter
-
-// Updated for Cx1: GetPresets loads the preset values defined in the Checkmarx backend
 func (sys *SystemInstance) GetPresets() ([]Preset, error) {
 	sys.logger.Debug("Getting Presets...")
 	var presets []Preset
@@ -991,7 +1005,6 @@ func (sys *SystemInstance) GetPresets() ([]Preset, error) {
 	return presets, err
 }
 
-// New for Cx1
 func (sys *SystemInstance) GetProjectConfiguration(projectID string) ([]ProjectConfigurationSetting, error) {
 	sys.logger.Debug("Getting project configuration")
 	var projectConfigurations []ProjectConfigurationSetting
@@ -1009,8 +1022,6 @@ func (sys *SystemInstance) GetProjectConfiguration(projectID string) ([]ProjectC
 	return projectConfigurations, err
 }
 
-// UpdateProjectConfiguration updates the configuration of the project addressed by projectID
-// Updated for Cx1
 func (sys *SystemInstance) UpdateProjectConfiguration(projectID string, settings []ProjectConfigurationSetting) error {
 	if len(settings) == 0 {
 		return errors.New("Empty list of settings provided.")
@@ -1101,7 +1112,6 @@ func (sys *SystemInstance) GetScanMetadata(scanID string) (ScanMetadata, error)
 	return scanmeta, nil
 }
 
-// GetScans returns all scan status on the project addressed by projectID
 func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error) {
 	var workflow []WorkflowLog
 
@@ -1115,7 +1125,6 @@ func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error)
 	return workflow, nil
 }
 
-// GetScans returns all scan status on the project addressed by projectID
 func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, error) {
 	var scanResponse struct {
 		TotalCount         uint64
@@ -1379,3 +1388,17 @@ func (sys *SystemInstance) DownloadReport(reportUrl string) ([]byte, error) {
 	}
 	return data, nil
 }
+
+func (sys *SystemInstance) GetVersion() (VersionInfo, error) {
+	sys.logger.Debug("Getting Version information...")
+	var version VersionInfo
+
+	data, err := sendRequest(sys, http.MethodGet, "/versions", nil, http.Header{}, []int{})
+	if err != nil {
+		sys.logger.Errorf("Fetching versions failed: %s", err)
+		return version, err
+	}
+
+	err = json.Unmarshal(data, &version)
+	return version, err
+}

pkg/checkmarxone/cxjson_to_sarif.go

@@ -8,7 +8,6 @@ import (
 	"github.com/SAP/jenkins-library/pkg/format"
 	"github.com/SAP/jenkins-library/pkg/log"
 	"github.com/SAP/jenkins-library/pkg/piperutils"
-	"github.com/pkg/errors"
 )
 
 // ConvertCxJSONToSarif is the entrypoint for the Parse function
@@ -24,14 +23,9 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul
 	sarif.Runs = append(sarif.Runs, checkmarxRun)
 	rulesArray := []format.SarifRule{}
 
-	queries, err := sys.GetQueries()
-	if err != nil {
-		return sarif, errors.Wrap(err, "Failed to retrieve list of queries")
-	}
-
-	baseURL := "https://" + serverURL + "/results/" + scanMeta.ScanID + "/" + scanMeta.ProjectID
+	baseURL := serverURL + "/results/" + scanMeta.ScanID + "/" + scanMeta.ProjectID
 
-	cweIdsForTaxonomies := make(map[int64]int) //use a map to avoid duplicates
+	cweIdsForTaxonomies := make(map[int]int) //use a map to avoid duplicates
 	cweCounter := 0
 	//maxretries := 5
 
@@ -41,15 +35,10 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul
 	log.Entry().Debug("[SARIF] Now handling results.")
 
 	for _, r := range *scanResults {
-		query := getQuery(queries, r.Data.QueryID)
-		if query == nil {
-			return sarif, errors.New(fmt.Sprintf("Unknown queryid in results: %d", r.Data.QueryID))
-		}
-
-		_, haskey := cweIdsForTaxonomies[query.CweID]
+		_, haskey := cweIdsForTaxonomies[r.VulnerabilityDetails.CweId]
 
 		if !haskey {
-			cweIdsForTaxonomies[query.CweID] = cweCounter
+			cweIdsForTaxonomies[r.VulnerabilityDetails.CweId] = cweCounter
 			cweCounter++
 		}
 
@@ -59,14 +48,14 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul
 		result := *new(format.Results)
 
 		//General
-		result.RuleID = fmt.Sprintf("checkmarxOne-%v/%d", query.Language, query.QueryID)
-		result.RuleIndex = cweIdsForTaxonomies[query.CweID]
+		result.RuleID = fmt.Sprintf("checkmarxOne-%v/%d", r.Data.LanguageName, r.Data.QueryID)
+		result.RuleIndex = cweIdsForTaxonomies[r.VulnerabilityDetails.CweId]
 		result.Level = "none"
 		msg := new(format.Message)
 		if apiDescription != "" {
 			msg.Text = apiDescription
 		} else {
-			msg.Text = query.Name
+			msg.Text = r.Data.QueryName
 		}
 		result.Message = msg
 
@@ -199,18 +188,18 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul
 		//handle the rules array
 		rule := *new(format.SarifRule)
 
-		rule.ID = fmt.Sprintf("checkmarxOne-%v/%d", query.Language, query.QueryID)
-		words := strings.Split(query.Name, "_")
+		rule.ID = fmt.Sprintf("checkmarxOne-%v/%d", r.Data.LanguageName, r.Data.QueryID)
+		words := strings.Split(r.Data.QueryName, "_")
 		for w := 0; w < len(words); w++ {
 			words[w] = piperutils.Title(strings.ToLower(words[w]))
 		}
 		rule.Name = strings.Join(words, "")
 
-		rule.HelpURI = fmt.Sprintf("%v/sast/description/%v/%v", baseURL, query.QueryDescriptionID, query.QueryID)
+		rule.HelpURI = fmt.Sprintf("%v/sast/description/%v/%v", baseURL, r.VulnerabilityDetails.CweId, r.Data.QueryID)
 		rule.Help = new(format.Help)
 		rule.Help.Text = rule.HelpURI
 		rule.ShortDescription = new(format.Message)
-		rule.ShortDescription.Text = query.Name
+		rule.ShortDescription.Text = r.Data.QueryName
 		rule.Properties = new(format.SarifRuleProperties)
 
 		if len(r.VulnerabilityDetails.Compliances) > 0 {
@@ -221,7 +210,7 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul
 				rule.Properties.Tags = append(rule.Properties.Tags, r.VulnerabilityDetails.Compliances[cat])
 			}
 		}
-		switch query.Severity {
+		switch r.Severity {
 		case "INFORMATION":
 			rule.Properties.SecuritySeverity = "0.0"
 		case "LOW":
@@ -234,8 +223,8 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul
 			rule.Properties.SecuritySeverity = "10.0"
 		}
 
-		if query.CweID != 0 {
-			rule.Properties.Tags = append(rule.Properties.Tags, fmt.Sprintf("external/cwe/cwe-%d", query.CweID))
+		if r.VulnerabilityDetails.CweId != 0 {
+			rule.Properties.Tags = append(rule.Properties.Tags, fmt.Sprintf("external/cwe/cwe-%d", r.VulnerabilityDetails.CweId))
 		}
 		rulesArray = append(rulesArray, rule)
 	}