From f39dec68a538ac3fc17be4239070ba4989e5e3ac Mon Sep 17 00:00:00 2001 From: michaelkubiaczyk <48311127+michaelkubiaczyk@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:24:03 +0100 Subject: [PATCH] Cxone updated release (#4723) * Initial in progress * compiling but not yet functional * Missed file * updated checkmarxone step * Working up to fetching a project then breaks * Missed file * Breaks when retrieving projects+proxy set * Create project & run scan working, now polling * Fixed polling * added back the zipfile remove command * Fixed polling again * Generates and downloads PDF report * Updated and working, prep for refactor * Added compliance steps * Cleanup, reporting, added groovy connector * fixed groovy file * checkmarxone to checkmarxOne * checkmarxone to checkmarxOne * split credentials (id+secret, apikey), renamed pullrequestname to branch, groovy fix * Fixed filenames & yaml * missed the metadata_generated.go * added json to sarif conversion * fix:type in new checkmarxone package * fix:type in new checkmarxone package * removed test logs, added temp error log for creds * extra debugging to fix crash * improved auth logging, fixed query parse issue * fixed bug with group fetch when using oauth user * CWE can be -1 if not defined, can't be uint * Query also had CweID * Disabled predicates-fetch in sarif generation * Removing leftover info log message * Better error handling * fixed default preset configuration * removing .bat files - sorry * Cleanup per initial review * refactoring per Gist, fixed project find, add apps * small fix - sorry for commit noise while testing * Fixing issues with incremental scans. * removing maxretries * Updated per PR feedback, further changes todo toda * JSON Report changes and reporting cleanup * removing .bat (again?) * adding docs, groovy unit test, linter fixes * Started adding tests maybe 15% covered * fix(checkmarxOne): test cases for pkg and reporting * fix(checkmarxOne):fix formatting * feat(checkmarxone): update interface with missing method * feat(checkmarxone):change runStep signature to be able to inject dependency * feat(checkmarxone): add tests for step (wip) * Adding a bit more coverage * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix code review * feat(checkmarxOne): fix integration test PR * adding scan-summary bug workaround, reportgen fail * enforceThresholds fix when no results passed in * fixed gap when preset empty in yaml & project conf * fixed another gap in preset selection * fix 0-result panic * fail when no preset is set anywhere * removed comment * initial project-under-app support * fixing sarif reportgen * some cleanup of error messages * post-merge test fixes * revert previous upstream merge * adding "incremental" to "full" triggers * wrong boolean * project-in-application api change prep * Fixing SARIF report without preset access * fix sarif deeplink * removing comments * fix(cxone):formatting * fix(cxone):formatting --------- Co-authored-by: thtri Co-authored-by: Thanh-Hai Trinh --- cmd/checkmarxOneExecuteScan.go | 40 ++++++++++++++++++++++++++-- cmd/checkmarxOneExecuteScan_test.go | 4 +++ pkg/checkmarxone/checkmarxone.go | 41 ++++++++++++++++++++++------- pkg/checkmarxone/cxjson_to_sarif.go | 39 ++++++++++----------------- 4 files changed, 88 insertions(+), 36 deletions(-) diff --git a/cmd/checkmarxOneExecuteScan.go b/cmd/checkmarxOneExecuteScan.go index 80bcc01e93..af790a3487 100644 --- a/cmd/checkmarxOneExecuteScan.go +++ b/cmd/checkmarxOneExecuteScan.go @@ -141,6 +141,10 @@ func runStep(config checkmarxOneExecuteScanOptions, influx *checkmarxOneExecuteS return fmt.Errorf("failed to determine incremental or full scan configuration: %s", err) } + if config.Incremental { + log.Entry().Warnf("If you change your file filter pattern it is recommended to run a Full scan instead of an incremental, to ensure full code coverage.") + } + zipFile, err := cx1sh.ZipFiles() if err != nil { return fmt.Errorf("failed to create zip file: %s", err) @@ -302,10 +306,31 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error { } currentPreset := "" + currentLanguageMode := "multi" // piper default for _, conf := range projectConf { if conf.Key == "scan.config.sast.presetName" { currentPreset = conf.Value - break + } + if conf.Key == "scan.config.sast.languageMode" { + currentLanguageMode = conf.Value + } + } + + if c.config.LanguageMode == "" || strings.EqualFold(c.config.LanguageMode, "multi") { // default multi if blank + if currentLanguageMode != "multi" { + log.Entry().Info("Pipeline yaml requests multi-language scan - updating project configuration") + c.sys.SetProjectLanguageMode(c.Project.ProjectID, "multi", true) + + if c.config.Incremental { + log.Entry().Warn("Pipeline yaml requests incremental scan, but switching from 'primary' to 'multi' language mode requires a full scan - switching from incremental to full") + c.config.Incremental = false + } + } + } else { // primary language mode + if currentLanguageMode != "primary" { + log.Entry().Info("Pipeline yaml requests primary-language scan - updating project configuration") + c.sys.SetProjectLanguageMode(c.Project.ProjectID, "primary", true) + // no need to switch incremental to full here (multi-language scan includes single-language scan coverage) } } @@ -319,6 +344,11 @@ func (c *checkmarxOneExecuteScanHelper) SetProjectPreset() error { } else if currentPreset != c.config.Preset { log.Entry().Infof("Project configured preset (%v) does not match pipeline yaml (%v) - updating project configuration.", currentPreset, c.config.Preset) c.sys.SetProjectPreset(c.Project.ProjectID, c.config.Preset, true) + + if c.config.Incremental { + log.Entry().Warn("Changing project settings requires a full scan to take effect - switching from incremental to full") + c.config.Incremental = false + } } else { log.Entry().Infof("Project is already configured to use pipeline preset %v", currentPreset) } @@ -717,7 +747,13 @@ func (c *checkmarxOneExecuteScanHelper) getDetailedResults(scan *checkmarxOne.Sc resultMap["LinesOfCodeScanned"] = scanmeta.LOC resultMap["FilesScanned"] = scanmeta.FileCount - resultMap["ToolVersion"] = "Cx1 Gap: No API for this" + + version, err := c.sys.GetVersion() + if err != nil { + resultMap["ToolVersion"] = "Error fetching current version" + } else { + resultMap["ToolVersion"] = fmt.Sprintf("CxOne: %v, SAST: %v, KICS: %v", version.CxOne, version.SAST, version.KICS) + } if scanmeta.IsIncremental { resultMap["ScanType"] = "Incremental" diff --git a/cmd/checkmarxOneExecuteScan_test.go b/cmd/checkmarxOneExecuteScan_test.go index 96db7b8d26..453e8a1780 100644 --- a/cmd/checkmarxOneExecuteScan_test.go +++ b/cmd/checkmarxOneExecuteScan_test.go @@ -240,6 +240,10 @@ func (sys *checkmarxOneSystemMock) UpdateProjectConfiguration(projectID string, return nil } +func (sys *checkmarxOneSystemMock) GetVersion() (checkmarxOne.VersionInfo, error) { + return checkmarxOne.VersionInfo{}, nil +} + type checkmarxOneExecuteScanHelperMock struct { ctx context.Context config checkmarxOneExecuteScanOptions diff --git a/pkg/checkmarxone/checkmarxone.go b/pkg/checkmarxone/checkmarxone.go index 296d64a992..db22059424 100644 --- a/pkg/checkmarxone/checkmarxone.go +++ b/pkg/checkmarxone/checkmarxone.go @@ -259,6 +259,12 @@ type Status struct { Details ScanStatusDetails `json:"details"` } +type VersionInfo struct { + CxOne string `json:"CxOne"` + KICS string `json:"KICS"` + SAST string `json:"SAST"` +} + type WorkflowLog struct { Source string `json:"Source"` Info string `json:"Info"` @@ -327,6 +333,8 @@ type System interface { GetProjectConfiguration(projectID string) ([]ProjectConfigurationSetting, error) UpdateProjectConfiguration(projectID string, settings []ProjectConfigurationSetting) error + + GetVersion() (VersionInfo, error) } // NewSystemInstance returns a new Checkmarx client for communicating with the backend @@ -833,6 +841,16 @@ func (sys *SystemInstance) CreateProjectInApplication(projectName, applicationID header.Set("Content-Type", "application/json") data, err := sendRequest(sys, http.MethodPost, fmt.Sprintf("/projects/application/%v", applicationID), bytes.NewBuffer(jsonValue), header, []int{}) + + if err != nil && err.Error()[0:8] == "HTTP 404" { // At some point, the api /projects/applications will be removed and instead the normal /projects API will do the job. + jsonData["applicationIds"] = []string{applicationID} + jsonValue, err = json.Marshal(data) + if err != nil { + return project, err + } + data, err = sendRequest(sys, http.MethodPost, "/projects", bytes.NewReader(jsonValue), header, []int{}) + } + if err != nil { return project, errors.Wrapf(err, "failed to create project %v under %v", projectName, applicationID) } @@ -973,10 +991,6 @@ func (sys *SystemInstance) ScanProject(projectID, sourceUrl, branch, scanType st return Scan{}, errors.New("Invalid scanType provided, must be 'upload' or 'git'") } -//func (sys *SystemInstance) UpdateProjectExcludeSettings(projectID string, excludeFolders string, excludeFiles string) error { -// replaced by SetProjectFileFilter - -// Updated for Cx1: GetPresets loads the preset values defined in the Checkmarx backend func (sys *SystemInstance) GetPresets() ([]Preset, error) { sys.logger.Debug("Getting Presets...") var presets []Preset @@ -991,7 +1005,6 @@ func (sys *SystemInstance) GetPresets() ([]Preset, error) { return presets, err } -// New for Cx1 func (sys *SystemInstance) GetProjectConfiguration(projectID string) ([]ProjectConfigurationSetting, error) { sys.logger.Debug("Getting project configuration") var projectConfigurations []ProjectConfigurationSetting @@ -1009,8 +1022,6 @@ func (sys *SystemInstance) GetProjectConfiguration(projectID string) ([]ProjectC return projectConfigurations, err } -// UpdateProjectConfiguration updates the configuration of the project addressed by projectID -// Updated for Cx1 func (sys *SystemInstance) UpdateProjectConfiguration(projectID string, settings []ProjectConfigurationSetting) error { if len(settings) == 0 { return errors.New("Empty list of settings provided.") @@ -1101,7 +1112,6 @@ func (sys *SystemInstance) GetScanMetadata(scanID string) (ScanMetadata, error) return scanmeta, nil } -// GetScans returns all scan status on the project addressed by projectID func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error) { var workflow []WorkflowLog @@ -1115,7 +1125,6 @@ func (sys *SystemInstance) GetScanWorkflow(scanID string) ([]WorkflowLog, error) return workflow, nil } -// GetScans returns all scan status on the project addressed by projectID func (sys *SystemInstance) GetLastScans(projectID string, limit int) ([]Scan, error) { var scanResponse struct { TotalCount uint64 @@ -1379,3 +1388,17 @@ func (sys *SystemInstance) DownloadReport(reportUrl string) ([]byte, error) { } return data, nil } + +func (sys *SystemInstance) GetVersion() (VersionInfo, error) { + sys.logger.Debug("Getting Version information...") + var version VersionInfo + + data, err := sendRequest(sys, http.MethodGet, "/versions", nil, http.Header{}, []int{}) + if err != nil { + sys.logger.Errorf("Fetching versions failed: %s", err) + return version, err + } + + err = json.Unmarshal(data, &version) + return version, err +} diff --git a/pkg/checkmarxone/cxjson_to_sarif.go b/pkg/checkmarxone/cxjson_to_sarif.go index f9b1112749..27d8e0cecc 100644 --- a/pkg/checkmarxone/cxjson_to_sarif.go +++ b/pkg/checkmarxone/cxjson_to_sarif.go @@ -8,7 +8,6 @@ import ( "github.com/SAP/jenkins-library/pkg/format" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" - "github.com/pkg/errors" ) // ConvertCxJSONToSarif is the entrypoint for the Parse function @@ -24,14 +23,9 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul sarif.Runs = append(sarif.Runs, checkmarxRun) rulesArray := []format.SarifRule{} - queries, err := sys.GetQueries() - if err != nil { - return sarif, errors.Wrap(err, "Failed to retrieve list of queries") - } - - baseURL := "https://" + serverURL + "/results/" + scanMeta.ScanID + "/" + scanMeta.ProjectID + baseURL := serverURL + "/results/" + scanMeta.ScanID + "/" + scanMeta.ProjectID - cweIdsForTaxonomies := make(map[int64]int) //use a map to avoid duplicates + cweIdsForTaxonomies := make(map[int]int) //use a map to avoid duplicates cweCounter := 0 //maxretries := 5 @@ -41,15 +35,10 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul log.Entry().Debug("[SARIF] Now handling results.") for _, r := range *scanResults { - query := getQuery(queries, r.Data.QueryID) - if query == nil { - return sarif, errors.New(fmt.Sprintf("Unknown queryid in results: %d", r.Data.QueryID)) - } - - _, haskey := cweIdsForTaxonomies[query.CweID] + _, haskey := cweIdsForTaxonomies[r.VulnerabilityDetails.CweId] if !haskey { - cweIdsForTaxonomies[query.CweID] = cweCounter + cweIdsForTaxonomies[r.VulnerabilityDetails.CweId] = cweCounter cweCounter++ } @@ -59,14 +48,14 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul result := *new(format.Results) //General - result.RuleID = fmt.Sprintf("checkmarxOne-%v/%d", query.Language, query.QueryID) - result.RuleIndex = cweIdsForTaxonomies[query.CweID] + result.RuleID = fmt.Sprintf("checkmarxOne-%v/%d", r.Data.LanguageName, r.Data.QueryID) + result.RuleIndex = cweIdsForTaxonomies[r.VulnerabilityDetails.CweId] result.Level = "none" msg := new(format.Message) if apiDescription != "" { msg.Text = apiDescription } else { - msg.Text = query.Name + msg.Text = r.Data.QueryName } result.Message = msg @@ -199,18 +188,18 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul //handle the rules array rule := *new(format.SarifRule) - rule.ID = fmt.Sprintf("checkmarxOne-%v/%d", query.Language, query.QueryID) - words := strings.Split(query.Name, "_") + rule.ID = fmt.Sprintf("checkmarxOne-%v/%d", r.Data.LanguageName, r.Data.QueryID) + words := strings.Split(r.Data.QueryName, "_") for w := 0; w < len(words); w++ { words[w] = piperutils.Title(strings.ToLower(words[w])) } rule.Name = strings.Join(words, "") - rule.HelpURI = fmt.Sprintf("%v/sast/description/%v/%v", baseURL, query.QueryDescriptionID, query.QueryID) + rule.HelpURI = fmt.Sprintf("%v/sast/description/%v/%v", baseURL, r.VulnerabilityDetails.CweId, r.Data.QueryID) rule.Help = new(format.Help) rule.Help.Text = rule.HelpURI rule.ShortDescription = new(format.Message) - rule.ShortDescription.Text = query.Name + rule.ShortDescription.Text = r.Data.QueryName rule.Properties = new(format.SarifRuleProperties) if len(r.VulnerabilityDetails.Compliances) > 0 { @@ -221,7 +210,7 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul rule.Properties.Tags = append(rule.Properties.Tags, r.VulnerabilityDetails.Compliances[cat]) } } - switch query.Severity { + switch r.Severity { case "INFORMATION": rule.Properties.SecuritySeverity = "0.0" case "LOW": @@ -234,8 +223,8 @@ func ConvertCxJSONToSarif(sys System, serverURL string, scanResults *[]ScanResul rule.Properties.SecuritySeverity = "10.0" } - if query.CweID != 0 { - rule.Properties.Tags = append(rule.Properties.Tags, fmt.Sprintf("external/cwe/cwe-%d", query.CweID)) + if r.VulnerabilityDetails.CweId != 0 { + rule.Properties.Tags = append(rule.Properties.Tags, fmt.Sprintf("external/cwe/cwe-%d", r.VulnerabilityDetails.CweId)) } rulesArray = append(rulesArray, rule) }