Skip to content

Commit

Permalink
adding pagination to get more than 10k results
Browse files Browse the repository at this point in the history
  • Loading branch information
@igorlombacx
igorlombacx committed Oct 28, 2023
1 parent 5933079 commit 2d31f7d
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 64 deletions.
83 changes: 33 additions & 50 deletions internal/commands/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ func summaryReport(
policies *wrappers.PolicyResponseModel,
risksOverviewWrapper wrappers.RisksOverviewWrapper,
resultsWrapper wrappers.ResultsWrapper,
results *wrappers.ScanResultsCollection,
) (*wrappers.ResultSummary, error) {
if summary.HasAPISecurity() {
apiSecRisks, err := getResultsForAPISecScanner(risksOverviewWrapper, summary.ScanID)
Expand All @@ -407,10 +408,7 @@ func summaryReport(
summary.Policies = filterViolatedRules(*policies)
}

err := enhanceWithScanSummary(summary, resultsWrapper)
if err != nil {
return nil, err
}
enhanceWithScanSummary(summary, resultsWrapper, results)

setNotAvailableNumberIfZero(summary, &summary.SastIssues, commonParams.SastType)
setNotAvailableNumberIfZero(summary, &summary.ScaIssues, commonParams.ScaType)
Expand Down Expand Up @@ -441,51 +439,11 @@ func setRiskMsgAndStyle(summary *wrappers.ResultSummary) {
}
}

func enhanceWithScanSummary(summary *wrappers.ResultSummary, resultsWrapper wrappers.ResultsWrapper) error {
scanSummary, errModel, err := resultsWrapper.GetScanSummariesByScanIDS(map[string]string{
commonParams.ScanIDsQueryParam: summary.ScanID,
})
if err != nil {
return err
}
if errModel != nil {
return errors.Errorf("%s: CODE: %d, %s", failedGettingScan, errModel.Code, errModel.Message)
}

if len(scanSummary.ScansSummaries) != 1 {
return errors.Errorf("error - scan summary is nil or has more than one element")
func enhanceWithScanSummary(summary *wrappers.ResultSummary, resultsWrapper wrappers.ResultsWrapper, results *wrappers.ScanResultsCollection) {

Check failure on line 442 in internal/commands/result.go

View workflow job for this annotation

GitHub Actions / lint 

`enhanceWithScanSummary` - `resultsWrapper` is unused (unparam)
for _, result := range results.Results {
countResult(summary, result)
}

updateSummaryWithScanSummary(summary, &scanSummary.ScansSummaries[0])
summary.TotalIssues = summary.SastIssues + summary.ScaIssues + summary.KicsIssues
return nil
}

func updateSummaryWithScanSummary(summary *wrappers.ResultSummary, scanSummary *wrappers.ScanSumaries) {
summary.SastIssues += scanSummary.SastCounters.TotalCounter
summary.KicsIssues += scanSummary.KicsCounters.TotalCounter
summary.ScaIssues += scanSummary.ScaCounters.TotalCounter
summary.ScaIssues += scanSummary.ScaContainersCounters.TotalVulnerabilitiesCounter

updateSeverityCounts(summary, scanSummary.SastCounters.SeverityCounters)
updateSeverityCounts(summary, scanSummary.KicsCounters.SeverityCounters)
updateSeverityCounts(summary, scanSummary.ScaCounters.SeverityCounters)
updateSeverityCounts(summary, scanSummary.ScaContainersCounters.SeverityCounters)
}

func updateSeverityCounts(summary *wrappers.ResultSummary, severityCounts []wrappers.SeverityCounters) {
for _, sev := range severityCounts {
switch strings.ToLower(sev.Severity) {
case highLabel:
summary.HighIssues += sev.Counter
case mediumLabel:
summary.MediumIssues += sev.Counter
case lowLabel:
summary.LowIssues += sev.Counter
case infoLabel:
summary.InfoIssues += sev.Counter
}
}
}

func writeHTMLSummary(targetFile string, summary *wrappers.ResultSummary) error {
Expand Down Expand Up @@ -742,8 +700,7 @@ func CreateScanReport(
if err != nil {
return err
}
isResultsNeeded := verifyFormatsByReportList(reportList, resultsFormats...)
if isResultsNeeded && !scanPending {
if !scanPending {
results, err = ReadResults(resultsWrapper, scan, params)
if err != nil {
return err
Expand All @@ -752,7 +709,7 @@ func CreateScanReport(
}
isSummaryNeeded := verifyFormatsByReportList(reportList, summaryFormats...)
if isSummaryNeeded && !scanPending {
summary, err = summaryReport(summary, policyResponseModel, risksOverviewWrapper, resultsWrapper)
summary, err = summaryReport(summary, policyResponseModel, risksOverviewWrapper, resultsWrapper, results)
if err != nil {
return err
}
Expand All @@ -767,6 +724,32 @@ func CreateScanReport(
return nil
}

func countResult(summary *wrappers.ResultSummary, result *wrappers.ScanResult) {
engineType := strings.TrimSpace(result.Type)
if contains(summary.EnginesEnabled, engineType) && isExploitable(result.State) {
if engineType == commonParams.SastType {
summary.SastIssues++
summary.TotalIssues++
} else if engineType == commonParams.ScaType {
summary.ScaIssues++
summary.TotalIssues++
} else if engineType == commonParams.KicsType {
summary.KicsIssues++
summary.TotalIssues++
}
severity := strings.ToLower(result.Severity)
if severity == highLabel {
summary.HighIssues++
} else if severity == lowLabel {
summary.LowIssues++
} else if severity == mediumLabel {
summary.MediumIssues++
} else if severity == infoLabel {
summary.InfoIssues++
}
}
}

func verifyFormatsByReportList(reportFormats []string, formats ...string) bool {
for _, reportFormat := range reportFormats {
for _, format := range formats {
Expand Down
74 changes: 60 additions & 14 deletions internal/wrappers/results-http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@ package wrappers
import (
"encoding/json"
"fmt"
"net/http"

"github.com/checkmarx/ast-cli/internal/logger"
commonParams "github.com/checkmarx/ast-cli/internal/params"
"github.com/spf13/viper"
"net/http"

Check failure on line 9 in internal/wrappers/results-http.go

View workflow job for this annotation

GitHub Actions / lint 

File is not `goimports`-ed with -local github.com/golangci/golangci-lint (goimports)

"github.com/pkg/errors"
)
Expand All @@ -19,6 +18,9 @@ const (
respStatusCode = "response status code %d"
sort = "sort"
sortResultsDefault = "-severity"
offset = "offset"
astAPIPageLen = 1000
astAPIPagingValue = "1000"
)

type ResultsHTTPWrapper struct {
Expand All @@ -40,14 +42,53 @@ func (r *ResultsHTTPWrapper) GetAllResultsByScanID(params map[string]string) (
*WebError,
error,
) {
clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
// AST has a limit of 10000 results, this makes it get all of them
DefaultMapValue(params, limit, limitValue)
var scanModelslice []ScanResultsCollection
var scanModel ScanResultsCollection
DefaultMapValue(params, limit, astAPIPagingValue)
DefaultMapValue(params, sort, sortResultsDefault)

resp, err := SendPrivateHTTPRequestWithQueryParams(http.MethodGet, r.resultsPath, params, http.NoBody, clientTimeout)
webErr, err := getResultsWithPagination(r.resultsPath, params, &scanModelslice)
if err != nil {
return nil, nil, err
return &scanModel, nil, err
}
if webErr != nil {
return &scanModel, webErr, nil
}
for _, resultsPage := range scanModelslice {
scanModel.Results = append(scanModel.Results, resultsPage.Results...)
}
return &scanModel, nil, nil
}
func getResultsWithPagination(resultPath string, queryParams map[string]string, slice *[]ScanResultsCollection) (*WebError, error) {
clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
var currentPage = 0
for {
queryParams[offset] = fmt.Sprintf("%d", currentPage)
target, hasNextPage, weberr, err := getResultsByOffset(resultPath, queryParams, currentPage, clientTimeout)
if err != nil {
return nil, err
}

if weberr != nil {
return weberr, nil
}

*slice = append(*slice, *target)

if !hasNextPage {
break
}
if astAPIPageLen > int(target.TotalCount) {
break
}
currentPage += astAPIPageLen
}
return nil, nil
}
func getResultsByOffset(resultPath string, params map[string]string, currentPage int, clientTimeout uint) (*ScanResultsCollection, bool, *WebError, error) {
resp, err := SendPrivateHTTPRequestWithQueryParams(http.MethodGet, resultPath, params, http.NoBody, clientTimeout)
if err != nil {
return nil, false, nil, err
}

defer func() {
Expand All @@ -61,22 +102,26 @@ func (r *ResultsHTTPWrapper) GetAllResultsByScanID(params map[string]string) (
errorModel := WebError{}
err = decoder.Decode(&errorModel)
if err != nil {
return nil, nil, errors.Wrapf(err, failedToParseGetResults)
return nil, false, nil, errors.Wrapf(err, failedToParseGetResults)
}
return nil, &errorModel, nil
return nil, false, &errorModel, nil
case http.StatusOK:
model := ScanResultsCollection{}
err = decoder.Decode(&model)
if err != nil {
return nil, nil, errors.Wrapf(err, failedToParseGetResults)
return nil, false, nil, errors.Wrapf(err, failedToParseGetResults)
}

return &model, nil, nil
if err != nil {
return nil, false, nil, errors.Wrapf(err, failedToParseGetResults)
}
if int(model.TotalCount) < currentPage {
return &model, false, nil, nil
}
return &model, true, nil, nil
default:
return nil, nil, errors.Errorf(respStatusCode, resp.StatusCode)
return nil, false, nil, errors.Errorf(respStatusCode, resp.StatusCode)
}
}

func (r *ResultsHTTPWrapper) GetAllResultsPackageByScanID(params map[string]string) (
*[]ScaPackageCollection,
*WebError,
Expand Down Expand Up @@ -184,6 +229,7 @@ func (r *ResultsHTTPWrapper) GetResultsURL(projectID string) (string, error) {
return baseURI, nil
}

// GetScanSummariesByScanIDS will no longer be used because it does not support --filters flag
func (r *ResultsHTTPWrapper) GetScanSummariesByScanIDS(params map[string]string) (
*ScanSummariesModel,
*WebError,
Expand Down

0 comments on commit 2d31f7d

Please sign in to comment.