diff --git a/cmd/codeqlExecuteScan.go b/cmd/codeqlExecuteScan.go index 546a310dd5..cd97f0a981 100644 --- a/cmd/codeqlExecuteScan.go +++ b/cmd/codeqlExecuteScan.go @@ -59,9 +59,23 @@ func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry influx.step_data.fields.codeql = true } -func appendCodeqlQuery(cmd []string, codeqlQuery string) []string { - if len(codeqlQuery) > 0 { - cmd = append(cmd, codeqlQuery) +func appendCodeqlQuerySuite(utils codeqlExecuteScanUtils, cmd []string, querySuite, transformString string) []string { + if len(querySuite) > 0 { + if len(transformString) > 0 { + var bufferOut, bufferErr bytes.Buffer + utils.Stdout(&bufferOut) + defer utils.Stdout(log.Writer()) + utils.Stderr(&bufferErr) + defer utils.Stderr(log.Writer()) + if err := utils.RunExecutable("sh", []string{"-c", fmt.Sprintf("echo %s | sed -E \"%s\"", querySuite, transformString)}...); err != nil { + log.Entry().WithError(err).Error("failed to transform querySuite") + e := bufferErr.String() + log.Entry().Error(e) + } else { + querySuite = strings.TrimSpace(bufferOut.String()) + } + } + cmd = append(cmd, querySuite) } return cmd @@ -271,7 +285,7 @@ func runGithubUploadResults(config *codeqlExecuteScanOptions, repoInfo *codeql.R func executeAnalysis(format, reportName string, customFlags map[string]string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) { moduleTargetPath := filepath.Join(config.ModulePath, "target") report := filepath.Join(moduleTargetPath, reportName) - cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, format, report) + cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, format, report) if err != nil { log.Entry().Errorf("failed to prepare command for codeql database analyze (format=%s)", format) return nil, err @@ -323,11 +337,11 @@ func prepareCmdForDatabaseCreate(customFlags map[string]string, config *codeqlEx return cmd, nil } -func prepareCmdForDatabaseAnalyze(customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) { +func prepareCmdForDatabaseAnalyze(utils codeqlExecuteScanUtils, customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) { cmd := []string{"database", "analyze", "--format=" + format, "--output=" + reportName, config.Database} cmd = codeql.AppendThreadsAndRam(cmd, config.Threads, config.Ram, customFlags) cmd = codeql.AppendCustomFlags(cmd, customFlags) - cmd = appendCodeqlQuery(cmd, config.QuerySuite) + cmd = appendCodeqlQuerySuite(utils, cmd, config.QuerySuite, config.TransformQuerySuite) return cmd, nil } diff --git a/cmd/codeqlExecuteScan_generated.go b/cmd/codeqlExecuteScan_generated.go index bd4ac20aba..91e6118d12 100644 --- a/cmd/codeqlExecuteScan_generated.go +++ b/cmd/codeqlExecuteScan_generated.go @@ -46,6 +46,7 @@ type codeqlExecuteScanOptions struct { DatabaseCreateFlags string `json:"databaseCreateFlags,omitempty"` DatabaseAnalyzeFlags string `json:"databaseAnalyzeFlags,omitempty"` CustomCommand string `json:"customCommand,omitempty"` + TransformQuerySuite string `json:"transformQuerySuite,omitempty"` } type codeqlExecuteScanInflux struct { @@ -273,6 +274,7 @@ func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScan cmd.Flags().StringVar(&stepConfig.DatabaseCreateFlags, "databaseCreateFlags", os.Getenv("PIPER_databaseCreateFlags"), "A space-separated string of flags for the 'codeql database create' command.") cmd.Flags().StringVar(&stepConfig.DatabaseAnalyzeFlags, "databaseAnalyzeFlags", os.Getenv("PIPER_databaseAnalyzeFlags"), "A space-separated string of flags for the 'codeql database analyze' command.") cmd.Flags().StringVar(&stepConfig.CustomCommand, "customCommand", os.Getenv("PIPER_customCommand"), "A custom user-defined command to run between codeql analysis and results upload.") + cmd.Flags().StringVar(&stepConfig.TransformQuerySuite, "transformQuerySuite", os.Getenv("PIPER_transformQuerySuite"), "A transform string that will be applied to the querySuite using the sed command.") cmd.MarkFlagRequired("buildTool") } @@ -538,6 +540,15 @@ func codeqlExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_customCommand"), }, + { + Name: "transformQuerySuite", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_transformQuerySuite"), + }, }, }, Containers: []config.Container{ diff --git a/cmd/codeqlExecuteScan_test.go b/cmd/codeqlExecuteScan_test.go index 775ddea346..2270c6b106 100644 --- a/cmd/codeqlExecuteScan_test.go +++ b/cmd/codeqlExecuteScan_test.go @@ -4,6 +4,8 @@ package cmd import ( + "fmt" + "io" "os" "strings" "testing" @@ -21,7 +23,11 @@ type codeqlExecuteScanMockUtils struct { func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils { utils := codeqlExecuteScanMockUtils{ - ExecMockRunner: &mock.ExecMockRunner{}, + ExecMockRunner: &mock.ExecMockRunner{ + Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error { + return nil + }, + }, FilesMock: &mock.FilesMock{}, HttpClientMock: &mock.HttpClientMock{}, } @@ -406,12 +412,13 @@ func TestPrepareCmdForDatabaseCreate(t *testing.T) { func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { t.Parallel() + utils := codeqlExecuteScanMockUtils{} t.Run("No additional flags, no querySuite, sarif format", func(t *testing.T) { config := &codeqlExecuteScanOptions{ Database: "codeqlDB", } - cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif") + cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 5, len(cmd)) @@ -422,7 +429,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { config := &codeqlExecuteScanOptions{ Database: "codeqlDB", } - cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "csv", "target/codeqlReport.csv") + cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "csv", "target/codeqlReport.csv") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 5, len(cmd)) @@ -434,7 +441,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { Database: "codeqlDB", QuerySuite: "security.ql", } - cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif") + cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 6, len(cmd)) @@ -448,7 +455,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { Threads: "1", Ram: "2000", } - cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif") + cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 8, len(cmd)) @@ -465,7 +472,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { customFlags := map[string]string{ "--threads": "--threads=2", } - cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif") + cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 8, len(cmd)) @@ -482,7 +489,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { customFlags := map[string]string{ "-j": "-j=2", } - cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif") + cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 8, len(cmd)) @@ -499,7 +506,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { customFlags := map[string]string{ "--no-download": "--no-download", } - cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif") + cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif") assert.NoError(t, err) assert.NotEmpty(t, cmd) assert.Equal(t, 9, len(cmd)) @@ -559,21 +566,54 @@ func TestPrepareCmdForUploadResults(t *testing.T) { }) } -func TestAppendCodeqlQuery(t *testing.T) { +func TestAppendCodeqlQuerySuite(t *testing.T) { t.Parallel() t.Run("Empty query", func(t *testing.T) { + utils := newCodeqlExecuteScanTestsUtils() cmd := []string{"database", "analyze"} - query := "" - cmd = appendCodeqlQuery(cmd, query) + querySuite := "" + cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, "") assert.Equal(t, 2, len(cmd)) }) t.Run("Not empty query", func(t *testing.T) { + utils := newCodeqlExecuteScanTestsUtils() + cmd := []string{"database", "analyze"} + querySuite := "java-extended.ql" + cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, "") + assert.Equal(t, 3, len(cmd)) + }) + + t.Run("Add prefix to querySuite", func(t *testing.T) { + utils := codeqlExecuteScanMockUtils{ + ExecMockRunner: &mock.ExecMockRunner{ + Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error { + stdout.Write([]byte("test-java-security-extended.qls")) + return nil + }, + }, + } + cmd := []string{"database", "analyze"} + querySuite := "java-security-extended.qls" + cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, `s/^(java|python)-(security-extended\.qls|security-and-quality\.qls)/test-\1-\2/`) + assert.Equal(t, 3, len(cmd)) + assert.Equal(t, "test-java-security-extended.qls", cmd[2]) + }) + + t.Run("Don't add prefix to querySuite", func(t *testing.T) { + utils := codeqlExecuteScanMockUtils{ + ExecMockRunner: &mock.ExecMockRunner{ + Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error { + return fmt.Errorf("error") + }, + }, + } cmd := []string{"database", "analyze"} - query := "java-extended.ql" - cmd = appendCodeqlQuery(cmd, query) + querySuite := "php-security-extended.qls" + cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, `s/^(java|python)-(security-extended\.qls|security-and-quality\.qls)/test-\1-\2/`) assert.Equal(t, 3, len(cmd)) + assert.Equal(t, "php-security-extended.qls", cmd[2]) }) } diff --git a/resources/metadata/codeqlExecuteScan.yaml b/resources/metadata/codeqlExecuteScan.yaml index eb9c5d9374..a35952d976 100644 --- a/resources/metadata/codeqlExecuteScan.yaml +++ b/resources/metadata/codeqlExecuteScan.yaml @@ -240,6 +240,13 @@ spec: - STEPS - STAGES - PARAMETERS + - name: transformQuerySuite + type: string + description: "A transform string that will be applied to the querySuite using the sed command." + scope: + - STEPS + - STAGES + - PARAMETERS containers: - image: "" outputs: