added sed cmd to transform querySuite (#4945)

* added sed cmd to transform querySuite * changed sh to bash * added checking os * fixed tests by mocking utils * fixed tests * renamed param --------- Co-authored-by: sumeet patil <[email protected]>
SAP · Jun 6, 2024 · 67ed27f · 67ed27f
1 parent e2a5b09
commit 67ed27f
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 19 deletions.
diff --git a/cmd/codeqlExecuteScan.go b/cmd/codeqlExecuteScan.go
@@ -59,9 +59,23 @@ func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry
 	influx.step_data.fields.codeql = true
 }
 
-func appendCodeqlQuery(cmd []string, codeqlQuery string) []string {
-	if len(codeqlQuery) > 0 {
-		cmd = append(cmd, codeqlQuery)
+func appendCodeqlQuerySuite(utils codeqlExecuteScanUtils, cmd []string, querySuite, transformString string) []string {
+	if len(querySuite) > 0 {
+		if len(transformString) > 0 {
+			var bufferOut, bufferErr bytes.Buffer
+			utils.Stdout(&bufferOut)
+			defer utils.Stdout(log.Writer())
+			utils.Stderr(&bufferErr)
+			defer utils.Stderr(log.Writer())
+			if err := utils.RunExecutable("sh", []string{"-c", fmt.Sprintf("echo %s | sed -E \"%s\"", querySuite, transformString)}...); err != nil {
+				log.Entry().WithError(err).Error("failed to transform querySuite")
+				e := bufferErr.String()
+				log.Entry().Error(e)
+			} else {
+				querySuite = strings.TrimSpace(bufferOut.String())
+			}
+		}
+		cmd = append(cmd, querySuite)
 	}
 
 	return cmd
@@ -271,7 +285,7 @@ func runGithubUploadResults(config *codeqlExecuteScanOptions, repoInfo *codeql.R
 func executeAnalysis(format, reportName string, customFlags map[string]string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) ([]piperutils.Path, error) {
 	moduleTargetPath := filepath.Join(config.ModulePath, "target")
 	report := filepath.Join(moduleTargetPath, reportName)
-	cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, format, report)
+	cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, format, report)
 	if err != nil {
 		log.Entry().Errorf("failed to prepare command for codeql database analyze (format=%s)", format)
 		return nil, err
@@ -323,11 +337,11 @@ func prepareCmdForDatabaseCreate(customFlags map[string]string, config *codeqlEx
 	return cmd, nil
 }
 
-func prepareCmdForDatabaseAnalyze(customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) {
+func prepareCmdForDatabaseAnalyze(utils codeqlExecuteScanUtils, customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) {
 	cmd := []string{"database", "analyze", "--format=" + format, "--output=" + reportName, config.Database}
 	cmd = codeql.AppendThreadsAndRam(cmd, config.Threads, config.Ram, customFlags)
 	cmd = codeql.AppendCustomFlags(cmd, customFlags)
-	cmd = appendCodeqlQuery(cmd, config.QuerySuite)
+	cmd = appendCodeqlQuerySuite(utils, cmd, config.QuerySuite, config.TransformQuerySuite)
 	return cmd, nil
 }
 

diff --git a/cmd/codeqlExecuteScan_generated.go b/cmd/codeqlExecuteScan_generated.go
diff --git a/cmd/codeqlExecuteScan_test.go b/cmd/codeqlExecuteScan_test.go
@@ -4,6 +4,8 @@
 package cmd
 
 import (
+	"fmt"
+	"io"
 	"os"
 	"strings"
 	"testing"
@@ -21,7 +23,11 @@ type codeqlExecuteScanMockUtils struct {
 
 func newCodeqlExecuteScanTestsUtils() codeqlExecuteScanMockUtils {
 	utils := codeqlExecuteScanMockUtils{
-		ExecMockRunner: &mock.ExecMockRunner{},
+		ExecMockRunner: &mock.ExecMockRunner{
+			Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
+				return nil
+			},
+		},
 		FilesMock:      &mock.FilesMock{},
 		HttpClientMock: &mock.HttpClientMock{},
 	}
@@ -406,12 +412,13 @@ func TestPrepareCmdForDatabaseCreate(t *testing.T) {
 
 func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 	t.Parallel()
+	utils := codeqlExecuteScanMockUtils{}
 
 	t.Run("No additional flags, no querySuite, sarif format", func(t *testing.T) {
 		config := &codeqlExecuteScanOptions{
 			Database: "codeqlDB",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 5, len(cmd))
@@ -422,7 +429,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 		config := &codeqlExecuteScanOptions{
 			Database: "codeqlDB",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "csv", "target/codeqlReport.csv")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "csv", "target/codeqlReport.csv")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 5, len(cmd))
@@ -434,7 +441,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 			Database:   "codeqlDB",
 			QuerySuite: "security.ql",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 6, len(cmd))
@@ -448,7 +455,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 			Threads:    "1",
 			Ram:        "2000",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, map[string]string{}, config, "sarif-latest", "target/codeqlReport.sarif")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 8, len(cmd))
@@ -465,7 +472,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 		customFlags := map[string]string{
 			"--threads": "--threads=2",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 8, len(cmd))
@@ -482,7 +489,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 		customFlags := map[string]string{
 			"-j": "-j=2",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 8, len(cmd))
@@ -499,7 +506,7 @@ func TestPrepareCmdForDatabaseAnalyze(t *testing.T) {
 		customFlags := map[string]string{
 			"--no-download": "--no-download",
 		}
-		cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
+		cmd, err := prepareCmdForDatabaseAnalyze(utils, customFlags, config, "sarif-latest", "target/codeqlReport.sarif")
 		assert.NoError(t, err)
 		assert.NotEmpty(t, cmd)
 		assert.Equal(t, 9, len(cmd))
@@ -559,21 +566,54 @@ func TestPrepareCmdForUploadResults(t *testing.T) {
 	})
 }
 
-func TestAppendCodeqlQuery(t *testing.T) {
+func TestAppendCodeqlQuerySuite(t *testing.T) {
 	t.Parallel()
 
 	t.Run("Empty query", func(t *testing.T) {
+		utils := newCodeqlExecuteScanTestsUtils()
 		cmd := []string{"database", "analyze"}
-		query := ""
-		cmd = appendCodeqlQuery(cmd, query)
+		querySuite := ""
+		cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, "")
 		assert.Equal(t, 2, len(cmd))
 	})
 
 	t.Run("Not empty query", func(t *testing.T) {
+		utils := newCodeqlExecuteScanTestsUtils()
+		cmd := []string{"database", "analyze"}
+		querySuite := "java-extended.ql"
+		cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, "")
+		assert.Equal(t, 3, len(cmd))
+	})
+
+	t.Run("Add prefix to querySuite", func(t *testing.T) {
+		utils := codeqlExecuteScanMockUtils{
+			ExecMockRunner: &mock.ExecMockRunner{
+				Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
+					stdout.Write([]byte("test-java-security-extended.qls"))
+					return nil
+				},
+			},
+		}
+		cmd := []string{"database", "analyze"}
+		querySuite := "java-security-extended.qls"
+		cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, `s/^(java|python)-(security-extended\.qls|security-and-quality\.qls)/test-\1-\2/`)
+		assert.Equal(t, 3, len(cmd))
+		assert.Equal(t, "test-java-security-extended.qls", cmd[2])
+	})
+
+	t.Run("Don't add prefix to querySuite", func(t *testing.T) {
+		utils := codeqlExecuteScanMockUtils{
+			ExecMockRunner: &mock.ExecMockRunner{
+				Stub: func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error {
+					return fmt.Errorf("error")
+				},
+			},
+		}
 		cmd := []string{"database", "analyze"}
-		query := "java-extended.ql"
-		cmd = appendCodeqlQuery(cmd, query)
+		querySuite := "php-security-extended.qls"
+		cmd = appendCodeqlQuerySuite(utils, cmd, querySuite, `s/^(java|python)-(security-extended\.qls|security-and-quality\.qls)/test-\1-\2/`)
 		assert.Equal(t, 3, len(cmd))
+		assert.Equal(t, "php-security-extended.qls", cmd[2])
 	})
 }
 

diff --git a/resources/metadata/codeqlExecuteScan.yaml b/resources/metadata/codeqlExecuteScan.yaml
@@ -240,6 +240,13 @@ spec:
           - STEPS
           - STAGES
           - PARAMETERS
+      - name: transformQuerySuite
+        type: string
+        description: "A transform string that will be applied to the querySuite using the sed command."
+        scope:
+          - STEPS
+          - STAGES
+          - PARAMETERS
   containers:
     - image: ""
   outputs: