feat(codeqlExecuteScan): cloning project from non-github scm to github …

…#4630


Co-authored-by: sumeet patil <[email protected]>

cmd/codeqlExecuteScan.go

@@ -120,12 +120,13 @@ func getGitRepoInfo(repoUri string, repoInfo *RepoInfo) error {
 	return fmt.Errorf("Invalid repository %s", repoUri)
 }
 
-func initGitInfo(config *codeqlExecuteScanOptions) RepoInfo {
+func initGitInfo(config *codeqlExecuteScanOptions) (RepoInfo, error) {
 	var repoInfo RepoInfo
 	err := getGitRepoInfo(config.Repository, &repoInfo)
 	if err != nil {
 		log.Entry().Error(err)
 	}
+
 	repoInfo.ref = config.AnalyzedRef
 	repoInfo.commitId = config.CommitID
 
@@ -148,8 +149,25 @@ func initGitInfo(config *codeqlExecuteScanOptions) RepoInfo {
 			}
 		}
 	}
+	if len(config.TargetGithubRepoURL) > 0 {
+		if strings.Contains(repoInfo.serverUrl, "github") {
+			log.Entry().Errorf("TargetGithubRepoURL should not be set as the source repo is on github.")
+			return repoInfo, errors.New("TargetGithubRepoURL should not be set as the source repo is on github.")
+		}
+		err := getGitRepoInfo(config.TargetGithubRepoURL, &repoInfo)
+		if err != nil {
+			log.Entry().Error(err)
+			return repoInfo, err
+		}
+		if len(config.TargetGithubBranchName) > 0 {
+			repoInfo.ref = config.TargetGithubBranchName
+			if len(strings.Split(config.TargetGithubBranchName, "/")) < 3 {
+				repoInfo.ref = "refs/heads/" + config.TargetGithubBranchName
+			}
+		}
+	}
 
-	return repoInfo
+	return repoInfo, nil
 }
 
 func getToken(config *codeqlExecuteScanOptions) (bool, string) {
@@ -311,11 +329,37 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem
 
 	reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.csv")})
 
-	repoInfo := initGitInfo(config)
+	repoInfo, err := initGitInfo(config)
+	if err != nil {
+		return reports, err
+	}
 	repoUrl := fmt.Sprintf("%s/%s/%s", repoInfo.serverUrl, repoInfo.owner, repoInfo.repo)
 	repoReference, err := buildRepoReference(repoUrl, repoInfo.ref)
 	repoCodeqlScanUrl := fmt.Sprintf("%s/security/code-scanning?query=is:open+ref:%s", repoUrl, repoInfo.ref)
 
+	if len(config.TargetGithubRepoURL) > 0 {
+		hasToken, token := getToken(config)
+		if !hasToken {
+			return reports, errors.New("failed running upload db sources to GitHub as githubToken was not specified")
+		}
+		repoUploader, err := codeql.NewGitUploaderInstance(
+			token,
+			repoInfo.ref,
+			config.Database,
+			repoInfo.commitId,
+			config.Repository,
+			config.TargetGithubRepoURL,
+		)
+		if err != nil {
+			return reports, err
+		}
+		targetCommitId, err := repoUploader.UploadProjectToGithub()
+		if err != nil {
+			return reports, errors.Wrap(err, "failed uploading db sources from non-GitHub SCM to GitHub")
+		}
+		repoInfo.commitId = targetCommitId
+	}
+
 	if !config.UploadResults {
 		log.Entry().Warn("The sarif results will not be uploaded to the repository and compliance report will not be generated as uploadResults is set to false.")
 	} else {

cmd/codeqlExecuteScan_test.go

@@ -180,7 +180,8 @@ func TestGetGitRepoInfo(t *testing.T) {
 func TestInitGitInfo(t *testing.T) {
 	t.Run("Valid URL1", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "Testing", repoInfo.owner)
 		assert.Equal(t, "codeql", repoInfo.repo)
@@ -190,7 +191,8 @@ func TestInitGitInfo(t *testing.T) {
 
 	t.Run("Valid URL2", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "Testing", repoInfo.owner)
 		assert.Equal(t, "codeql", repoInfo.repo)
@@ -200,7 +202,8 @@ func TestInitGitInfo(t *testing.T) {
 
 	t.Run("Valid url with dots URL1", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/com.sap.codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "Testing", repoInfo.owner)
 		assert.Equal(t, "com.sap.codeql", repoInfo.repo)
@@ -210,7 +213,8 @@ func TestInitGitInfo(t *testing.T) {
 
 	t.Run("Valid url with dots URL2", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://github.hello.test/Testing/com.sap.codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "Testing", repoInfo.owner)
 		assert.Equal(t, "com.sap.codeql", repoInfo.repo)
@@ -220,7 +224,8 @@ func TestInitGitInfo(t *testing.T) {
 
 	t.Run("Valid url with username and token URL1", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://username:[email protected]/Testing/codeql.git", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "Testing", repoInfo.owner)
 		assert.Equal(t, "codeql", repoInfo.repo)
@@ -230,7 +235,8 @@ func TestInitGitInfo(t *testing.T) {
 
 	t.Run("Valid url with username and token URL2", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://username:[email protected]/Testing/codeql", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "Testing", repoInfo.owner)
 		assert.Equal(t, "codeql", repoInfo.repo)
@@ -240,8 +246,9 @@ func TestInitGitInfo(t *testing.T) {
 
 	t.Run("Invalid URL with no org/reponame", func(t *testing.T) {
 		config := codeqlExecuteScanOptions{Repository: "https://github.hello.test", AnalyzedRef: "refs/head/branch", CommitID: "abcd1234"}
-		repoInfo := initGitInfo(&config)
-		_, err := orchestrator.NewOrchestratorSpecificConfigProvider()
+		repoInfo, err := initGitInfo(&config)
+		assert.NoError(t, err)
+		_, err = orchestrator.NewOrchestratorSpecificConfigProvider()
 		assert.Equal(t, "abcd1234", repoInfo.commitId)
 		assert.Equal(t, "refs/head/branch", repoInfo.ref)
 		if err != nil {

go.mod

@@ -23,8 +23,8 @@ require (
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/getsentry/sentry-go v0.11.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
-	github.com/go-git/go-billy/v5 v5.3.1
-	github.com/go-git/go-git/v5 v5.4.2
+	github.com/go-git/go-billy/v5 v5.4.1
+	github.com/go-git/go-git/v5 v5.8.1
 	github.com/go-openapi/runtime v0.24.1
 	github.com/go-openapi/strfmt v0.21.3
 	github.com/go-playground/locales v0.14.0
@@ -69,6 +69,7 @@ require (
 
 require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.23 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
@@ -101,9 +102,11 @@ require (
 	github.com/okta/okta-sdk-golang/v2 v2.12.1 // indirect
 	github.com/oracle/oci-go-sdk/v60 v60.0.0 // indirect
 	github.com/pires/go-proxyproto v0.6.1 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/pquerna/otp v1.2.1-0.20191009055518-468c2dd2b58d // indirect
 	github.com/shirou/gopsutil/v3 v3.22.6 // indirect
+	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/sony/gobreaker v0.4.2-0.20210216022020-dd874f9dd33b // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
@@ -143,8 +146,8 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec // indirect
-	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/aliyun/alibaba-cloud-sdk-go v1.62.301 // indirect
 	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
@@ -189,12 +192,12 @@ require (
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
-	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/frankban/quicktest v1.14.4 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
@@ -265,7 +268,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
@@ -325,7 +328,7 @@ require (
 	github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/vmware/govmomi v0.18.0 // indirect
-	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
@@ -355,7 +358,7 @@ require (
 	k8s.io/client-go v0.27.2 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
-	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
+	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 	oras.land/oras-go v1.2.3 // indirect
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect