Initial implementation of go-git client

Signed-off-by: Azeem Shaikh <[email protected]>
ossf · Mar 14, 2023 · 274448f · 274448f
1 parent b5254fe
commit 274448f
Show file tree

Hide file tree

Showing 47 changed files with 1,489 additions and 1,437 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -62,7 +62,7 @@ jobs:
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
 
-      uses: github/codeql-action/init@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v1
+      uses: github/codeql-action/init@16964e90ba004cdf0cd845b866b5df21038b7723 # v1
       with:
         languages: ${{ matrix.language }}
         queries: +security-extended
@@ -74,7 +74,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v1
+      uses: github/codeql-action/autobuild@16964e90ba004cdf0cd845b866b5df21038b7723 # v1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -88,4 +88,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v1
+      uses: github/codeql-action/analyze@16964e90ba004cdf0cd845b866b5df21038b7723 # v1
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -41,7 +41,7 @@ jobs:
         fetch-depth: 2
     - id: files
       name: Get changed files
-      uses: tj-actions/changed-files@5ce975c6021a0b11062c547acb6c26c96a34a8c5 #v35.6.2
+      uses: tj-actions/changed-files@bd376fbcfae914347656e4c70801e2a3fafed05b #v35.7.0
       with:
         files_ignore: '**.md'
     - id: docs_only_check
@@ -70,7 +70,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache
@@ -118,7 +118,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache
@@ -166,7 +166,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache
@@ -214,7 +214,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache
@@ -262,7 +262,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache
@@ -310,7 +310,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache
@@ -358,7 +358,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
           # In order:
           # * Module download cache

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -71,3 +71,17 @@ jobs:
         with:
          files: ./e2e-coverage.out
          verbose: true
+
+      - name: Run GitLab E2E  #using retry because the GitHub token is being throttled.
+        uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd
+        with:
+          max_attempts: 3
+          retry_on: error
+          timeout_minutes: 30
+          command: make e2e-gitlab
+
+      - name: codecov
+        uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # 2.1.0
+        with:
+         files: ./e2e-coverage.out
+         verbose: true
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -43,7 +43,7 @@ jobs:
 
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 #v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -88,7 +88,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -136,7 +136,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -219,7 +219,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -267,7 +267,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -315,7 +315,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -363,7 +363,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -411,7 +411,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -459,7 +459,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -507,7 +507,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -555,7 +555,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -603,7 +603,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -651,7 +651,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -699,7 +699,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -771,7 +771,7 @@ jobs:
 
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod
@@ -818,7 +818,7 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache builds
        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
-       uses: actions/cache@69d9d449aced6a2ede0bc19182fadc3a0a42d2b0 # v3.2.6
+       uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
        with:
          path: |
            ~/go/pkg/mod

diff --git a/.github/workflows/scorecard-analysis.yml b/.github/workflows/scorecard-analysis.yml
@@ -47,6 +47,6 @@ jobs:
           retention-days: 5
 
       - name: "Upload SARIF results"
-        uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v1
+        uses: github/codeql-action/upload-sarif@16964e90ba004cdf0cd845b866b5df21038b7723 # v1
         with:
           sarif_file: results.sarif
diff --git a/Makefile b/Makefile
@@ -334,6 +334,12 @@ e2e-gh-token: build-scorecard check-env | $(GINKGO)
 	# Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this.
 	TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
 
+e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN
+	TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab Token' ./...
+
+e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something
+	TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --focus '.*GitLab' ./...
+
 e2e-attestor: ## Runs e2e tests for scorecard-attestor
 	cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../..
 

diff --git a/checker/client.go b/checker/client.go
@@ -17,9 +17,11 @@ package checker
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"github.com/ossf/scorecard/v4/clients"
 	ghrepo "github.com/ossf/scorecard/v4/clients/githubrepo"
+	glrepo "github.com/ossf/scorecard/v4/clients/gitlabrepo"
 	"github.com/ossf/scorecard/v4/clients/localdir"
 	"github.com/ossf/scorecard/v4/clients/ossfuzz"
 	"github.com/ossf/scorecard/v4/log"
@@ -35,7 +37,9 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
 	clients.VulnerabilitiesClient, // vulnClient
 	error,
 ) {
-	var githubRepo clients.Repo
+	var repo clients.Repo
+	var makeRepoError error
+
 	if localURI != "" {
 		localRepo, errLocal := localdir.MakeLocalDirRepo(localURI)
 		var retErr error
@@ -50,18 +54,46 @@ func GetClients(ctx context.Context, repoURI, localURI string, logger *log.Logge
 			retErr
 	}
 
-	githubRepo, errGitHub := ghrepo.MakeGithubRepo(repoURI)
-	if errGitHub != nil {
-		return githubRepo,
-			nil,
-			nil,
-			nil,
-			nil,
-			fmt.Errorf("getting local directory client: %w", errGitHub)
+	_, experimental := os.LookupEnv("SCORECARD_EXPERIMENTAL")
+	var repoClient clients.RepoClient
+
+	//nolint:nestif
+	if experimental && glrepo.DetectGitLab(repoURI) {
+		repo, makeRepoError = glrepo.MakeGitlabRepo(repoURI)
+		if makeRepoError != nil {
+			return repo,
+				nil,
+				nil,
+				nil,
+				nil,
+				fmt.Errorf("getting local directory client: %w", makeRepoError)
+		}
+
+		var err error
+		repoClient, err = glrepo.CreateGitlabClientWithToken(ctx, os.Getenv("GITLAB_AUTH_TOKEN"), repo)
+		if err != nil {
+			return repo,
+				nil,
+				nil,
+				nil,
+				nil,
+				fmt.Errorf("error creating gitlab client: %w", err)
+		}
+	} else {
+		repo, makeRepoError = ghrepo.MakeGithubRepo(repoURI)
+		if makeRepoError != nil {
+			return repo,
+				nil,
+				nil,
+				nil,
+				nil,
+				fmt.Errorf("getting local directory client: %w", makeRepoError)
+		}
+		repoClient = ghrepo.CreateGithubRepoClient(ctx, logger)
 	}
 
-	return githubRepo, /*repo*/
-		ghrepo.CreateGithubRepoClient(ctx, logger), /*repoClient*/
+	return repo, /*repo*/
+		repoClient, /*repoClient*/
 		ossfuzz.CreateOSSFuzzClient(ossfuzz.StatusURL), /*ossFuzzClient*/
 		clients.DefaultCIIBestPracticesClient(), /*ciiClient*/
 		clients.DefaultVulnerabilitiesClient(), /*vulnClient*/