From 40dc02d51cc4376d3b425cb5a1ace7ed31a37879 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Sat, 4 Mar 2023 17:13:21 +0000 Subject: [PATCH] Initial implementation of go-git client Signed-off-by: Azeem Shaikh --- clients/git/client.go | 203 ++++++++++++++++++++++++++++++ clients/git/e2e_test.go | 155 +++++++++++++++++++++++ clients/git/gitrepo_suite_test.go | 32 +++++ go.mod | 1 + go.sum | 7 ++ 5 files changed, 398 insertions(+) create mode 100644 clients/git/client.go create mode 100644 clients/git/e2e_test.go create mode 100644 clients/git/gitrepo_suite_test.go diff --git a/clients/git/client.go b/clients/git/client.go new file mode 100644 index 000000000000..2831cd8004d1 --- /dev/null +++ b/clients/git/client.go @@ -0,0 +1,203 @@ +// Copyright 2021 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package git defines helper functions for clients.RepoClient interface. +package git + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + cp "github.com/otiai10/copy" + + "github.com/ossf/scorecard/v4/clients" +) + +const repoDir = "repo*" + +type Client struct { + gitRepo *git.Repository + worktree *git.Worktree + listCommits *sync.Once + tempDir string + errListCommits error + commits []clients.Commit + commitDepth int +} + +func (c *Client) InitRepo(uri, commitSHA string, commitDepth int) error { + // cleanup previous state, if any. + c.Close() + c.listCommits = new(sync.Once) + c.commits = nil + + // init + c.commitDepth = commitDepth + tempDir, err := os.MkdirTemp("", repoDir) + if err != nil { + return fmt.Errorf("os.MkdirTemp: %w", err) + } + + // git clone + const filePrefix = "file://" + if strings.HasPrefix(uri, filePrefix) { + if err := cp.Copy(strings.TrimPrefix(uri, filePrefix), tempDir); err != nil { + return fmt.Errorf("cp.Copy: %w", err) + } + c.gitRepo, err = git.PlainOpen(tempDir) + if err != nil { + return fmt.Errorf("git.PlainOpen: %w", err) + } + } else { + c.gitRepo, err = git.PlainClone(tempDir, false /*isBare*/, &git.CloneOptions{ + URL: uri, + Progress: os.Stdout, + }) + if err != nil { + return fmt.Errorf("git.PlainClone: %w", err) + } + } + c.tempDir = tempDir + c.worktree, err = c.gitRepo.Worktree() + if err != nil { + return fmt.Errorf("git.Worktree: %w", err) + } + + // git checkout + if commitSHA != clients.HeadSHA { + if err := c.worktree.Checkout(&git.CheckoutOptions{ + Hash: plumbing.NewHash(commitSHA), + Force: true, // throw away any unsaved changes. + }); err != nil { + return fmt.Errorf("git.Worktree: %w", err) + } + } + + return nil +} + +func (c *Client) ListCommits() ([]clients.Commit, error) { + c.listCommits.Do(func() { + commitIter, err := c.gitRepo.Log(&git.LogOptions{ + Order: git.LogOrderCommitterTime, + }) + if err != nil { + c.errListCommits = fmt.Errorf("git.CommitObjects: %w", err) + return + } + for i := 0; i < c.commitDepth; i++ { + commit, err := commitIter.Next() + if err != nil && !errors.Is(err, io.EOF) { + c.errListCommits = fmt.Errorf("commitIter.Next: %w", err) + return + } + // No more commits. + if errors.Is(err, io.EOF) { + break + } + + if commit == nil { + // Not sure in what case a nil commit is returned. Fail explicitly. + c.errListCommits = fmt.Errorf("nil commit found") + return + } + + c.commits = append(c.commits, clients.Commit{ + SHA: commit.Hash.String(), + Message: commit.Message, + CommittedDate: commit.Committer.When, + Committer: clients.User{ + Login: commit.Committer.Email, + }, + }) + } + }) + return c.commits, c.errListCommits +} + +func (c *Client) Search(request clients.SearchRequest) (clients.SearchResponse, error) { + // Pattern + if request.Query == "" { + return clients.SearchResponse{}, fmt.Errorf("Query is empty") + } + queryRegexp, err := regexp.Compile(request.Query) + if err != nil { + return clients.SearchResponse{}, fmt.Errorf("regexp.Compile: %w", err) + } + grepOpts := &git.GrepOptions{ + Patterns: []*regexp.Regexp{queryRegexp}, + } + + // path/filename + var pathExpr string + switch { + case request.Path != "" && request.Filename != "": + pathExpr = filepath.Join(fmt.Sprintf("^%s", request.Path), + fmt.Sprintf(".*%s", request.Filename)) + case request.Path != "": + pathExpr = fmt.Sprintf("^%s", request.Path) + case request.Filename != "": + pathExpr = filepath.Join(".*", request.Filename) + } + if pathExpr != "" { + pathRegexp, err := regexp.Compile(pathExpr) + if err != nil { + return clients.SearchResponse{}, fmt.Errorf("regexp.Compile: %w", err) + } + grepOpts.PathSpecs = append(grepOpts.PathSpecs, pathRegexp) + } + + // Grep + grepResults, err := c.worktree.Grep(grepOpts) + if err != nil { + return clients.SearchResponse{}, fmt.Errorf("git.Grep: %w", err) + } + + ret := clients.SearchResponse{} + for _, grepResult := range grepResults { + ret.Results = append(ret.Results, clients.SearchResult{ + Path: grepResult.FileName, + }) + } + ret.Hits = len(grepResults) + return ret, nil +} + +// TODO(#1709): Implement below fns using go-git. +func (c *Client) SearchCommits(request clients.SearchCommitsOptions) ([]clients.Commit, error) { + return nil, nil +} + +func (c *Client) ListFiles(predicate func(string) (bool, error)) ([]string, error) { + return nil, nil +} + +func (c *Client) GetFileContent(filename string) ([]byte, error) { + return nil, nil +} + +func (c *Client) Close() error { + if err := os.RemoveAll(c.tempDir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("os.RemoveAll: %w", err) + } + return nil +} diff --git a/clients/git/e2e_test.go b/clients/git/e2e_test.go new file mode 100644 index 000000000000..758fc04883c2 --- /dev/null +++ b/clients/git/e2e_test.go @@ -0,0 +1,155 @@ +// Copyright 2021 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/ossf/scorecard/v4/clients" +) + +var _ = DescribeTable("Test ListCommits commit-depth for HEAD", + func(uri string) { + const commitSHA = clients.HeadSHA + const commitDepth = 1 + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + commits, err := client.ListCommits() + Expect(err).To(BeNil()) + Expect(len(commits)).Should(BeEquivalentTo(commitDepth)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard"), + Entry("Local", "file://../../"), + Entry("GitLab", "https://gitlab.haskell.org/haskell/filepath"), +) + +var _ = DescribeTable("Test ListCommits commit-depth and latest commit at [0]", + func(uri, commitSHA string) { + const commitDepth = 10 + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + commits, err := client.ListCommits() + Expect(err).To(BeNil()) + Expect(len(commits)).Should(BeEquivalentTo(commitDepth)) + Expect(commits[0].SHA).Should(BeEquivalentTo(commitSHA)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard", "c06ac740cc49fea404c54c036000731d5ea6ebe3"), + Entry("Local", "file://../../", "c06ac740cc49fea404c54c036000731d5ea6ebe3"), + Entry("GitLab", "https://gitlab.haskell.org/haskell/filepath", "98f8bba9eac8c7183143d290d319be7df76c258b"), +) + +var _ = DescribeTable("Test ListCommits without enough commits", + func(uri string) { + const commitSHA = "dc1835b7ffe526969d65436b621e171e3386771e" + const commitDepth = 10 + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + commits, err := client.ListCommits() + Expect(err).To(BeNil()) + Expect(len(commits)).Should(BeEquivalentTo(3)) + Expect(commits[0].SHA).Should(BeEquivalentTo(commitSHA)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard"), + Entry("Local", "file://../../"), + // TODO(#1709): Add equivalent test for GitLab. +) + +var _ = DescribeTable("Test Search across a repo", + func(uri string) { + const ( + commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3" + commitDepth = 10 + ) + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + resp, err := client.Search(clients.SearchRequest{ + Query: "github/codeql-action/analyze", + }) + Expect(err).To(BeNil()) + Expect(resp.Hits).Should(BeNumerically(">=", 1)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard"), + Entry("Local", "file://../../"), + // TODO(#1709): Add equivalent test for GitLab. +) + +var _ = DescribeTable("Test Search within a path", + func(uri string) { + const ( + commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3" + commitDepth = 10 + ) + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + resp, err := client.Search(clients.SearchRequest{ + Query: "github/codeql-action/analyze", + Path: ".github/workflows", + }) + Expect(err).To(BeNil()) + Expect(resp.Hits).Should(BeEquivalentTo(1)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard"), + Entry("Local", "file://../../"), + // TODO(#1709): Add equivalent test for GitLab. +) + +var _ = DescribeTable("Test Search within a filename", + func(uri string) { + const ( + commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3" + commitDepth = 10 + ) + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + resp, err := client.Search(clients.SearchRequest{ + Query: "github/codeql-action/analyze", + Filename: "codeql-analysis.yml", + }) + Expect(err).To(BeNil()) + Expect(resp.Hits).Should(BeEquivalentTo(1)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard"), + Entry("Local", "file://../../"), + // TODO(#1709): Add equivalent test for GitLab. +) + +var _ = DescribeTable("Test Search within path and filename", + func(uri string) { + const ( + commitSHA = "c06ac740cc49fea404c54c036000731d5ea6ebe3" + commitDepth = 10 + ) + client := &Client{} + Expect(client.InitRepo(uri, commitSHA, commitDepth)).To(BeNil()) + resp, err := client.Search(clients.SearchRequest{ + Query: "github/codeql-action/analyze", + Path: ".github/workflows", + Filename: "codeql-analysis.yml", + }) + Expect(err).To(BeNil()) + Expect(resp.Hits).Should(BeEquivalentTo(1)) + Expect(client.Close()).To(BeNil()) + }, + Entry("GitHub", "https://github.com/ossf/scorecard"), + Entry("Local", "file://../../"), + // TODO(#1709): Add equivalent test for GitLab. +) diff --git a/clients/git/gitrepo_suite_test.go b/clients/git/gitrepo_suite_test.go new file mode 100644 index 000000000000..811d75ebfa86 --- /dev/null +++ b/clients/git/gitrepo_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2021 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestGitRepo(t *testing.T) { + if val, exists := os.LookupEnv("SKIP_GINKGO"); exists && val == "1" { + t.Skip() + } + t.Parallel() + RegisterFailHandler(Fail) + RunSpecs(t, "GitRepo Suite") +} diff --git a/go.mod b/go.mod index 7a348da00040..79139a047e7c 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/google/osv-scanner v0.0.0-20221212045131-8aef1778b823 github.com/mcuadros/go-jsonschema-generator v0.0.0-20200330054847-ba7a369d4303 github.com/onsi/ginkgo/v2 v2.7.0 + github.com/otiai10/copy v1.9.0 sigs.k8s.io/release-utils v0.6.0 ) diff --git a/go.sum b/go.sum index c01c589fcf28..98b5560472bd 100644 --- a/go.sum +++ b/go.sum @@ -790,6 +790,13 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= +github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4= +github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= github.com/package-url/packageurl-go v0.1.0 h1:efWBc98O/dBZRg1pw2xiDzovnlMjCa9NPnfaiBduh8I= github.com/package-url/packageurl-go v0.1.0/go.mod h1:C/ApiuWpmbpni4DIOECf6WCjFUZV7O1Fx7VAzrZHgBw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=