From 2e062bc00045aa4ae8b506e3eba33a9ddfc94b37 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Mon, 27 Jun 2022 14:37:29 -0400 Subject: [PATCH] Get repo info from REST API if event file is unavailable (#576) Co-authored-by: Azeem Shaikh --- github/github.go | 130 ++++++++++++++++++++++----------------------- options/env.go | 1 + options/options.go | 58 +++++++++++++------- 3 files changed, 104 insertions(+), 85 deletions(-) diff --git a/github/github.go b/github/github.go index 8551d19f..134f8900 100644 --- a/github/github.go +++ b/github/github.go @@ -15,20 +15,18 @@ package github import ( + "bytes" "context" "encoding/json" "fmt" "io" + "io/ioutil" + "log" "net/http" "net/url" - "os" "github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper" - "github.com/ossf/scorecard/v4/log" -) - -const ( - baseRepoURL = "https://api.github.com/repos/" + sclog "github.com/ossf/scorecard/v4/log" ) // RepoInfo is a struct for repository information. @@ -43,9 +41,9 @@ type repo struct { GITHUB_REPOSITORY_IS_FORK is true if the repository is a fork. */ - DefaultBranch string `json:"default_branch"` - Fork bool `json:"fork"` - Private bool `json:"private"` + DefaultBranch *string `json:"default_branch"` + Fork *bool `json:"fork"` + Private *bool `json:"private"` } // Client holds a context and roundtripper for querying repo info from GitHub. @@ -54,20 +52,6 @@ type Client struct { rt http.RoundTripper } -// NewClient returns a new Client for querying repo info from GitHub. -func NewClient(ctx context.Context) *Client { - c := &Client{} - - defaultCtx := context.Background() - if ctx == nil { - ctx = defaultCtx - } - - c.SetContext(ctx) - c.SetDefaultTransport() - return c -} - // SetContext sets a context for a GitHub client. func (c *Client) SetContext(ctx context.Context) { c.ctx = ctx @@ -80,82 +64,96 @@ func (c *Client) SetTransport(rt http.RoundTripper) { // SetDefaultTransport sets the scorecard roundtripper for a GitHub client. func (c *Client) SetDefaultTransport() { - logger := log.NewLogger(log.DefaultLevel) + logger := sclog.NewLogger(sclog.DefaultLevel) rt := roundtripper.NewTransport(c.ctx, logger) c.rt = rt } -// WriteRepoInfo queries GitHub for repo info and writes it to a file. -func WriteRepoInfo(ctx context.Context, repoName, path string) error { - c := NewClient(ctx) - repoInfo, err := c.RepoInfo(repoName) - if err != nil { - return fmt.Errorf("getting repo info: %w", err) - } - - repoFile, err := os.Create(path) - if err != nil { - return fmt.Errorf("creating repo info file: %w", err) - } - defer repoFile.Close() - - resp := repoInfo.respBytes - _, writeErr := repoFile.Write(resp) - if writeErr != nil { - return fmt.Errorf("writing repo info: %w", writeErr) - } - - return nil -} - -// RepoInfo is a function to get the repository information. +// ParseFromURL is a function to get the repository information. // It is decided to not use the golang GitHub library because of the // dependency on the github.com/google/go-github/github library // which will in turn require other dependencies. -func (c *Client) RepoInfo(repoName string) (RepoInfo, error) { - var r RepoInfo - +func (c *Client) ParseFromURL(baseRepoURL, repoName string) (RepoInfo, error) { + var ret RepoInfo baseURL, err := url.Parse(baseRepoURL) if err != nil { - return r, fmt.Errorf("parsing base repo URL: %w", err) + return ret, fmt.Errorf("parsing base repo URL: %w", err) } - repoURL, err := baseURL.Parse(repoName) + repoURL, err := baseURL.Parse(fmt.Sprintf("repos/%s", repoName)) if err != nil { - return r, fmt.Errorf("parsing repo endpoint: %w", err) + return ret, fmt.Errorf("parsing repo endpoint: %w", err) } - method := "GET" + log.Printf("getting repo info from URL: %s", repoURL.String()) req, err := http.NewRequestWithContext( c.ctx, - method, + http.MethodGet, repoURL.String(), - nil, - ) + nil /*body*/) if err != nil { - return r, fmt.Errorf("error creating request: %w", err) + return ret, fmt.Errorf("error creating request: %w", err) } resp, err := http.DefaultClient.Do(req) if err != nil { - return r, fmt.Errorf("error creating request: %w", err) + return ret, fmt.Errorf("error creating request: %w", err) } defer resp.Body.Close() if err != nil { - return r, fmt.Errorf("error reading response body: %w", err) + return ret, fmt.Errorf("error reading response body: %w", err) } respBytes, err := io.ReadAll(resp.Body) if err != nil { - return r, fmt.Errorf("error reading response body: %w", err) + return ret, fmt.Errorf("error reading response body: %w", err) } - r.respBytes = respBytes + prettyPrintJSON(respBytes) + ret.respBytes = respBytes + if err := json.Unmarshal(respBytes, &ret.Repo); err != nil { + return ret, fmt.Errorf("error decoding response body: %w", err) + } + return ret, nil +} + +// ParseFromFile is a function to get the repository information +// from GitHub event file. +func (c *Client) ParseFromFile(filepath string) (RepoInfo, error) { + var ret RepoInfo - err = json.Unmarshal(respBytes, &r) + log.Printf("getting repo info from file: %s", filepath) + repoInfo, err := ioutil.ReadFile(filepath) if err != nil { - return r, fmt.Errorf("error decoding response body: %w", err) + return ret, fmt.Errorf("reading GitHub event path: %w", err) } - return r, nil + prettyPrintJSON(repoInfo) + if err := json.Unmarshal(repoInfo, &ret); err != nil { + return ret, fmt.Errorf("unmarshalling repo info: %w", err) + } + + return ret, nil +} + +// NewClient returns a new Client for querying repo info from GitHub. +func NewClient(ctx context.Context) *Client { + c := &Client{ + ctx: ctx, + } + + if c.ctx == nil { + c.SetContext(context.Background()) + } + c.SetDefaultTransport() + return c +} + +func prettyPrintJSON(jsonBytes []byte) { + var buf bytes.Buffer + if err := json.Indent(&buf, jsonBytes, "", ""); err != nil { + log.Printf("%v", err) + return + } + log.Println(buf.String()) } diff --git a/options/env.go b/options/env.go index 33354ccf..84e202e8 100644 --- a/options/env.go +++ b/options/env.go @@ -21,6 +21,7 @@ import ( // Environment variables. // TODO(env): Remove once environment variables are not used for config. +// //nolint:revive,nolintlint const ( EnvEnableSarif = "ENABLE_SARIF" diff --git a/options/options.go b/options/options.go index ab19218a..5b9945db 100644 --- a/options/options.go +++ b/options/options.go @@ -15,15 +15,14 @@ package options import ( - "encoding/json" "errors" "fmt" - "io/ioutil" "os" "strconv" "strings" "github.com/caarlos0/env/v6" + "golang.org/x/net/context" "github.com/ossf/scorecard-action/github" "github.com/ossf/scorecard/v4/checks" @@ -44,6 +43,7 @@ var ( // Errors. errGithubEventPathEmpty = errors.New("GitHub event path is empty") errResultsPathEmpty = errors.New("results path is empty") + errGitHubRepoInfoUnavailable = errors.New("GitHub repo info inaccessible") errOnlyDefaultBranchSupported = errors.New("only default branch is supported") ) @@ -67,6 +67,7 @@ type Options struct { GithubRef string `env:"GITHUB_REF"` GithubRepository string `env:"GITHUB_REPOSITORY"` GithubWorkspace string `env:"GITHUB_WORKSPACE"` + GithubAPIURL string `env:"GITHUB_API_URL"` DefaultBranch string `env:"SCORECARD_DEFAULT_BRANCH"` // TODO(options): This may be better as a bool @@ -87,6 +88,13 @@ func New() (*Options, error) { if err := env.Parse(opts); err != nil { return opts, fmt.Errorf("parsing entrypoint env vars: %w", err) } + // GITHUB_AUTH_TOKEN + // Needs to be set *before* setRepoInfo() is invoked. + // setRepoInfo() uses the GITHUB_AUTH_TOKEN env for querying the REST API. + if _, tokenSet := os.LookupEnv(EnvGithubAuthToken); !tokenSet { + inputToken := os.Getenv(EnvInputRepoToken) + os.Setenv(EnvGithubAuthToken, inputToken) + } if err := opts.setRepoInfo(); err != nil { return opts, fmt.Errorf("parsing repo info: %w", err) } @@ -143,12 +151,6 @@ func (o *Options) Print() { func (o *Options) setScorecardOpts() { o.ScorecardOpts = scopts.New() - // GITHUB_AUTH_TOKEN - _, tokenSet := os.LookupEnv(EnvGithubAuthToken) - if !tokenSet { - inputToken := os.Getenv(EnvInputRepoToken) - os.Setenv(EnvGithubAuthToken, inputToken) - } // --repo= | --local // This section restores functionality that was removed in @@ -194,6 +196,8 @@ func (o *Options) setScorecardOpts() { // setPublishResults sets whether results should be published based on a // repository's visibility. func (o *Options) setPublishResults() { + inputVal := o.PublishResults + o.PublishResults = false privateRepo, err := strconv.ParseBool(o.PrivateRepoStr) if err != nil { // TODO(options): Consider making this an error. @@ -202,9 +206,10 @@ func (o *Options) setPublishResults() { o.PrivateRepoStr, err, ) + return } - o.PublishResults = o.PublishResults && !privateRepo + o.PublishResults = inputVal && !privateRepo } // setRepoInfo gets the path to the GitHub event and sets the @@ -217,21 +222,36 @@ func (o *Options) setRepoInfo() error { return errGithubEventPathEmpty } - repoInfo, err := ioutil.ReadFile(eventPath) - if err != nil { - return fmt.Errorf("reading GitHub event path: %w", err) + ghClient := github.NewClient(context.Background()) + if repoInfo, err := ghClient.ParseFromFile(eventPath); err == nil && + o.parseFromRepoInfo(repoInfo) { + return nil } - var r github.RepoInfo - if err := json.Unmarshal(repoInfo, &r); err != nil { - return fmt.Errorf("unmarshalling repo info: %w", err) + if repoInfo, err := ghClient.ParseFromURL(o.GithubAPIURL, o.GithubRepository); err == nil && + o.parseFromRepoInfo(repoInfo) { + return nil } - o.PrivateRepoStr = strconv.FormatBool(r.Repo.Private) - o.IsForkStr = strconv.FormatBool(r.Repo.Fork) - o.DefaultBranch = r.Repo.DefaultBranch + return errGitHubRepoInfoUnavailable +} - return nil +func (o *Options) parseFromRepoInfo(repoInfo github.RepoInfo) bool { + if repoInfo.Repo.DefaultBranch == nil && + repoInfo.Repo.Fork == nil && + repoInfo.Repo.Private == nil { + return false + } + if repoInfo.Repo.Private != nil { + o.PrivateRepoStr = strconv.FormatBool(*repoInfo.Repo.Private) + } + if repoInfo.Repo.Fork != nil { + o.IsForkStr = strconv.FormatBool(*repoInfo.Repo.Fork) + } + if repoInfo.Repo.DefaultBranch != nil { + o.DefaultBranch = *repoInfo.Repo.DefaultBranch + } + return true } func (o *Options) isPullRequestEvent() bool {