diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 782f296f8ae0..4571224a00a7 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -15,12 +15,12 @@ import ( "time" "golang.org/x/exp/rand" + "golang.org/x/oauth2" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/go-logr/logr" "github.com/gobwas/glob" "github.com/google/go-github/v62/github" - "golang.org/x/oauth2" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -480,8 +480,17 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas s.apiClient = ghClient for _, org := range s.orgsCache.Keys() { - if err := s.getReposByOrg(ctx, org); err != nil { - s.log.Error(err, "error fetching repos for org or user") + orgCtx := context.WithValue(ctx, "account", org) + userType, err := s.getReposByOrgOrUser(ctx, org) + if err != nil { + orgCtx.Logger().Error(err, "error fetching repos for org or user") + continue + } + + if userType == organization && s.conn.ScanUsers { + if err := s.addMembersByOrg(ctx, org); err != nil { + orgCtx.Logger().Error(err, "Unable to add members by org") + } } } @@ -499,17 +508,15 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin } for _, org := range s.orgsCache.Keys() { - if err := s.getReposByOrg(ctx, org); err != nil { - s.log.Error(err, "error fetching repos for org") - } - - // We probably don't need to do this, since getting repos by org makes more sense? - if err := s.getReposByUser(ctx, org); err != nil { - s.log.Error(err, "error fetching repos for user") + orgCtx := context.WithValue(ctx, "account", org) + userType, err := s.getReposByOrgOrUser(ctx, org) + if err != nil { + orgCtx.Logger().Error(err, "error fetching repos for org or user") + continue } - if s.conn.ScanUsers { - s.log.Info("Enumerating unauthenticated does not support scanning organization members") + if userType == organization && s.conn.ScanUsers { + orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)") } } } @@ -562,16 +569,16 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri if s.orgsCache.Count() > 0 { specificScope = true for _, org := range s.orgsCache.Keys() { - logger := s.log.WithValues("org", org) - if err := s.getReposByOrg(ctx, org); err != nil { - logger.Error(err, "error fetching repos for org") + orgCtx := context.WithValue(ctx, "account", org) + userType, err := s.getReposByOrgOrUser(ctx, org) + if err != nil { + orgCtx.Logger().Error(err, "error fetching repos for org or user") + continue } - if s.conn.ScanUsers { - err := s.addMembersByOrg(ctx, org) - if err != nil { - logger.Error(err, "Unable to add members by org") - continue + if userType == organization && s.conn.ScanUsers { + if err := s.addMembersByOrg(ctx, org); err != nil { + orgCtx.Logger().Error(err, "Unable to add members by org") } } } @@ -593,27 +600,28 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri } for _, org := range s.orgsCache.Keys() { - logger := s.log.WithValues("org", org) - if err := s.getReposByOrg(ctx, org); err != nil { - logger.Error(err, "error fetching repos by org") - } - - if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil { - logger.Error(err, "error fetching repos by user") + orgCtx := context.WithValue(ctx, "account", org) + userType, err := s.getReposByOrgOrUser(ctx, org) + if err != nil { + orgCtx.Logger().Error(err, "error fetching repos for org or user") + continue } - if s.conn.ScanUsers { - err := s.addMembersByOrg(ctx, org) - if err != nil { - logger.Error(err, "Unable to add members by org for org") + if userType == organization && s.conn.ScanUsers { + if err := s.addMembersByOrg(ctx, org); err != nil { + orgCtx.Logger().Error(err, "Unable to add members by org for org") } } } + if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil { + s.log.Error(err, "error fetching repos for the current user", "user", ghUser.GetLogin()) + } + // If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs. // So if we don't have ScanUsers enabled, add the user gists as normal. if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil { - s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin()) + s.log.Error(err, "error fetching gists for the current user", "user", ghUser.GetLogin()) } return nil @@ -693,7 +701,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app * s.log.Info("Scanning repos", "org_members", len(s.memberCache)) for member := range s.memberCache { logger := s.log.WithValues("member", member) - if err := s.getReposByUser(ctx, member); err != nil { + if err := s.addUserGistsToCache(ctx, member); err != nil { logger.Error(err, "error fetching gists by user") } if err := s.getReposByUser(ctx, member); err != nil { diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index 35a1f42b299c..f1eb88ce592c 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -1,6 +1,7 @@ package github import ( + "errors" "fmt" "io" "net/http" @@ -169,14 +170,6 @@ func (a *appListOptions) getListOptions() *github.ListOptions { return &a.ListOptions } -func (s *Source) getReposByApp(ctx context.Context) error { - return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{ - ListOptions: github.ListOptions{ - PerPage: defaultPagination, - }, - }) -} - func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions()) if someRepos != nil { @@ -185,6 +178,14 @@ func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoLis return nil, res, err } +func (s *Source) getReposByApp(ctx context.Context) error { + return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{ + ListOptions: github.ListOptions{ + PerPage: defaultPagination, + }, + }) +} + type userListOptions struct { github.RepositoryListByUserOptions } @@ -193,6 +194,10 @@ func (u *userListOptions) getListOptions() *github.ListOptions { return &u.ListOptions } +func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { + return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions) +} + func (s *Source) getReposByUser(ctx context.Context, user string) error { return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{ RepositoryListByUserOptions: github.RepositoryListByUserOptions{ @@ -203,10 +208,6 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error { }) } -func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions) -} - type orgListOptions struct { github.RepositoryListByOrgOptions } @@ -215,6 +216,10 @@ func (o *orgListOptions) getListOptions() *github.ListOptions { return &o.ListOptions } +func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { + return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) +} + func (s *Source) getReposByOrg(ctx context.Context, org string) error { return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{ RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{ @@ -225,8 +230,55 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error { }) } -func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) +// userType indicates whether an account belongs to a person or organization. +// +// See: +// - https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts +// - https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user +type userType int + +const ( + // Default invalid state. + unknown userType = iota + // The account is a person (https://docs.github.com/en/rest/users/users). + user + // The account is an organization (https://docs.github.com/en/rest/orgs/orgs). + organization +) + +func (s *Source) getReposByOrgOrUser(ctx context.Context, name string) (userType, error) { + var err error + + // List repositories for the organization |name|. + err = s.getReposByOrg(ctx, name) + if err == nil { + return organization, nil + } else if !isGitHub404Error(err) { + return unknown, err + } + + // List repositories for the user |name|. + err = s.getReposByUser(ctx, name) + if err == nil { + if err := s.addUserGistsToCache(ctx, name); err != nil { + ctx.Logger().Error(err, "Unable to add user to cache") + } + return user, nil + } else if !isGitHub404Error(err) { + return unknown, err + } + + return unknown, fmt.Errorf("account '%s' not found", name) +} + +// isGitHub404Error returns true if |err| is a `github.ErrorResponse` and has the status code `404`. +func isGitHub404Error(err error) bool { + var ghErr *github.ErrorResponse + if !errors.As(err, &ghErr) { + return false + } + + return ghErr.Response.StatusCode == http.StatusNotFound } func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {