Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[github] Scan user repositories #2814

Merged
merged 1 commit into from
May 23, 2024
+107 −47
Merged

[github] Scan user repositories #2814

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 41 additions & 33 deletions pkg/sources/github/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ import (
"time"

"golang.org/x/exp/rand"
"golang.org/x/oauth2"

"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-logr/logr"
"github.com/gobwas/glob"
"github.com/google/go-github/v62/github"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
Expand Down Expand Up @@ -480,8 +480,17 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
s.apiClient = ghClient

for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org or user")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}

Expand All @@ -499,17 +508,15 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
}

for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org")
}

// We probably don't need to do this, since getting repos by org makes more sense?
if err := s.getReposByUser(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for user")
rgmz marked this conversation as resolved.
Show resolved Hide resolved
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if s.conn.ScanUsers {
s.log.Info("Enumerating unauthenticated does not support scanning organization members")
if userType == organization && s.conn.ScanUsers {
orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)")
}
}
}
Expand Down Expand Up @@ -562,16 +569,16 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
if s.orgsCache.Count() > 0 {
specificScope = true
for _, org := range s.orgsCache.Keys() {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
logger.Error(err, "error fetching repos for org")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org")
continue
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}
Expand All @@ -593,27 +600,28 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
}

for _, org := range s.orgsCache.Keys() {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
logger.Error(err, "error fetching repos by org")
}

if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
logger.Error(err, "error fetching repos by user")
rgmz marked this conversation as resolved.
Show resolved Hide resolved
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org for org")
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org for org")
}
}
}

if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching repos for the current user", "user", ghUser.GetLogin())
}

// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
// So if we don't have ScanUsers enabled, add the user gists as normal.
if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())
s.log.Error(err, "error fetching gists for the current user", "user", ghUser.GetLogin())
}

return nil
Expand Down Expand Up @@ -693,7 +701,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
s.log.Info("Scanning repos", "org_members", len(s.memberCache))
for member := range s.memberCache {
logger := s.log.WithValues("member", member)
if err := s.getReposByUser(ctx, member); err != nil {
if err := s.addUserGistsToCache(ctx, member); err != nil {
rgmz marked this conversation as resolved.
Show resolved Hide resolved
logger.Error(err, "error fetching gists by user")
}
if err := s.getReposByUser(ctx, member); err != nil {
Expand Down
80 changes: 66 additions & 14 deletions pkg/sources/github/repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package github

import (
"errors"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -169,14 +170,6 @@ func (a *appListOptions) getListOptions() *github.ListOptions {
return &a.ListOptions
}

func (s *Source) getReposByApp(ctx context.Context) error {
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
},
})
}

func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())
if someRepos != nil {
Expand All @@ -185,6 +178,14 @@ func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoLis
return nil, res, err
}

func (s *Source) getReposByApp(ctx context.Context) error {
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
},
})
}

type userListOptions struct {
github.RepositoryListByUserOptions
}
Expand All @@ -193,6 +194,10 @@ func (u *userListOptions) getListOptions() *github.ListOptions {
return &u.ListOptions
}

func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}

func (s *Source) getReposByUser(ctx context.Context, user string) error {
return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{
RepositoryListByUserOptions: github.RepositoryListByUserOptions{
Expand All @@ -203,10 +208,6 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {
})
}

func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}

type orgListOptions struct {
github.RepositoryListByOrgOptions
}
Expand All @@ -215,6 +216,10 @@ func (o *orgListOptions) getListOptions() *github.ListOptions {
return &o.ListOptions
}

func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
}

func (s *Source) getReposByOrg(ctx context.Context, org string) error {
return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{
RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{
Expand All @@ -225,8 +230,55 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {
})
}

func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
// userType indicates whether an account belongs to a person or organization.
//
// See:
// - https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts
// - https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user
type userType int

const (
// Default invalid state.
unknown userType = iota
// The account is a person (https://docs.github.com/en/rest/users/users).
user
// The account is an organization (https://docs.github.com/en/rest/orgs/orgs).
organization
)

func (s *Source) getReposByOrgOrUser(ctx context.Context, name string) (userType, error) {
var err error

// List repositories for the organization |name|.
err = s.getReposByOrg(ctx, name)
if err == nil {
return organization, nil
} else if !isGitHub404Error(err) {
return unknown, err
}

// List repositories for the user |name|.
err = s.getReposByUser(ctx, name)
if err == nil {
if err := s.addUserGistsToCache(ctx, name); err != nil {
ctx.Logger().Error(err, "Unable to add user to cache")
}
return user, nil
} else if !isGitHub404Error(err) {
return unknown, err
}

return unknown, fmt.Errorf("account '%s' not found", name)
}

// isGitHub404Error returns true if |err| is a `github.ErrorResponse` and has the status code `404`.
func isGitHub404Error(err error) bool {
var ghErr *github.ErrorResponse
if !errors.As(err, &ghErr) {
return false
}

return ghErr.Response.StatusCode == http.StatusNotFound
}

func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {
Expand Down
Loading