From 5f1697540724c4c10593c66927b6528b94519c6f Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Thu, 8 Feb 2024 21:15:34 -0500 Subject: [PATCH] fix(github): lookup wikis per-repo --- pkg/sources/github/github.go | 4 ++-- pkg/sources/github/repo.go | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 7c195fe676a8..7972e562d00e 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -720,13 +720,13 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch } // Scan the wiki, if enabled, and the repo has one. - if s.conn.IncludeWikis && repoInfo.hasWiki { + if s.conn.IncludeWikis && repoInfo.hasWiki && s.hasWiki(ctx, repoURL) { wikiURL := strings.TrimSuffix(repoURL, ".git") + ".wiki.git" wikiCtx := context.WithValue(ctx, "repo", wikiURL) _, err := s.cloneAndScanRepo(wikiCtx, installationClient, wikiURL, repoInfo, chunksChan) if err != nil { - scanErrs.Add(err) + scanErrs.Add(fmt.Errorf("error scanning wiki: %s", wikiURL)) // Don't return, it still might be possible to scan comments. } } diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index aea6fad14125..6973b6637a5a 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -269,7 +269,7 @@ func (s *Source) processRepos(ctx context.Context, target string, listRepos repo owner: r.GetOwner().GetLogin(), name: r.GetName(), fullName: r.GetFullName(), - hasWiki: s.conn.GetIncludeWikis() && s.hasWiki(ctx, r, repoURL), + hasWiki: r.GetHasWiki(), size: r.GetSize(), } if r.GetPrivate() { @@ -295,11 +295,7 @@ func (s *Source) processRepos(ctx context.Context, target string, listRepos repo // hasWiki returns true if the "has_wiki" property is true AND https://github.com/$org/$repo/wiki is not redirected. // Unfortunately, this isn't 100% accurate. Some repositories meet both criteria yet don't have a cloneable wiki. -func (s *Source) hasWiki(ctx context.Context, repo *github.Repository, repoURL string) bool { - if !repo.GetHasWiki() { - return false - } - +func (s *Source) hasWiki(ctx context.Context, repoURL string) bool { wikiURL := strings.TrimSuffix(repoURL, ".git") + "/wiki" req, err := http.NewRequestWithContext(ctx, http.MethodHead, wikiURL, nil) if err != nil {