From 488b53823350caa706de3f01ec0eded9350c7da7 Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Sat, 10 Feb 2024 17:07:55 -0800 Subject: [PATCH] Plumb contexts everywhere Signed-off-by: Jon Johnson --- pkg/advisory/create.go | 13 ++++---- pkg/advisory/create_test.go | 5 +-- pkg/advisory/diff_test.go | 5 +-- pkg/advisory/discover.go | 2 +- pkg/advisory/discover_aliases.go | 8 ++--- pkg/advisory/discover_aliases_test.go | 2 +- pkg/advisory/export_test.go | 3 +- pkg/advisory/secdb_test.go | 3 +- pkg/advisory/update.go | 7 ++-- pkg/advisory/update_test.go | 5 +-- pkg/advisory/validate_test.go | 18 +++++------ pkg/checks/update.go | 16 +++++----- pkg/cli/advisory_alias_discover.go | 2 +- pkg/cli/advisory_create.go | 6 ++-- pkg/cli/advisory_diff.go | 4 +-- pkg/cli/advisory_discover.go | 7 ++-- pkg/cli/advisory_export.go | 2 +- pkg/cli/advisory_list.go | 2 +- pkg/cli/advisory_migrate.go | 2 +- pkg/cli/advisory_secdb.go | 2 +- pkg/cli/advisory_update.go | 6 ++-- pkg/cli/advisory_validate.go | 6 ++-- pkg/cli/gh_gc_branch.go | 7 ++-- pkg/cli/gh_gc_issues.go | 7 ++-- pkg/cli/gh_release.go | 26 +++++++++++++-- pkg/cli/image_apk.go | 7 ++-- pkg/cli/ls.go | 2 +- pkg/cli/ruby_check_upgrade.go | 3 +- pkg/cli/ruby_code_search.go | 3 +- pkg/cli/sbom.go | 4 +-- pkg/cli/scan.go | 10 +++--- pkg/cli/update.go | 2 +- pkg/cli/update_package.go | 25 ++++++++++++++- pkg/configs/advisory/v1/advisory.go | 13 ++++---- pkg/configs/advisory/v2/index.go | 13 ++++---- pkg/configs/build/build.go | 14 ++++---- pkg/configs/build/build_test.go | 13 ++++---- pkg/configs/build/selection_test.go | 3 +- pkg/configs/entry.go | 7 ++-- pkg/configs/index.go | 33 +++++++++---------- pkg/configs/index_test.go | 2 +- pkg/configs/selection.go | 5 +-- pkg/dag/graph.go | 20 ++++++------ pkg/gh/pull_request.go | 4 +-- pkg/gh/release.go | 34 +++----------------- pkg/http/http.go | 4 +-- pkg/ls/ls_test.go | 3 +- pkg/sbom/cache.go | 5 +-- pkg/sbom/sbom.go | 4 +-- pkg/scan/filter_test.go | 5 +-- pkg/update/package.go | 39 ++++------------------- pkg/update/update.go | 46 +++++++++++++-------------- 52 files changed, 248 insertions(+), 241 deletions(-) diff --git a/pkg/advisory/create.go b/pkg/advisory/create.go index 7ad29bc2..72d75734 100644 --- a/pkg/advisory/create.go +++ b/pkg/advisory/create.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "fmt" "sort" @@ -16,7 +17,7 @@ type CreateOptions struct { // Create creates a new advisory in the `advisories` section of the document at // the provided path. -func Create(req Request, opts CreateOptions) error { +func Create(ctx context.Context, req Request, opts CreateOptions) error { err := req.Validate() if err != nil { return err @@ -28,7 +29,7 @@ func Create(req Request, opts CreateOptions) error { switch count { case 0: // i.e. no advisories file for this package yet - return createAdvisoryConfig(opts.AdvisoryDocs, req) + return createAdvisoryConfig(ctx, opts.AdvisoryDocs, req) case 1: newAdvisoryID := req.VulnerabilityID @@ -52,13 +53,13 @@ func Create(req Request, opts CreateOptions) error { return advisories, nil }) - err := documents.Update(u) + err := documents.Update(ctx, u) if err != nil { return fmt.Errorf("unable to create advisory %q for %q: %w", newAdvisoryID, req.Package, err) } // Update the schema version to the latest version. - err = documents.Update(v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) + err = documents.Update(ctx, v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) if err != nil { return fmt.Errorf("unable to update schema version for %q: %w", req.Package, err) } @@ -69,7 +70,7 @@ func Create(req Request, opts CreateOptions) error { return fmt.Errorf("cannot create advisory: found %d advisory documents for package %q", count, req.Package) } -func createAdvisoryConfig(documents *configs.Index[v2.Document], req Request) error { +func createAdvisoryConfig(ctx context.Context, documents *configs.Index[v2.Document], req Request) error { newAdvisoryID := req.VulnerabilityID newAdvisory := v2.Advisory{ ID: newAdvisoryID, @@ -77,7 +78,7 @@ func createAdvisoryConfig(documents *configs.Index[v2.Document], req Request) er Events: []v2.Event{req.Event}, } - err := documents.Create(fmt.Sprintf("%s.advisories.yaml", req.Package), v2.Document{ + err := documents.Create(ctx, fmt.Sprintf("%s.advisories.yaml", req.Package), v2.Document{ SchemaVersion: v2.SchemaVersion, Package: v2.Package{ Name: req.Package, diff --git a/pkg/advisory/create_test.go b/pkg/advisory/create_test.go index 74b5f2b8..2556e224 100644 --- a/pkg/advisory/create_test.go +++ b/pkg/advisory/create_test.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "os" "testing" "time" @@ -190,10 +191,10 @@ func TestCreate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // We want a fresh memfs for each test case. fsys := memfs.New(dirFS) - advisoryDocs, err := v2.NewIndex(fsys) + advisoryDocs, err := v2.NewIndex(context.Background(), fsys) require.NoError(t, err) - err = Create(tt.req, CreateOptions{ + err = Create(context.Background(), tt.req, CreateOptions{ AdvisoryDocs: advisoryDocs, }) diff --git a/pkg/advisory/diff_test.go b/pkg/advisory/diff_test.go index dc7e3472..f19754df 100644 --- a/pkg/advisory/diff_test.go +++ b/pkg/advisory/diff_test.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "path/filepath" "testing" "time" @@ -248,9 +249,9 @@ func TestIndexDiff(t *testing.T) { bDir := filepath.Join("testdata", "diff", tt.name, "b") aFsys := rwos.DirFS(aDir) bFsys := rwos.DirFS(bDir) - aIndex, err := v2.NewIndex(aFsys) + aIndex, err := v2.NewIndex(context.Background(), aFsys) require.NoError(t, err) - bIndex, err := v2.NewIndex(bFsys) + bIndex, err := v2.NewIndex(context.Background(), bFsys) require.NoError(t, err) diff := IndexDiff(aIndex, bIndex) diff --git a/pkg/advisory/discover.go b/pkg/advisory/discover.go index 3f652fca..6206c225 100644 --- a/pkg/advisory/discover.go +++ b/pkg/advisory/discover.go @@ -101,7 +101,7 @@ func (opts DiscoverOptions) discoverMatchesForPackage(ctx context.Context, pkg s for i := range matches { match := matches[i] - err := Create(Request{ + err := Create(ctx, Request{ Package: pkg, VulnerabilityID: match.Vulnerability.ID, Aliases: nil, diff --git a/pkg/advisory/discover_aliases.go b/pkg/advisory/discover_aliases.go index 73550982..4b9ce11d 100644 --- a/pkg/advisory/discover_aliases.go +++ b/pkg/advisory/discover_aliases.go @@ -54,13 +54,13 @@ func (opts *DiscoverAliasesOptions) discoverAliasesForAdvisoriesWithCVEIDs(ctx c return advisories, nil }) - err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(u) + err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(ctx, u) if err != nil { return err } // Update the schema version to the latest version. - err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) + err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(ctx, v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) if err != nil { return fmt.Errorf("unable to update schema version for %q: %w", doc.Name(), err) } @@ -120,13 +120,13 @@ func (opts *DiscoverAliasesOptions) discoverAliasesForAdvisoriesWithGHSAIDs(ctx return advisories, nil }) - err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(u) + err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(ctx, u) if err != nil { return err } // Update the schema version to the latest version. - err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) + err = opts.AdvisoryDocs.Select().WhereName(doc.Name()).Update(ctx, v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) if err != nil { return fmt.Errorf("unable to update schema version for %q: %w", doc.Name(), err) } diff --git a/pkg/advisory/discover_aliases_test.go b/pkg/advisory/discover_aliases_test.go index 077d3fc1..5718b2da 100644 --- a/pkg/advisory/discover_aliases_test.go +++ b/pkg/advisory/discover_aliases_test.go @@ -140,7 +140,7 @@ func TestDiscoverAliases(t *testing.T) { // back to disk. fsys := memfs.New(os.DirFS("testdata/discover_aliases/advisories")) - advisoryDocs, err := v2.NewIndex(fsys) + advisoryDocs, err := v2.NewIndex(context.Background(), fsys) if err != nil { t.Fatalf("unable to create advisory docs index: %v", err) } diff --git a/pkg/advisory/export_test.go b/pkg/advisory/export_test.go index 76fba230..fa48c924 100644 --- a/pkg/advisory/export_test.go +++ b/pkg/advisory/export_test.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "io" "os" "testing" @@ -36,7 +37,7 @@ func Test_ExportFuncs(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { advisoryFsys := rwos.DirFS(testdataDir) - advisoryDocs, err := v2.NewIndex(advisoryFsys) + advisoryDocs, err := v2.NewIndex(context.Background(), advisoryFsys) require.NoError(t, err) indices := []*configs.Index[v2.Document]{advisoryDocs} diff --git a/pkg/advisory/secdb_test.go b/pkg/advisory/secdb_test.go index 58c14ada..7c453973 100644 --- a/pkg/advisory/secdb_test.go +++ b/pkg/advisory/secdb_test.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "os" "testing" @@ -65,7 +66,7 @@ func TestBuildSecurityDatabase(t *testing.T) { indices := make([]*configs.Index[v2.Document], 0, len(tt.advisoryDirs)) for _, dir := range tt.advisoryDirs { advisoryFsys := rwos.DirFS(dir) - advisoryCfgs, err := v2.NewIndex(advisoryFsys) + advisoryCfgs, err := v2.NewIndex(context.Background(), advisoryFsys) require.NoError(t, err) indices = append(indices, advisoryCfgs) } diff --git a/pkg/advisory/update.go b/pkg/advisory/update.go index c8b5775a..ddfe7279 100644 --- a/pkg/advisory/update.go +++ b/pkg/advisory/update.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "fmt" "sort" @@ -16,7 +17,7 @@ type UpdateOptions struct { // Update adds a new event to an existing advisory (named by the vuln parameter) // in the document at the provided path. -func Update(req Request, opts UpdateOptions) error { +func Update(ctx context.Context, req Request, opts UpdateOptions) error { vulnID := req.VulnerabilityID documents := opts.AdvisoryDocs.Select().WhereName(req.Package) @@ -40,13 +41,13 @@ func Update(req Request, opts UpdateOptions) error { return advisories, nil }) - err := documents.Update(u) + err := documents.Update(ctx, u) if err != nil { return fmt.Errorf("unable to add entry for advisory %q in %q: %w", vulnID, req.Package, err) } // Update the schema version to the latest version. - err = documents.Update(v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) + err = documents.Update(ctx, v2.NewSchemaVersionSectionUpdater(v2.SchemaVersion)) if err != nil { return fmt.Errorf("unable to update schema version for %q: %w", req.Package, err) } diff --git a/pkg/advisory/update_test.go b/pkg/advisory/update_test.go index 5d694c3f..1119498b 100644 --- a/pkg/advisory/update_test.go +++ b/pkg/advisory/update_test.go @@ -1,6 +1,7 @@ package advisory import ( + "context" "os" "testing" "time" @@ -122,10 +123,10 @@ func TestUpdate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // We want a fresh memfs for each test case. memFS := memfs.New(dirFS) - advisoryDocs, err := v2.NewIndex(memFS) + advisoryDocs, err := v2.NewIndex(context.Background(), memFS) require.NoError(t, err) - err = Update(tt.req, UpdateOptions{ + err = Update(context.Background(), tt.req, UpdateOptions{ AdvisoryDocs: advisoryDocs, }) diff --git a/pkg/advisory/validate_test.go b/pkg/advisory/validate_test.go index 201e1381..32771c09 100644 --- a/pkg/advisory/validate_test.go +++ b/pkg/advisory/validate_test.go @@ -66,9 +66,9 @@ func TestValidate(t *testing.T) { bDir := filepath.Join("testdata", "diff", tt.name, "b") aFsys := rwos.DirFS(aDir) bFsys := rwos.DirFS(bDir) - aIndex, err := v2.NewIndex(aFsys) + aIndex, err := v2.NewIndex(context.Background(), aFsys) require.NoError(t, err) - bIndex, err := v2.NewIndex(bFsys) + bIndex, err := v2.NewIndex(context.Background(), bFsys) require.NoError(t, err) err = Validate(context.Background(), ValidateOptions{ @@ -186,9 +186,9 @@ func TestValidate(t *testing.T) { bDir := filepath.Join("testdata", "diff", tt.name, "b") aFsys := rwos.DirFS(aDir) bFsys := rwos.DirFS(bDir) - aIndex, err := v2.NewIndex(aFsys) + aIndex, err := v2.NewIndex(context.Background(), aFsys) require.NoError(t, err) - bIndex, err := v2.NewIndex(bFsys) + bIndex, err := v2.NewIndex(context.Background(), bFsys) require.NoError(t, err) err = Validate(context.Background(), ValidateOptions{ @@ -241,7 +241,7 @@ func TestValidate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dir := filepath.Join("testdata", "validate", tt.name) fsys := rwos.DirFS(dir) - index, err := v2.NewIndex(fsys) + index, err := v2.NewIndex(context.Background(), fsys) require.NoError(t, err) err = Validate(context.Background(), ValidateOptions{ @@ -318,7 +318,7 @@ func TestValidate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dir := filepath.Join("testdata", "validate", "fixed-version") fsys := rwos.DirFS(dir) - index, err := v2.NewIndex(fsys) + index, err := v2.NewIndex(context.Background(), fsys) require.NoError(t, err) err = Validate(context.Background(), ValidateOptions{ @@ -359,7 +359,7 @@ func TestValidate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dir := filepath.Join("testdata", "validate", tt.name) fsys := rwos.DirFS(dir) - index, err := v2.NewIndex(fsys) + index, err := v2.NewIndex(context.Background(), fsys) require.NoError(t, err) err = Validate(context.Background(), ValidateOptions{ @@ -378,14 +378,14 @@ func TestValidate(t *testing.T) { func distroWithKo(t *testing.T) *configs.Index[config.Configuration] { fsys := rwos.DirFS(filepath.Join("testdata", "validate", "package-existence", "distro")) - index, err := build.NewIndex(fsys) + index, err := build.NewIndex(context.Background(), fsys) require.NoError(t, err) return index } func distroWithNothing(t *testing.T) *configs.Index[config.Configuration] { fsys := rwos.DirFS(filepath.Join("testdata", "validate", "package-existence", "distro-empty")) - index, err := build.NewIndex(fsys) + index, err := build.NewIndex(context.Background(), fsys) require.NoError(t, err) return index } diff --git a/pkg/checks/update.go b/pkg/checks/update.go index b4761f46..6141eb8b 100644 --- a/pkg/checks/update.go +++ b/pkg/checks/update.go @@ -34,8 +34,8 @@ type CheckUpdateOptions struct { } // SetupUpdate will create the options needed to call wolfictl update functions -func SetupUpdate() (*update.Options, lint.EvalRuleErrors) { - o := update.New() +func SetupUpdate(ctx context.Context) (*update.Options, lint.EvalRuleErrors) { + o := update.New(ctx) o.GithubReleaseQuery = true o.ReleaseMonitoringQuery = true o.ErrorMessages = make(map[string]string) @@ -47,7 +47,7 @@ func SetupUpdate() (*update.Options, lint.EvalRuleErrors) { // CheckUpdates will use the melange update config to get the latest versions and validate fetch and git-checkout pipelines func (o CheckUpdateOptions) CheckUpdates(ctx context.Context, files []string) error { - updateOpts, checkErrors := SetupUpdate() + updateOpts, checkErrors := SetupUpdate(ctx) changedPackages := GetPackagesToUpdate(files) @@ -254,7 +254,7 @@ func (o CheckUpdateOptions) processUpdates(ctx context.Context, latestVersions m } // download or git clone sources into a temp folder to validate the update config - verifyPipelines(o, updated, mutations, checkErrors) + verifyPipelines(ctx, o, updated, mutations, checkErrors) } return nil } @@ -265,13 +265,13 @@ func applyOverrides(options *CheckUpdateOptions, dryRunConfig *config.Configurat } } -func verifyPipelines(o CheckUpdateOptions, updated *config.Configuration, mutations map[string]string, checkErrors *lint.EvalRuleErrors) { +func verifyPipelines(ctx context.Context, o CheckUpdateOptions, updated *config.Configuration, mutations map[string]string, checkErrors *lint.EvalRuleErrors) { for i := range updated.Pipeline { var err error pipeline := updated.Pipeline[i] if pipeline.Uses == "fetch" { - err = o.verifyFetch(&pipeline, mutations) + err = o.verifyFetch(ctx, &pipeline, mutations) } if pipeline.Uses == "git-checkout" { err = o.verifyGitCheckout(&pipeline, mutations) @@ -282,7 +282,7 @@ func verifyPipelines(o CheckUpdateOptions, updated *config.Configuration, mutati } } -func (o CheckUpdateOptions) verifyFetch(p *config.Pipeline, m map[string]string) error { +func (o CheckUpdateOptions) verifyFetch(ctx context.Context, p *config.Pipeline, m map[string]string) error { uriValue := p.With["uri"] if uriValue == "" { return fmt.Errorf("no uri to fetch") @@ -296,7 +296,7 @@ func (o CheckUpdateOptions) verifyFetch(p *config.Pipeline, m map[string]string) o.Logger.Printf("downloading sources from %s into a temporary directory, this may take a while", evaluatedURI) - filename, err := util.DownloadFile(context.TODO(), evaluatedURI) + filename, err := util.DownloadFile(ctx, evaluatedURI) if err != nil { return fmt.Errorf("failed to verify fetch %s: %w", evaluatedURI, err) } diff --git a/pkg/cli/advisory_alias_discover.go b/pkg/cli/advisory_alias_discover.go index c768cce9..28dac989 100644 --- a/pkg/cli/advisory_alias_discover.go +++ b/pkg/cli/advisory_alias_discover.go @@ -64,7 +64,7 @@ than attempting any kind of merge of the separate advisories. } advisoriesFsys := rwos.DirFS(advisoriesRepoDir) - advisoryDocs, err := v2.NewIndex(advisoriesFsys) + advisoryDocs, err := v2.NewIndex(cmd.Context(), advisoriesFsys) if err != nil { return err } diff --git a/pkg/cli/advisory_create.go b/pkg/cli/advisory_create.go index 631b2016..546807a8 100644 --- a/pkg/cli/advisory_create.go +++ b/pkg/cli/advisory_create.go @@ -75,13 +75,13 @@ newly created advisory and any other advisories for the same package.`, } advisoryFsys := rwos.DirFS(advisoriesRepoDir) - advisoryCfgs, err := v2.NewIndex(advisoryFsys) + advisoryCfgs, err := v2.NewIndex(cmd.Context(), advisoryFsys) if err != nil { return err } fsys := rwos.DirFS(distroRepoDir) - buildCfgs, err := buildconfigs.NewIndex(fsys) + buildCfgs, err := buildconfigs.NewIndex(cmd.Context(), fsys) if err != nil { return fmt.Errorf("unable to select packages: %w", err) } @@ -147,7 +147,7 @@ newly created advisory and any other advisories for the same package.`, AdvisoryDocs: advisoryCfgs, } - err = advisory.Create(req, opts) + err = advisory.Create(cmd.Context(), req, opts) if err != nil { return fmt.Errorf("unable to create advisory: %w", err) } diff --git a/pkg/cli/advisory_diff.go b/pkg/cli/advisory_diff.go index 9b38c504..a0a4b6e7 100644 --- a/pkg/cli/advisory_diff.go +++ b/pkg/cli/advisory_diff.go @@ -46,12 +46,12 @@ func cmdAdvisoryDiff() *cobra.Command { } baseAdvisoriesFsys := rwos.DirFS(cloneDir) - baseAdvisoriesIndex, err := v2.NewIndex(baseAdvisoriesFsys) + baseAdvisoriesIndex, err := v2.NewIndex(cmd.Context(), baseAdvisoriesFsys) if err != nil { return err } - currentAdvisoriesIndex, err := v2.NewIndex(rwos.DirFS(d.Local.AdvisoriesRepo.Dir)) + currentAdvisoriesIndex, err := v2.NewIndex(cmd.Context(), rwos.DirFS(d.Local.AdvisoriesRepo.Dir)) if err != nil { return err } diff --git a/pkg/cli/advisory_discover.go b/pkg/cli/advisory_discover.go index 215ded62..4db5dba7 100644 --- a/pkg/cli/advisory_discover.go +++ b/pkg/cli/advisory_discover.go @@ -1,7 +1,6 @@ package cli import ( - "context" "errors" "fmt" "log" @@ -58,13 +57,13 @@ func cmdAdvisoryDiscover() *cobra.Command { } advisoriesFsys := rwfsOS.DirFS(advisoriesRepoDir) - advisoryCfgs, err := v2.NewIndex(advisoriesFsys) + advisoryCfgs, err := v2.NewIndex(cmd.Context(), advisoriesFsys) if err != nil { return err } fsys := rwfsOS.DirFS(distroRepoDir) - buildCfgs, err := buildconfigs.NewIndex(fsys) + buildCfgs, err := buildconfigs.NewIndex(cmd.Context(), fsys) if err != nil { return fmt.Errorf("unable to select packages: %w", err) } @@ -72,7 +71,7 @@ func cmdAdvisoryDiscover() *cobra.Command { selectedPackages := getSelectedOrDistroPackages(p.packageName, buildCfgs) apiKey := p.resolveNVDAPIKey() - ctx := context.Background() + ctx := cmd.Context() g, ctx := errgroup.WithContext(ctx) events := make(chan interface{}) diff --git a/pkg/cli/advisory_export.go b/pkg/cli/advisory_export.go index eea74914..d0e75a3d 100644 --- a/pkg/cli/advisory_export.go +++ b/pkg/cli/advisory_export.go @@ -40,7 +40,7 @@ func cmdAdvisoryExport() *cobra.Command { indices := make([]*configs.Index[v2.Document], 0, len(p.advisoriesRepoDirs)) for _, dir := range p.advisoriesRepoDirs { advisoryFsys := rwos.DirFS(dir) - index, err := v2.NewIndex(advisoryFsys) + index, err := v2.NewIndex(cmd.Context(), advisoryFsys) if err != nil { return fmt.Errorf("unable to index advisory configs for directory %q: %w", dir, err) } diff --git a/pkg/cli/advisory_list.go b/pkg/cli/advisory_list.go index ff535328..af39e67d 100644 --- a/pkg/cli/advisory_list.go +++ b/pkg/cli/advisory_list.go @@ -64,7 +64,7 @@ investigation over time for a given package/vulnerability match.' } advisoriesFsys := rwos.DirFS(advisoriesRepoDir) - advisoryCfgs, err := v2.NewIndex(advisoriesFsys) + advisoryCfgs, err := v2.NewIndex(cmd.Context(), advisoriesFsys) if err != nil { return err } diff --git a/pkg/cli/advisory_migrate.go b/pkg/cli/advisory_migrate.go index cfbe76cf..4256825e 100644 --- a/pkg/cli/advisory_migrate.go +++ b/pkg/cli/advisory_migrate.go @@ -82,7 +82,7 @@ func cmdAdvisoryMigrate() *cobra.Command { advisoriesRepoDir := "." advisoriesFsys := rwos.DirFS(advisoriesRepoDir) - v1Index, err := v1.NewIndex(advisoriesFsys) + v1Index, err := v1.NewIndex(cmd.Context(), advisoriesFsys) if err != nil { return err } diff --git a/pkg/cli/advisory_secdb.go b/pkg/cli/advisory_secdb.go index a283a22f..eea654e6 100644 --- a/pkg/cli/advisory_secdb.go +++ b/pkg/cli/advisory_secdb.go @@ -38,7 +38,7 @@ func cmdAdvisorySecDB() *cobra.Command { indices := make([]*configs.Index[v2.Document], 0, len(p.advisoriesRepoDirs)) for _, dir := range p.advisoriesRepoDirs { advisoryFsys := rwos.DirFS(dir) - index, err := v2.NewIndex(advisoryFsys) + index, err := v2.NewIndex(cmd.Context(), advisoryFsys) if err != nil { return fmt.Errorf("unable to index advisory configs for directory %q: %w", dir, err) } diff --git a/pkg/cli/advisory_update.go b/pkg/cli/advisory_update.go index 0f12cd90..797c4306 100644 --- a/pkg/cli/advisory_update.go +++ b/pkg/cli/advisory_update.go @@ -71,13 +71,13 @@ required fields are missing.`, } advisoryFsys := rwos.DirFS(advisoriesRepoDir) - advisoryCfgs, err := v2.NewIndex(advisoryFsys) + advisoryCfgs, err := v2.NewIndex(cmd.Context(), advisoryFsys) if err != nil { return err } fsys := rwos.DirFS(distroRepoDir) - buildCfgs, err := buildconfigs.NewIndex(fsys) + buildCfgs, err := buildconfigs.NewIndex(cmd.Context(), fsys) if err != nil { return fmt.Errorf("unable to select packages: %w", err) } @@ -151,7 +151,7 @@ required fields are missing.`, AdvisoryDocs: advisoryCfgs, } - err = advisory.Update(req, opts) + err = advisory.Update(cmd.Context(), req, opts) if err != nil { return err } diff --git a/pkg/cli/advisory_validate.go b/pkg/cli/advisory_validate.go index 67bdb9c6..c6d0fa98 100644 --- a/pkg/cli/advisory_validate.go +++ b/pkg/cli/advisory_validate.go @@ -154,13 +154,13 @@ print an error message that specifies where and how the data is invalid.`, logger.Debug("cloned upstream advisories repo for comparison", "dir", cloneDir) - baseAdvisoriesIndex, err = v2.NewIndex(rwos.DirFS(cloneDir)) + baseAdvisoriesIndex, err = v2.NewIndex(cmd.Context(), rwos.DirFS(cloneDir)) if err != nil { return fmt.Errorf("unable to create index of upstream advisories for comparison: %w", err) } } - advisoriesIndex, err := v2.NewIndex(rwos.DirFS(advisoriesRepoDir)) + advisoriesIndex, err := v2.NewIndex(cmd.Context(), rwos.DirFS(advisoriesRepoDir)) if err != nil { return fmt.Errorf("unable to create index of advisories repo: %w", err) } @@ -172,7 +172,7 @@ print an error message that specifies where and how the data is invalid.`, var packageConfigurationsIndex *configs.Index[config.Configuration] if !p.skipPackageExistenceValidation { - packageConfigurationsIndex, err = build.NewIndex(rwos.DirFS(packagesRepoDir)) + packageConfigurationsIndex, err = build.NewIndex(cmd.Context(), rwos.DirFS(packagesRepoDir)) if err != nil { return fmt.Errorf("unable to create index of distro package configurations: %w", err) } diff --git a/pkg/cli/gh_gc_branch.go b/pkg/cli/gh_gc_branch.go index 71d7cb35..12db2d2a 100644 --- a/pkg/cli/gh_gc_branch.go +++ b/pkg/cli/gh_gc_branch.go @@ -39,13 +39,13 @@ wolfictl gh gc branch https://github.com/wolfi-dev/os --match "wolfictl-" return errors.New("you must either pass --all to close all branches or provide a match pattern with --match") } client := &http2.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ghTokenSource{}), + Client: oauth2.NewClient(cmd.Context(), ghTokenSource{}), // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits Ratelimiter: rate.NewLimiter(rate.Every(5*time.Second), 1), } - return gcBranches(client, args[0], match) + return gcBranches(cmd.Context(), client, args[0], match) }, } @@ -64,9 +64,8 @@ func (ghTokenSource) Token() (*oauth2.Token, error) { return nil, errors.New("could not find github token") } -func gcBranches(ghclient *http2.RLHTTPClient, repo, match string) error { +func gcBranches(ctx context.Context, ghclient *http2.RLHTTPClient, repo, match string) error { logger := log.New(log.Writer(), "wolfictl gh gc branch: ", log.LstdFlags|log.Lmsgprefix) - ctx := context.Background() gitURL, err := wgit.ParseGitURL(repo) if err != nil { diff --git a/pkg/cli/gh_gc_issues.go b/pkg/cli/gh_gc_issues.go index df4b7977..b12483a2 100644 --- a/pkg/cli/gh_gc_issues.go +++ b/pkg/cli/gh_gc_issues.go @@ -40,13 +40,13 @@ wolfictl gc issues https://github.com/wolfi-dev/versions --match "version-stream &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, ) client := &http2.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ts), + Client: oauth2.NewClient(cmd.Context(), ts), // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits Ratelimiter: rate.NewLimiter(rate.Every(5*time.Second), 1), } - return gcIssues(client, args[0], match) + return gcIssues(cmd.Context(), client, args[0], match) }, } @@ -56,9 +56,8 @@ wolfictl gc issues https://github.com/wolfi-dev/versions --match "version-stream return cmd } -func gcIssues(ghclient *http2.RLHTTPClient, repo, match string) error { +func gcIssues(ctx context.Context, ghclient *http2.RLHTTPClient, repo, match string) error { logger := log.New(log.Writer(), "wolfictl gh gc issues: ", log.LstdFlags|log.Lmsgprefix) - ctx := context.Background() gitURL, err := wgit.ParseGitURL(repo) if err != nil { diff --git a/pkg/cli/gh_release.go b/pkg/cli/gh_release.go index 1682aaaa..5e3c0cc7 100644 --- a/pkg/cli/gh_release.go +++ b/pkg/cli/gh_release.go @@ -2,13 +2,22 @@ package cli import ( "errors" + "log" + "os" + "time" + "github.com/google/go-github/v58/github" "github.com/spf13/cobra" "github.com/wolfi-dev/wolfictl/pkg/gh" + http2 "github.com/wolfi-dev/wolfictl/pkg/http" + "golang.org/x/oauth2" + "golang.org/x/time/rate" ) func Release() *cobra.Command { - releaseOpts := gh.NewReleaseOptions() + releaseOpts := gh.ReleaseOptions{ + Logger: log.New(log.Writer(), "wolfictl gh release: ", log.LstdFlags|log.Lmsgprefix), + } cmd := &cobra.Command{ Use: "release", @@ -26,6 +35,19 @@ wolfictl gh release --bump-prerelease-with-prefix rc `, Args: cobra.RangeArgs(0, 0), RunE: func(cmd *cobra.Command, _ []string) error { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ) + + ratelimit := &http2.RLHTTPClient{ + Client: oauth2.NewClient(cmd.Context(), ts), + + // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits + Ratelimiter: rate.NewLimiter(rate.Every(3*time.Second), 1), + } + + releaseOpts.GithubClient = github.NewClient(ratelimit.Client) + if !releaseOpts.BumpMajor && !releaseOpts.BumpMinor && !releaseOpts.BumpPatch && @@ -33,7 +55,7 @@ wolfictl gh release --bump-prerelease-with-prefix rc return errors.New("missing flag to bump release version") } - return releaseOpts.Release() + return releaseOpts.Release(cmd.Context()) }, } diff --git a/pkg/cli/image_apk.go b/pkg/cli/image_apk.go index 49c4b3f9..9e1640fd 100644 --- a/pkg/cli/image_apk.go +++ b/pkg/cli/image_apk.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "path" "slices" @@ -81,7 +82,7 @@ func cmdImageAPK() *cobra.Command { return nil } - configPaths, err := apkConfigPaths(apks, p.distroDirPaths) + configPaths, err := apkConfigPaths(cmd.Context(), apks, p.distroDirPaths) if err != nil { return fmt.Errorf("unable to find configuration files for APK packages: %w", err) } @@ -205,12 +206,12 @@ func (r *syftResults) indexPackageOwnerships() (ownershipsByOwnedName, error) { return index, nil } -func apkConfigPaths(apks []pkg.Package, distroDirPaths []string) ([]string, error) { +func apkConfigPaths(ctx context.Context, apks []pkg.Package, distroDirPaths []string) ([]string, error) { configIndexes := make([]*configs.Index[config.Configuration], 0, len(distroDirPaths)) indexRootPaths := make([]string, 0, len(distroDirPaths)) for _, p := range distroDirPaths { - index, err := build.NewIndex(rwos.DirFS(p)) + index, err := build.NewIndex(ctx, rwos.DirFS(p)) if err != nil { return nil, fmt.Errorf("unable to index configuration files from %q: %w", p, err) } diff --git a/pkg/cli/ls.go b/pkg/cli/ls.go index 22ed62c6..23741620 100644 --- a/pkg/cli/ls.go +++ b/pkg/cli/ls.go @@ -39,7 +39,7 @@ func cmdLs() *cobra.Command { indices := make([]*configs.Index[config.Configuration], 0, len(p.distroRepoDirs)) for _, dir := range p.distroRepoDirs { distroFsys := rwos.DirFS(dir) - index, err := buildconfigs.NewIndex(distroFsys) + index, err := buildconfigs.NewIndex(cmd.Context(), distroFsys) if err != nil { return fmt.Errorf("unable to index build configs for directory %q: %w", dir, err) } diff --git a/pkg/cli/ruby_check_upgrade.go b/pkg/cli/ruby_check_upgrade.go index 3256359f..ff7965b9 100644 --- a/pkg/cli/ruby_check_upgrade.go +++ b/pkg/cli/ruby_check_upgrade.go @@ -1,7 +1,6 @@ package cli import ( - "context" "fmt" "time" @@ -47,7 +46,7 @@ wolfictl ruby check-upgrade . --ruby-version 3.2 --ruby-upgrade-version 3.3 } client := &http2.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ghTokenSource{}), + Client: oauth2.NewClient(ctx, ghTokenSource{}), // 1 request every (n) second(s) to avoid DOS'ing server. // https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits diff --git a/pkg/cli/ruby_code_search.go b/pkg/cli/ruby_code_search.go index 1a64c407..c63085dd 100644 --- a/pkg/cli/ruby_code_search.go +++ b/pkg/cli/ruby_code_search.go @@ -1,7 +1,6 @@ package cli import ( - "context" "fmt" "time" @@ -51,7 +50,7 @@ wolfictl ruby code-search . --ruby-version 3.2 --search-terms 'language:ruby rac } client := &http2.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ghTokenSource{}), + Client: oauth2.NewClient(ctx, ghTokenSource{}), // 1 request every (n) second(s) to avoid DOS'ing server. // https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits diff --git a/pkg/cli/sbom.go b/pkg/cli/sbom.go index b164f05c..d9c9ff9f 100644 --- a/pkg/cli/sbom.go +++ b/pkg/cli/sbom.go @@ -48,9 +48,9 @@ func cmdSBOM() *cobra.Command { var s *sbomSyft.SBOM if p.disableSBOMCache { - s, err = sbom.Generate(apkFilePath, apkFile, p.distro) + s, err = sbom.Generate(cmd.Context(), apkFilePath, apkFile, p.distro) } else { - s, err = sbom.CachedGenerate(apkFilePath, apkFile, p.distro) + s, err = sbom.CachedGenerate(cmd.Context(), apkFilePath, apkFile, p.distro) } if err != nil { return fmt.Errorf("failed to generate SBOM: %w", err) diff --git a/pkg/cli/scan.go b/pkg/cli/scan.go index 147c7da2..06a829d1 100644 --- a/pkg/cli/scan.go +++ b/pkg/cli/scan.go @@ -191,7 +191,7 @@ wolfictl scan package1 package2 --remote for _, dir := range p.advisoriesRepoDirs { advisoryFsys := rwos.DirFS(dir) - index, err := v2.NewIndex(advisoryFsys) + index, err := v2.NewIndex(cmd.Context(), advisoryFsys) if err != nil { return fmt.Errorf("unable to index advisory configs for directory %q: %w", dir, err) } @@ -306,7 +306,7 @@ func scanEverything(ctx context.Context, p *scanParams, inputs []string, advisor } // Get the SBOM of the APK - apkSBOM, err := p.generateSBOM(inputFile) + apkSBOM, err := p.generateSBOM(ctx, inputFile) if err != nil { return fmt.Errorf("failed to generate SBOM: %w", err) } @@ -448,16 +448,16 @@ func (p *scanParams) doScanCommandForSingleInput( return result, nil } -func (p *scanParams) generateSBOM(f *os.File) (*sbomSyft.SBOM, error) { +func (p *scanParams) generateSBOM(ctx context.Context, f *os.File) (*sbomSyft.SBOM, error) { if p.sbomInput { return sbom.FromSyftJSON(f) } if p.disableSBOMCache { - return sbom.Generate(f.Name(), f, p.distro) + return sbom.Generate(ctx, f.Name(), f, p.distro) } - return sbom.CachedGenerate(f.Name(), f, p.distro) + return sbom.CachedGenerate(ctx, f.Name(), f, p.distro) } // resolveInputFilePathsFromBuildLog takes the given path to a Melange build log diff --git a/pkg/cli/update.go b/pkg/cli/update.go index 6c36b69f..8f6463b8 100644 --- a/pkg/cli/update.go +++ b/pkg/cli/update.go @@ -56,7 +56,7 @@ func cmdUpdate() *cobra.Command { } func (o options) UpdateCmd(ctx context.Context, repoURI string) error { - updateContext := update.New() + updateContext := update.New(ctx) if !o.dryRun { if _, err := (ghTokenSource{}).Token(); err != nil { diff --git a/pkg/cli/update_package.go b/pkg/cli/update_package.go index 98cd03a8..7bfe4b3f 100644 --- a/pkg/cli/update_package.go +++ b/pkg/cli/update_package.go @@ -1,12 +1,22 @@ package cli import ( + "log" + "os" + "time" + + "github.com/google/go-github/v58/github" "github.com/spf13/cobra" + "github.com/wolfi-dev/wolfictl/pkg/http" "github.com/wolfi-dev/wolfictl/pkg/update" + "golang.org/x/oauth2" + "golang.org/x/time/rate" ) func Package() *cobra.Command { - o := update.NewPackageOptions() + o := update.PackageOptions{ + Logger: log.New(log.Writer(), "wolfictl update: ", log.LstdFlags|log.Lmsgprefix), + } cmd := &cobra.Command{ Use: "package", @@ -21,6 +31,19 @@ func Package() *cobra.Command { } } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ) + + ratelimit := &http.RLHTTPClient{ + Client: oauth2.NewClient(cmd.Context(), ts), + + // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits + Ratelimiter: rate.NewLimiter(rate.Every(3*time.Second), 1), + } + + o.GithubClient = github.NewClient(ratelimit.Client) + o.PackageName = args[0] return o.UpdatePackageCmd(cmd.Context()) }, diff --git a/pkg/configs/advisory/v1/advisory.go b/pkg/configs/advisory/v1/advisory.go index a2221eef..35b606fa 100644 --- a/pkg/configs/advisory/v1/advisory.go +++ b/pkg/configs/advisory/v1/advisory.go @@ -1,6 +1,7 @@ package v1 import ( + "context" "io" "io/fs" "time" @@ -11,16 +12,16 @@ import ( "gopkg.in/yaml.v3" ) -func NewIndex(fsys rwfs.FS) (*configs.Index[Document], error) { - return configs.NewIndex[Document](fsys, newConfigurationDecodeFunc(fsys)) +func NewIndex(ctx context.Context, fsys rwfs.FS) (*configs.Index[Document], error) { + return configs.NewIndex[Document](ctx, fsys, newConfigurationDecodeFunc(fsys)) } -func NewIndexFromPaths(fsys rwfs.FS, paths ...string) (*configs.Index[Document], error) { - return configs.NewIndexFromPaths[Document](fsys, newConfigurationDecodeFunc(fsys), paths...) +func NewIndexFromPaths(ctx context.Context, fsys rwfs.FS, paths ...string) (*configs.Index[Document], error) { + return configs.NewIndexFromPaths[Document](ctx, fsys, newConfigurationDecodeFunc(fsys), paths...) } -func newConfigurationDecodeFunc(fsys fs.FS) func(string) (*Document, error) { - return func(path string) (*Document, error) { +func newConfigurationDecodeFunc(fsys fs.FS) func(context.Context, string) (*Document, error) { + return func(ctx context.Context, path string) (*Document, error) { file, err := fsys.Open(path) if err != nil { return nil, err diff --git a/pkg/configs/advisory/v2/index.go b/pkg/configs/advisory/v2/index.go index bdf109c1..9f4f9391 100644 --- a/pkg/configs/advisory/v2/index.go +++ b/pkg/configs/advisory/v2/index.go @@ -1,22 +1,23 @@ package v2 import ( + "context" "io/fs" "github.com/wolfi-dev/wolfictl/pkg/configs" "github.com/wolfi-dev/wolfictl/pkg/configs/rwfs" ) -func NewIndex(fsys rwfs.FS) (*configs.Index[Document], error) { - return configs.NewIndex[Document](fsys, newConfigurationDecodeFunc(fsys)) +func NewIndex(ctx context.Context, fsys rwfs.FS) (*configs.Index[Document], error) { + return configs.NewIndex[Document](ctx, fsys, newConfigurationDecodeFunc(fsys)) } -func NewIndexFromPaths(fsys rwfs.FS, paths ...string) (*configs.Index[Document], error) { - return configs.NewIndexFromPaths[Document](fsys, newConfigurationDecodeFunc(fsys), paths...) +func NewIndexFromPaths(ctx context.Context, fsys rwfs.FS, paths ...string) (*configs.Index[Document], error) { + return configs.NewIndexFromPaths[Document](ctx, fsys, newConfigurationDecodeFunc(fsys), paths...) } -func newConfigurationDecodeFunc(fsys fs.FS) func(string) (*Document, error) { - return func(path string) (*Document, error) { +func newConfigurationDecodeFunc(fsys fs.FS) func(context.Context, string) (*Document, error) { + return func(ctx context.Context, path string) (*Document, error) { file, err := fsys.Open(path) if err != nil { return nil, err diff --git a/pkg/configs/build/build.go b/pkg/configs/build/build.go index 7d1c5bf9..7b23526b 100644 --- a/pkg/configs/build/build.go +++ b/pkg/configs/build/build.go @@ -11,17 +11,17 @@ import ( "github.com/wolfi-dev/wolfictl/pkg/internal" ) -func NewIndex(fsys rwfs.FS) (*configs.Index[config.Configuration], error) { - return configs.NewIndex[config.Configuration](fsys, newConfigurationDecodeFunc(fsys)) +func NewIndex(ctx context.Context, fsys rwfs.FS) (*configs.Index[config.Configuration], error) { + return configs.NewIndex[config.Configuration](ctx, fsys, newConfigurationDecodeFunc(fsys)) } -func NewIndexFromPaths(fsys rwfs.FS, paths ...string) (*configs.Index[config.Configuration], error) { - return configs.NewIndexFromPaths[config.Configuration](fsys, newConfigurationDecodeFunc(fsys), paths...) +func NewIndexFromPaths(ctx context.Context, fsys rwfs.FS, paths ...string) (*configs.Index[config.Configuration], error) { + return configs.NewIndexFromPaths[config.Configuration](ctx, fsys, newConfigurationDecodeFunc(fsys), paths...) } -func newConfigurationDecodeFunc(fsys fs.FS) func(string) (*config.Configuration, error) { - return func(path string) (*config.Configuration, error) { - ctx := clog.WithLogger(context.Background(), clog.NewLogger(internal.NopLogger())) +func newConfigurationDecodeFunc(fsys fs.FS) func(context.Context, string) (*config.Configuration, error) { + return func(ctx context.Context, path string) (*config.Configuration, error) { + ctx = clog.WithLogger(ctx, clog.NewLogger(internal.NopLogger())) return config.ParseConfiguration(ctx, path, config.WithFS(fsys)) } } diff --git a/pkg/configs/build/build_test.go b/pkg/configs/build/build_test.go index 29544994..d9bde0a6 100644 --- a/pkg/configs/build/build_test.go +++ b/pkg/configs/build/build_test.go @@ -1,6 +1,7 @@ package build import ( + "context" "testing" "chainguard.dev/melange/pkg/config" @@ -20,7 +21,7 @@ func TestBuildConfigsIndex(t *testing.T) { ) require.NoError(t, err) - index, err := NewIndexFromPaths(fsys, testfiles...) + index, err := NewIndexFromPaths(context.Background(), fsys, testfiles...) require.NoError(t, err) const modifiedPackageName = "foobar" @@ -32,7 +33,7 @@ func TestBuildConfigsIndex(t *testing.T) { }) s := index.Select().WhereName("cheese") - err = s.Update(packageSectionUpdater) + err = s.Update(context.Background(), packageSectionUpdater) require.NoError(t, err) if diff := fsys.DiffAll(); diff != "" { @@ -52,7 +53,7 @@ func TestBuildConfigsIndexUpdatePipelines(t *testing.T) { ) require.NoError(t, err) - index, err := NewIndexFromPaths(fsys, testfiles...) + index, err := NewIndexFromPaths(context.Background(), fsys, testfiles...) require.NoError(t, err) pipelineSectionUpdater := NewPipelineSectionUpdater(func(cfg config.Configuration) ([]config.Pipeline, error) { @@ -62,7 +63,7 @@ func TestBuildConfigsIndexUpdatePipelines(t *testing.T) { }) s := index.Select().WhereName("blah") - err = s.Update(pipelineSectionUpdater) + err = s.Update(context.Background(), pipelineSectionUpdater) require.NoError(t, err) if diff := fsys.DiffAll(); diff != "" { @@ -81,7 +82,7 @@ func TestBuildConfigsIndexUpdateSubpackages(t *testing.T) { ) require.NoError(t, err) - index, err := NewIndexFromPaths(fsys, testfiles...) + index, err := NewIndexFromPaths(context.Background(), fsys, testfiles...) require.NoError(t, err) const modifiedSubPackageName = "foobar" @@ -92,7 +93,7 @@ func TestBuildConfigsIndexUpdateSubpackages(t *testing.T) { }) s := index.Select().WhereName("blah") - err = s.Update(subpackagesSectionUpdater) + err = s.Update(context.Background(), subpackagesSectionUpdater) require.NoError(t, err) if diff := fsys.DiffAll(); diff != "" { diff --git a/pkg/configs/build/selection_test.go b/pkg/configs/build/selection_test.go index d69c0388..f252dc8b 100644 --- a/pkg/configs/build/selection_test.go +++ b/pkg/configs/build/selection_test.go @@ -1,6 +1,7 @@ package build import ( + "context" "testing" "chainguard.dev/melange/pkg/config" @@ -13,7 +14,7 @@ import ( func TestSelection(t *testing.T) { fsys := rwos.DirFS("testdata/index-1") - index, err := configs.NewIndex[config.Configuration](fsys, newConfigurationDecodeFunc(fsys)) + index, err := configs.NewIndex[config.Configuration](context.Background(), fsys, newConfigurationDecodeFunc(fsys)) require.NoError(t, err) s := index.Select() diff --git a/pkg/configs/entry.go b/pkg/configs/entry.go index 939e4a5f..b09ab775 100644 --- a/pkg/configs/entry.go +++ b/pkg/configs/entry.go @@ -1,6 +1,7 @@ package configs import ( + "context" "fmt" "gopkg.in/yaml.v3" @@ -21,7 +22,7 @@ type Entry[T Configuration] interface { getPath() string // Update applies the given entryUpdater to the entry. - Update(updater EntryUpdater[T]) error + Update(ctx context.Context, updater EntryUpdater[T]) error // Configuration returns the entry as a decoded configuration. Configuration() *T @@ -51,8 +52,8 @@ func (e entry[T]) getPath() string { return e.path } -func (e entry[T]) Update(updater EntryUpdater[T]) error { - err := e.getIndex().update(e, updater) +func (e entry[T]) Update(ctx context.Context, updater EntryUpdater[T]) error { + err := e.getIndex().update(ctx, e, updater) if err != nil { return fmt.Errorf("unable to update %q: %w", e.id(), err) } diff --git a/pkg/configs/index.go b/pkg/configs/index.go index 7ffc391b..67834188 100644 --- a/pkg/configs/index.go +++ b/pkg/configs/index.go @@ -1,6 +1,7 @@ package configs import ( + "context" "errors" "fmt" "io/fs" @@ -29,7 +30,7 @@ type Index[T Configuration] struct { paths []string yamlRoots []*yaml.Node cfgs []T - cfgDecodeFunc func(string) (*T, error) + cfgDecodeFunc func(context.Context, string) (*T, error) byID map[string]int byName map[string]int byPath map[string]int @@ -39,7 +40,7 @@ type Index[T Configuration] struct { // filesystem. The provided cfgDecodeFunc should take a path to a YAML file in a // fs.FS, decode the file to type T, and return a reference the "type T" data, // or an error if there was a problem. -func NewIndex[T Configuration](fsys rwfs.FS, cfgDecodeFunc func(string) (*T, error)) (*Index[T], error) { +func NewIndex[T Configuration](ctx context.Context, fsys rwfs.FS, cfgDecodeFunc func(context.Context, string) (*T, error)) (*Index[T], error) { if cfgDecodeFunc == nil { return nil, errors.New("must supply a cfgDecodeFunc") } @@ -68,7 +69,7 @@ func NewIndex[T Configuration](fsys rwfs.FS, cfgDecodeFunc func(string) (*T, err return nil } - err = index.processAndAdd(path) + err = index.processAndAdd(ctx, path) if err != nil { return err } @@ -86,12 +87,12 @@ func NewIndex[T Configuration](fsys rwfs.FS, cfgDecodeFunc func(string) (*T, err // paths. The provided cfgDecodeFunc should take a path to a YAML file in a // fs.FS, decode the file to type T, and return a reference the "type T" data, // or an error if there was a problem. -func NewIndexFromPaths[T Configuration](fsys rwfs.FS, cfgDecodeFunc func(string) (*T, error), paths ...string) (*Index[T], error) { +func NewIndexFromPaths[T Configuration](ctx context.Context, fsys rwfs.FS, cfgDecodeFunc func(context.Context, string) (*T, error), paths ...string) (*Index[T], error) { index := newIndex(cfgDecodeFunc) index.fsys = fsys for _, filepath := range paths { - err := index.processAndAdd(filepath) + err := index.processAndAdd(ctx, filepath) if err != nil { return nil, err } @@ -103,7 +104,7 @@ func NewIndexFromPaths[T Configuration](fsys rwfs.FS, cfgDecodeFunc func(string) return &index, nil } -func newIndex[T Configuration](cfgDecodeFunc func(string) (*T, error)) Index[T] { +func newIndex[T Configuration](cfgDecodeFunc func(context.Context, string) (*T, error)) Index[T] { index := Index[T]{} index.cfgDecodeFunc = cfgDecodeFunc index.byID = make(map[string]int) @@ -131,7 +132,7 @@ func (i *Index[T]) Select() Selection[T] { // Create creates a new configuration file at the given path, with the given // cfg. The new configuration is automatically added to the Index. -func (i *Index[T]) Create(filepath string, cfg T) error { +func (i *Index[T]) Create(ctx context.Context, filepath string, cfg T) error { file, err := i.fsys.Create(filepath) if err != nil { return err @@ -148,7 +149,7 @@ func (i *Index[T]) Create(filepath string, cfg T) error { return err } - err = i.processAndAdd(filepath) + err = i.processAndAdd(ctx, filepath) if err != nil { return err } @@ -195,14 +196,14 @@ func (i *Index[T]) format(filepath string) error { } // update updates the given entry in the index using the provided EntryUpdater. -func (i *Index[T]) update(entry Entry[T], entryUpdater EntryUpdater[T]) error { +func (i *Index[T]) update(ctx context.Context, entry Entry[T], entryUpdater EntryUpdater[T]) error { err := entryUpdater(i, entry) if err != nil { return err } id := entry.id() - err = i.processAndUpdate(entry.getPath(), i.byID[id]) + err = i.processAndUpdate(ctx, entry.getPath(), i.byID[id]) if err != nil { return fmt.Errorf("unable to process and update index entry for %q: %w", id, err) } @@ -213,8 +214,8 @@ func (i *Index[T]) update(entry Entry[T], entryUpdater EntryUpdater[T]) error { // processAndAdd decodes the configuration file at the given path into both a // YAML AST and a configuration (type T), and it then adds a new entry to the // Index. -func (i *Index[T]) processAndAdd(filepath string) error { - entry, err := i.process(filepath) +func (i *Index[T]) processAndAdd(ctx context.Context, filepath string) error { + entry, err := i.process(ctx, filepath) if err != nil { return err } @@ -226,8 +227,8 @@ func (i *Index[T]) processAndAdd(filepath string) error { return nil } -func (i *Index[T]) processAndUpdate(filepath string, entryIndex int) error { - entry, err := i.process(filepath) +func (i *Index[T]) processAndUpdate(ctx context.Context, filepath string, entryIndex int) error { + entry, err := i.process(ctx, filepath) if err != nil { return err } @@ -242,7 +243,7 @@ func (i *Index[T]) processAndUpdate(filepath string, entryIndex int) error { return nil } -func (i *Index[T]) process(filepath string) (*entry[T], error) { +func (i *Index[T]) process(ctx context.Context, filepath string) (*entry[T], error) { // TODO: for the follow operations, consider noting the error and moving on, rather than stopping the indexing. f, err := i.fsys.Open(filepath) @@ -256,7 +257,7 @@ func (i *Index[T]) process(filepath string) (*entry[T], error) { return nil, fmt.Errorf("unable to decode YAML at %q: %w", filepath, err) } - cfg, err := i.cfgDecodeFunc(filepath) + cfg, err := i.cfgDecodeFunc(ctx, filepath) if err != nil { return nil, fmt.Errorf("unable to parse configuration at %q: %w", filepath, err) } diff --git a/pkg/configs/index_test.go b/pkg/configs/index_test.go index 89af3d40..43f085dc 100644 --- a/pkg/configs/index_test.go +++ b/pkg/configs/index_test.go @@ -14,7 +14,7 @@ func TestNewIndex(t *testing.T) { ctx := context.Background() fsys := rwos.DirFS("testdata/index-1") - index, err := NewIndex[config.Configuration](fsys, func(path string) (*config.Configuration, error) { + index, err := NewIndex[config.Configuration](ctx, fsys, func(ctx context.Context, path string) (*config.Configuration, error) { return config.ParseConfiguration(ctx, path, config.WithFS(fsys)) }) require.NoError(t, err) diff --git a/pkg/configs/selection.go b/pkg/configs/selection.go index 724219f7..d2d412c0 100644 --- a/pkg/configs/selection.go +++ b/pkg/configs/selection.go @@ -1,6 +1,7 @@ package configs import ( + "context" "errors" "github.com/samber/lo" @@ -59,9 +60,9 @@ func (s Selection[T]) Len() int { // Update applies the given entryUpdater to all entries currently in the // Selection. -func (s Selection[T]) Update(entryUpdater EntryUpdater[T]) error { +func (s Selection[T]) Update(ctx context.Context, entryUpdater EntryUpdater[T]) error { for _, e := range s.entries { - err := e.Update(entryUpdater) + err := e.Update(ctx, entryUpdater) if err != nil { if errors.Is(err, ErrSkip) { continue diff --git a/pkg/dag/graph.go b/pkg/dag/graph.go index 6e7b3731..74975eaf 100644 --- a/pkg/dag/graph.go +++ b/pkg/dag/graph.go @@ -87,7 +87,7 @@ func NewGraph(ctx context.Context, pkgs *Packages, options ...GraphOptions) (*Gr localRepo := pkgs.Repository(opts.arch) localRepoSource := localRepo.Source() - localOnlyResolver := apk.NewPkgResolver(context.TODO(), []apk.NamedIndex{localRepo}) + localOnlyResolver := apk.NewPkgResolver(ctx, []apk.NamedIndex{localRepo}) g.resolvers[localRepoSource] = localOnlyResolver // the order of adding packages is quite important: @@ -101,7 +101,7 @@ func NewGraph(ctx context.Context, pkgs *Packages, options ...GraphOptions) (*Gr continue } // add the origin package as its own resolver, so that the subpackage can resolve to it - g.resolvers[c.String()] = singlePackageResolver(c, opts.arch) + g.resolvers[c.String()] = singlePackageResolver(ctx, c, opts.arch) for i := range c.Subpackages { subpkg := pkgs.Config(c.Subpackages[i].Name, false) for _, subpkgVersion := range subpkg { @@ -124,7 +124,8 @@ func NewGraph(ctx context.Context, pkgs *Packages, options ...GraphOptions) (*Gr for _, c := range pkgs.Packages() { // add packages from the runtime dependencies - resolverKey, err := g.addResolverForRepos(opts.arch, + resolverKey, err := g.addResolverForRepos(ctx, + opts.arch, localRepo, indexes, keys, @@ -144,7 +145,8 @@ func NewGraph(ctx context.Context, pkgs *Packages, options ...GraphOptions) (*Gr for _, c := range pkgs.Packages() { // add packages from the buildtime dependencies - resolverKey, err := g.addResolverForRepos(opts.arch, + resolverKey, err := g.addResolverForRepos(ctx, + opts.arch, localRepo, indexes, keys, @@ -172,7 +174,7 @@ func NewGraph(ctx context.Context, pkgs *Packages, options ...GraphOptions) (*Gr // addResolverForRepos adds a resolver for a specific set of repositories, // returning the resolverKey to look it up. // Adds the local repository if provided. -func (g *Graph) addResolverForRepos(arch string, localRepo apk.NamedIndex, indexes map[string]apk.NamedIndex, allKeys map[string][]byte, addRepos, addKeys []string) (resolverKey string, err error) { +func (g *Graph) addResolverForRepos(ctx context.Context, arch string, localRepo apk.NamedIndex, indexes map[string]apk.NamedIndex, allKeys map[string][]byte, addRepos, addKeys []string) (resolverKey string, err error) { var ( repos []string lookupRepos = []apk.NamedIndex{} @@ -205,7 +207,7 @@ func (g *Graph) addResolverForRepos(arch string, localRepo apk.NamedIndex, index } } if len(repos) > 0 { - loadedRepos, err := apk.GetRepositoryIndexes(context.TODO(), repos, allKeys, arch) + loadedRepos, err := apk.GetRepositoryIndexes(ctx, repos, allKeys, arch) if err != nil { return "", err } @@ -234,7 +236,7 @@ func (g *Graph) addResolverForRepos(arch string, localRepo apk.NamedIndex, index // add packages from build-time, as they could be local only, or might have upstream if _, ok := g.resolvers[resolverKey]; !ok { - g.resolvers[resolverKey] = apk.NewPkgResolver(context.TODO(), lookupRepos) + g.resolvers[resolverKey] = apk.NewPkgResolver(ctx, lookupRepos) } return resolverKey, nil } @@ -858,7 +860,7 @@ func getKeyMaterial(key string) ([]byte, error) { return b, nil } -func singlePackageResolver(pkg *Configuration, arch string) *apk.PkgResolver { +func singlePackageResolver(ctx context.Context, pkg *Configuration, arch string) *apk.PkgResolver { repo := apk.NewRepositoryFromComponents(Local, "latest", "", arch) packages := []*apk.Package{ { @@ -879,5 +881,5 @@ func singlePackageResolver(pkg *Configuration, arch string) *apk.PkgResolver { Packages: packages, } idx := apk.NewNamedRepositoryWithIndex("", repo.WithIndex(index)) - return apk.NewPkgResolver(context.TODO(), []apk.NamedIndex{idx}) + return apk.NewPkgResolver(ctx, []apk.NamedIndex{idx}) } diff --git a/pkg/gh/pull_request.go b/pkg/gh/pull_request.go index 061fdedd..8452e097 100644 --- a/pkg/gh/pull_request.go +++ b/pkg/gh/pull_request.go @@ -20,7 +20,7 @@ type GetPullRequest struct { } // OpenPullRequest opens a pull request on GitHub -func (o GitOptions) OpenPullRequest(pr *NewPullRequest) (*github.PullRequest, error) { +func (o GitOptions) OpenPullRequest(ctx context.Context, pr *NewPullRequest) (*github.PullRequest, error) { // Configure pull request options that the GitHub client accepts when making calls to open new pull requests newPR := &github.NewPullRequest{ Title: github.String(pr.Title), @@ -31,7 +31,7 @@ func (o GitOptions) OpenPullRequest(pr *NewPullRequest) (*github.PullRequest, er var githubPR *github.PullRequest err := o.handleRateLimit(func() (*github.Response, error) { - createdPR, resp, err := o.GithubClient.PullRequests.Create(context.Background(), pr.Owner, pr.RepoName, newPR) + createdPR, resp, err := o.GithubClient.PullRequests.Create(ctx, pr.Owner, pr.RepoName, newPR) githubPR = createdPR return resp, err }) diff --git a/pkg/gh/release.go b/pkg/gh/release.go index c479853b..b979ab37 100644 --- a/pkg/gh/release.go +++ b/pkg/gh/release.go @@ -4,14 +4,8 @@ import ( "context" "fmt" "log" - "os" "strconv" "strings" - "time" - - http2 "github.com/wolfi-dev/wolfictl/pkg/http" - "golang.org/x/oauth2" - "golang.org/x/time/rate" "github.com/google/go-github/v58/github" @@ -31,28 +25,10 @@ type ReleaseOptions struct { Dir string } -func NewReleaseOptions() ReleaseOptions { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, - ) - - ratelimit := &http2.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ts), - - // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits - Ratelimiter: rate.NewLimiter(rate.Every(3*time.Second), 1), - } - - return ReleaseOptions{ - GithubClient: github.NewClient(ratelimit.Client), - Logger: log.New(log.Writer(), "wolfictl gh release: ", log.LstdFlags|log.Lmsgprefix), - } -} - const defaultStartVersion = "v0.0.0" // Release will create a new GitHub release -func (o ReleaseOptions) Release() error { +func (o ReleaseOptions) Release(ctx context.Context) error { // get the latest git tag current, err := wolfigit.GetVersionFromTag(o.Dir, 1) if current == nil || err != nil { @@ -83,7 +59,7 @@ func (o ReleaseOptions) Release() error { } // create the GitHub release - err = o.createGitHubRelease(next.Original()) + err = o.createGitHubRelease(ctx, next.Original()) if err != nil { return err } @@ -145,7 +121,7 @@ func (o ReleaseOptions) bumpReleaseVersion(current *version.Version) (*version.V } // createGitHubRelease creates a new release on GitHub -func (o ReleaseOptions) createGitHubRelease(v string) error { +func (o ReleaseOptions) createGitHubRelease(ctx context.Context, v string) error { repo, err := git.PlainOpen(o.Dir) if err != nil { return err @@ -161,7 +137,6 @@ func (o ReleaseOptions) createGitHubRelease(v string) error { TagName: github.String(v), } - ctx := context.Background() release, _, err := o.GithubClient.Repositories.CreateRelease(ctx, gitURL.Organisation, gitURL.Name, input) if err != nil { return err @@ -171,8 +146,7 @@ func (o ReleaseOptions) createGitHubRelease(v string) error { return nil } -func (o ReleaseOptions) GetReleaseURL(owner, repoName, v string) (string, error) { - ctx := context.Background() +func (o ReleaseOptions) GetReleaseURL(ctx context.Context, owner, repoName, v string) (string, error) { release, _, err := o.GithubClient.Repositories.GetReleaseByTag(ctx, owner, repoName, v) if err != nil { return "", fmt.Errorf("failed to get github release for %s/%s tag %s: %w", owner, repoName, v, err) diff --git a/pkg/http/http.go b/pkg/http/http.go index 679846ca..5a645ed7 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -1,7 +1,6 @@ package http import ( - "context" "net/http" "golang.org/x/time/rate" @@ -16,8 +15,7 @@ type RLHTTPClient struct { // Do dispatches the HTTP request to the network func (c *RLHTTPClient) Do(req *http.Request) (*http.Response, error) { // Comment out the below 5 lines to turn off ratelimiting - ctx := context.Background() - err := c.Ratelimiter.Wait(ctx) // This is a blocking call. Honors the rate limit + err := c.Ratelimiter.Wait(req.Context()) // This is a blocking call. Honors the rate limit if err != nil { return nil, err } diff --git a/pkg/ls/ls_test.go b/pkg/ls/ls_test.go index 2ba3db2b..45610ddd 100644 --- a/pkg/ls/ls_test.go +++ b/pkg/ls/ls_test.go @@ -1,6 +1,7 @@ package ls import ( + "context" "testing" "chainguard.dev/melange/pkg/config" @@ -101,7 +102,7 @@ func TestList(t *testing.T) { indices := make([]*configs.Index[config.Configuration], 0, len(tt.distroDirs)) for _, dir := range tt.distroDirs { distroFsys := rwos.DirFS(dir) - buildCfgs, err := buildconfigs.NewIndex(distroFsys) + buildCfgs, err := buildconfigs.NewIndex(context.Background(), distroFsys) require.NoError(t, err) indices = append(indices, buildCfgs) } diff --git a/pkg/sbom/cache.go b/pkg/sbom/cache.go index 15965169..f26081d0 100644 --- a/pkg/sbom/cache.go +++ b/pkg/sbom/cache.go @@ -1,6 +1,7 @@ package sbom import ( + "context" "crypto/sha256" "fmt" "io" @@ -32,7 +33,7 @@ func cachedSBOMPath(inputFilePath string, f io.Reader) (string, error) { // if a generated SBOM is already available in the cache for the given APK, // CachedGenerate will return the cached SBOM immediately instead of generating // a new SBOM. -func CachedGenerate(inputFilePath string, f io.ReadSeeker, distroID string) (*sbom.SBOM, error) { +func CachedGenerate(ctx context.Context, inputFilePath string, f io.ReadSeeker, distroID string) (*sbom.SBOM, error) { // Check cache first cachedPath, err := cachedSBOMPath(inputFilePath, f) @@ -53,7 +54,7 @@ func CachedGenerate(inputFilePath string, f io.ReadSeeker, distroID string) (*sb return nil, fmt.Errorf("failed to seek to start of input file: %w", err) } - s, err := Generate(inputFilePath, f, distroID) + s, err := Generate(ctx, inputFilePath, f, distroID) if err != nil { return nil, fmt.Errorf("failed to generate SBOM: %w", err) } diff --git a/pkg/sbom/sbom.go b/pkg/sbom/sbom.go index d74ac1f4..6383d0cf 100644 --- a/pkg/sbom/sbom.go +++ b/pkg/sbom/sbom.go @@ -25,7 +25,7 @@ import ( ) // Generate creates an SBOM for the given APK file. -func Generate(inputFilePath string, f io.Reader, distroID string) (*sbom.SBOM, error) { +func Generate(ctx context.Context, inputFilePath string, f io.Reader, distroID string) (*sbom.SBOM, error) { // Create a temp directory to house the unpacked APK file tempDir, err := os.MkdirTemp("", "wolfictl-sbom-*") if err != nil { @@ -67,7 +67,7 @@ func Generate(inputFilePath string, f io.Reader, distroID string) (*sbom.SBOM, e ), ) - createdSBOM, err := syft.CreateSBOM(context.Background(), src, cfg) + createdSBOM, err := syft.CreateSBOM(ctx, src, cfg) if err != nil { return nil, fmt.Errorf("failed to create SBOM: %w", err) } diff --git a/pkg/scan/filter_test.go b/pkg/scan/filter_test.go index e78076a7..2745ef25 100644 --- a/pkg/scan/filter_test.go +++ b/pkg/scan/filter_test.go @@ -1,6 +1,7 @@ package scan import ( + "context" "path" "testing" @@ -371,7 +372,7 @@ func getSingleAdvisoriesIndex(t *testing.T) []*configs.Index[v2.Document] { t.Helper() advFS := rwos.DirFS(path.Join("testdata", "advisories")) - index, err := v2.NewIndex(advFS) + index, err := v2.NewIndex(context.Background(), advFS) if err != nil { t.Fatal(err) } @@ -389,7 +390,7 @@ func getMultipleAdvisoriesIndices(t *testing.T) []*configs.Index[v2.Document] { "advisories-additional", } { advFS := rwos.DirFS(path.Join("testdata", dir)) - index, err := v2.NewIndex(advFS) + index, err := v2.NewIndex(context.Background(), advFS) if err != nil { t.Fatal(err) } diff --git a/pkg/update/package.go b/pkg/update/package.go index 514022c9..0b7d5c24 100644 --- a/pkg/update/package.go +++ b/pkg/update/package.go @@ -10,7 +10,6 @@ import ( "path/filepath" "regexp" "strings" - "time" v2 "github.com/wolfi-dev/wolfictl/pkg/configs/advisory/v2" rwfsOS "github.com/wolfi-dev/wolfictl/pkg/configs/rwfs/os" @@ -24,10 +23,7 @@ import ( wolfigit "github.com/wolfi-dev/wolfictl/pkg/git" "github.com/go-git/go-git/v5" - "golang.org/x/oauth2" - "golang.org/x/time/rate" - wolfihttp "github.com/wolfi-dev/wolfictl/pkg/http" "github.com/wolfi-dev/wolfictl/pkg/melange" ) @@ -45,27 +41,6 @@ type PackageOptions struct { GithubClient *github.Client } -// NewPackageOptions initialise clients -func NewPackageOptions() PackageOptions { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, - ) - - ratelimit := &wolfihttp.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ts), - - // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits - Ratelimiter: rate.NewLimiter(rate.Every(3*time.Second), 1), - } - - options := PackageOptions{ - GithubClient: github.NewClient(ratelimit.Client), - Logger: log.New(log.Writer(), "wolfictl update: ", log.LstdFlags|log.Lmsgprefix), - } - - return options -} - func (o *PackageOptions) UpdatePackageCmd(ctx context.Context) error { // clone the melange config git repo into a temp folder so we can work with it tempDir, err := os.MkdirTemp("", "wolfictl") @@ -97,7 +72,7 @@ func (o *PackageOptions) UpdatePackageCmd(ctx context.Context) error { return fmt.Errorf("failed to get package config for package name %s: %w", o.PackageName, err) } - uo := New() + uo := New(ctx) uo.PackageConfigs = o.PackageConfig uo.DryRun = o.DryRun uo.PullRequestBaseBranch = o.PullRequestBaseBranch @@ -112,7 +87,7 @@ func (o *PackageOptions) UpdatePackageCmd(ctx context.Context) error { // optionally update advisories based on commit since the previous release if o.Advisories { - err := o.updateAdvisories(repo) + err := o.updateAdvisories(ctx, repo) if err != nil { return fmt.Errorf("failed to update advisories: %w", err) } @@ -135,7 +110,7 @@ func (o *PackageOptions) UpdatePackageCmd(ctx context.Context) error { } // if we are executing the update command in a git repository then check for CVE fixes -func (o *PackageOptions) updateAdvisories(repo *git.Repository) error { +func (o *PackageOptions) updateAdvisories(ctx context.Context, repo *git.Repository) error { currentDir, err := os.Getwd() if err != nil { return err @@ -188,7 +163,7 @@ func (o *PackageOptions) updateAdvisories(repo *git.Repository) error { // run the equivalent of `wolfictl advisory create ./foo.melange.yaml --vuln 'CVE-2022-31130' --status 'fixed' --fixed-version '7.5.17-r1'` for _, fixComment := range cveFixes { o.Logger.Printf("adding advisory for %s\n", fixComment) - err = o.createAdvisories(fixComment) + err = o.createAdvisories(ctx, fixComment) if err != nil { return fmt.Errorf("failed to create advisory for CVE list from commits between previous tag, %s: %w", strings.Join(cveFixes, " "), err) } @@ -240,12 +215,12 @@ func (o *PackageOptions) getFixesCVEList(dir string, previous *version.Version) return fixedCVEs, nil } -func (o *PackageOptions) createAdvisories(vuln string) error { +func (o *PackageOptions) createAdvisories(ctx context.Context, vuln string) error { p := o.PackageConfig[o.PackageName] fullFilePath := filepath.Join(p.Dir, p.Filename) fsys := rwfsOS.DirFS("/") - advisoryCfgs, err := v2.NewIndexFromPaths(fsys, fullFilePath) + advisoryCfgs, err := v2.NewIndexFromPaths(ctx, fsys, fullFilePath) if err != nil { return fmt.Errorf("failed to get new index for package %s config file %s: %w", o.PackageName, p.Filename, err) } @@ -255,7 +230,7 @@ func (o *PackageOptions) createAdvisories(vuln string) error { return err } - err = advisory.Create(req, advisory.CreateOptions{ + err = advisory.Create(ctx, req, advisory.CreateOptions{ AdvisoryDocs: advisoryCfgs, }) if err != nil { diff --git a/pkg/update/update.go b/pkg/update/update.go index 459c5468..7ce7b0b1 100644 --- a/pkg/update/update.go +++ b/pkg/update/update.go @@ -78,7 +78,7 @@ const ( ) // New initialise including a map of existing wolfios packages -func New() Options { +func New(ctx context.Context) Options { ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, ) @@ -105,7 +105,7 @@ func New() Options { }, GitHubHTTPClient: &http2.RLHTTPClient{ - Client: oauth2.NewClient(context.Background(), ts), + Client: oauth2.NewClient(ctx, ts), // 1 request every (n) second(s) to avoid DOS'ing server. https://docs.github.com/en/rest/guides/best-practices-for-integrators?apiVersion=2022-11-28#dealing-with-secondary-rate-limits Ratelimiter: rate.NewLimiter(rate.Every(5*time.Second), 1), @@ -165,7 +165,7 @@ func (o *Options) Update(ctx context.Context) error { } // skip packages for which we already have an open issue or pull request - err = o.removeExistingUpdates(repo) + err = o.removeExistingUpdates(ctx, repo) if err != nil { return fmt.Errorf("failed to get package updates: %w", err) } @@ -198,7 +198,7 @@ func (o *Options) Update(ctx context.Context) error { continue } if o.CreateIssues { - issueURL, err := o.createErrorMessageIssue(repo, k, message) + issueURL, err := o.createErrorMessageIssue(ctx, repo, k, message) if err != nil { return err } @@ -358,7 +358,7 @@ func (o *Options) updateGitPackage(ctx context.Context, repo *git.Repository, pa // if manual update create an issue rather than a pull request if pc.Config.Update.Manual { - return o.createNewVersionIssue(repo, packageName, newVersion) + return o.createNewVersionIssue(ctx, repo, packageName, newVersion) } worktree, err := repo.Worktree() @@ -459,7 +459,7 @@ func (o *Options) updateGitPackage(ctx context.Context, repo *git.Repository, pa // if we're not running in batch mode, lets commit and PR each change if !o.DryRun { - pr, err := o.proposeChanges(repo, ref, packageName, newVersion) + pr, err := o.proposeChanges(ctx, repo, ref, packageName, newVersion) if err != nil { return fmt.Sprintf("failed to propose changes: %s", err.Error()), nil } @@ -600,7 +600,7 @@ func (o *Options) createBranch(repo *git.Repository) (plumbing.ReferenceName, er } // commits package update changes and creates a pull request -func (o *Options) proposeChanges(repo *git.Repository, ref plumbing.ReferenceName, packageName string, newVersion NewVersionResults) (string, error) { +func (o *Options) proposeChanges(ctx context.Context, repo *git.Repository, ref plumbing.ReferenceName, packageName string, newVersion NewVersionResults) (string, error) { gitURL, err := wgit.GetRemoteURL(repo) if err != nil { return "", fmt.Errorf("failed to find git origin URL: %w", err) @@ -670,24 +670,24 @@ func (o *Options) proposeChanges(repo *git.Repository, ref plumbing.ReferenceNam } // create the pull request - pr, err := gitOpts.OpenPullRequest(newPR) + pr, err := gitOpts.OpenPullRequest(ctx, newPR) prLink := pr.GetHTMLURL() if err != nil { return "", fmt.Errorf("failed to create pull request: %w", err) } - err = gitOpts.LabelIssue(context.Background(), newPR.Owner, newPR.RepoName, *pr.Number, &o.IssueLabels) + err = gitOpts.LabelIssue(ctx, newPR.Owner, newPR.RepoName, *pr.Number, &o.IssueLabels) if err != nil { log.Printf("Failed to apply labels [%s] to PR #%d", strings.Join(o.IssueLabels, ","), pr.Number) } if newVersion.ReplaceExistingPRNumber != 0 { - err = gitOpts.ClosePullRequest(context.Background(), gitURL.Organisation, gitURL.Name, newVersion.ReplaceExistingPRNumber) + err = gitOpts.ClosePullRequest(ctx, gitURL.Organisation, gitURL.Name, newVersion.ReplaceExistingPRNumber) if err != nil { return "", fmt.Errorf("failed to close pull request: %d: %w", newVersion.ReplaceExistingPRNumber, err) } // comment on the closed PR the new pull request link which supersedes it comment := fmt.Sprintf("superseded by %s", prLink) - _, err = gitOpts.CommentIssue(context.Background(), gitURL.Organisation, gitURL.Name, comment, newVersion.ReplaceExistingPRNumber) + _, err = gitOpts.CommentIssue(ctx, gitURL.Organisation, gitURL.Name, comment, newVersion.ReplaceExistingPRNumber) if err != nil { return "", fmt.Errorf("failed to comment pull request: %d: %w", newVersion.ReplaceExistingPRNumber, err) } @@ -732,7 +732,7 @@ func (o *Options) commitChanges(repo *git.Repository, packageName, latestVersion return nil } -func (o *Options) createErrorMessageIssue(repo *git.Repository, packageName, message string) (string, error) { +func (o *Options) createErrorMessageIssue(ctx context.Context, repo *git.Repository, packageName, message string) (string, error) { gitURL, err := wgit.GetRemoteURL(repo) if err != nil { return "", fmt.Errorf("failed to find git origin URL: %w", err) @@ -753,24 +753,24 @@ func (o *Options) createErrorMessageIssue(repo *git.Repository, packageName, mes Title: gh.GetErrorIssueTitle(bot, packageName), Labels: o.IssueLabels, } - existingIssue, err := gitOpts.CheckExistingIssue(context.Background(), i) + existingIssue, err := gitOpts.CheckExistingIssue(ctx, i) if err != nil { return "", err } if existingIssue > 0 { - exists, err := gitOpts.HasExistingComment(context.Background(), i, existingIssue, message) + exists, err := gitOpts.HasExistingComment(ctx, i, existingIssue, message) if exists { return fmt.Sprintf("existing issue %d already exists for error message: %s", existingIssue, message), err } // if this is a new error add a new comment - return gitOpts.CommentIssue(context.Background(), gitURL.Organisation, gitURL.Name, message, existingIssue) + return gitOpts.CommentIssue(ctx, gitURL.Organisation, gitURL.Name, message, existingIssue) } - return gitOpts.OpenIssue(context.Background(), i) + return gitOpts.OpenIssue(ctx, i) } -func (o *Options) createNewVersionIssue(repo *git.Repository, packageName string, version NewVersionResults) (string, error) { +func (o *Options) createNewVersionIssue(ctx context.Context, repo *git.Repository, packageName string, version NewVersionResults) (string, error) { gitURL, err := wgit.GetRemoteURL(repo) if err != nil { return "", fmt.Errorf("failed to find git origin URL: %w", err) @@ -791,7 +791,7 @@ func (o *Options) createNewVersionIssue(repo *git.Repository, packageName string Labels: o.IssueLabels, } - existingIssues, err := gitOpts.ListIssues(context.Background(), gitURL.Organisation, gitURL.Name, "open") + existingIssues, err := gitOpts.ListIssues(ctx, gitURL.Organisation, gitURL.Name, "open") if err != nil { return "", err } @@ -804,7 +804,7 @@ func (o *Options) createNewVersionIssue(repo *git.Repository, packageName string } // create a new issue - issueLink, err := gitOpts.OpenIssue(context.Background(), i) + issueLink, err := gitOpts.OpenIssue(ctx, i) if err != nil { return "", err } @@ -813,7 +813,7 @@ func (o *Options) createNewVersionIssue(repo *git.Repository, packageName string // if there's an existing issue with the same package but older version then close it for _, issue := range existingIssues { if strings.HasPrefix(*issue.Title, packageName+"/") { - err = gitOpts.CloseIssue(context.Background(), gitURL.Organisation, gitURL.Name, fmt.Sprintf("superseded by %s", issueLink), *issue.Number) + err = gitOpts.CloseIssue(ctx, gitURL.Organisation, gitURL.Name, fmt.Sprintf("superseded by %s", issueLink), *issue.Number) if err != nil { return "", err } @@ -885,7 +885,7 @@ func (o *Options) getPackagesToUpdate(latestVersions map[string]NewVersionResult // return updated map if an existing issue or pr for an older version should be closed // will also remove an update if we already have an open matching pull request or issue -func (o *Options) removeExistingUpdates(repo *git.Repository) error { +func (o *Options) removeExistingUpdates(ctx context.Context, repo *git.Repository) error { gitURL, err := wgit.GetRemoteURL(repo) if err != nil { return fmt.Errorf("failed to find git origin URL: %w", err) @@ -898,14 +898,14 @@ func (o *Options) removeExistingUpdates(repo *git.Repository) error { Logger: o.Logger, } - openPRs, err := gitOpts.ListPullRequests(context.Background(), gitURL.Organisation, gitURL.Name, "open") + openPRs, err := gitOpts.ListPullRequests(ctx, gitURL.Organisation, gitURL.Name, "open") if err != nil { return fmt.Errorf("failed to list open pull requests for %s/%s: %w", gitURL.Organisation, gitURL.Name, err) } o.removeExistingPullRequests(openPRs) - openIssues, err := gitOpts.ListIssues(context.Background(), gitURL.Organisation, gitURL.Name, "open") + openIssues, err := gitOpts.ListIssues(ctx, gitURL.Organisation, gitURL.Name, "open") if err != nil { return fmt.Errorf("failed to list open issues for %s/%s: %w", gitURL.Organisation, gitURL.Name, err) }