From 935f86239589c8286893ed635a9a40f04370fc9e Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Thu, 24 Oct 2024 19:33:59 -0700 Subject: [PATCH 1/5] Use searchable tokens for existing version check Filtering via @me assumes that the user is always the same when creating issues. This might not be true if a user runs this tool locally. This also breaks when the author is an app such as github-actions. As an alternative approach, GitHub will index for search any word within the issue body, even hidden within an HTML comment. We use base64 encoding to turn an upstream version such as "v5.72.1" into "djUuNzIuMQ==" which is a single word which is highly unlikely to appear in any other issue. Similarly, we'll include the word "pulumiupgradeproviderissue" in the HTML comment to allow listing all upgrade issues easily via search. We'll leave the existing "@me" based search in as a fallback until we can be confident that these tokens will be present in relevant issues. --- upgrade/steps_helpers.go | 73 ++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/upgrade/steps_helpers.go b/upgrade/steps_helpers.go index 6753471..12bd816 100644 --- a/upgrade/steps_helpers.go +++ b/upgrade/steps_helpers.go @@ -2,6 +2,7 @@ package upgrade import ( "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -677,40 +678,62 @@ var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ct upstreamProviderName := GetContext(ctx).UpstreamProviderName upstreamOrg := GetContext(ctx).UpstreamProviderOrg title := fmt.Sprintf("Upgrade %s to v%s", upstreamProviderName, version) + // Turn the version into a token that we can search for later. + versionToken := base64.RawStdEncoding.EncodeToString([]byte(version)) - searchIssues := stepv2.Cmd(ctx, "gh", "search", "issues", - title, - "--repo="+repoOrg+"/"+repoName, - "--json=title,number", - "--state=open", - "--author=@me", - ) + // Try to check if the issue already exists for the version via the token. + repoArg := fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName) + tokenIssues, err := searchIssues(ctx, repoArg, fmt.Sprintf("--search=%q", versionToken)) - var issues []struct { - Title string `json:"title"` - Number int `json:"number"` + if err != nil { + return err } - err := json.Unmarshal([]byte(searchIssues), &issues) + if len(tokenIssues) > 0 { + return nil + } + + // Fall back to checking if the issue exists by the title for the time being. + myIssues, err := searchIssues(ctx, repoArg, + fmt.Sprintf("--search=%q", title), + "--state=open", + "--author=@me") + if err != nil { - return fmt.Errorf("failed to unmarshal `gh search issues` output: %w", err) + return err } - // create new issue if none exist - createIssue := true + // check for exact title match from search results - for _, issue := range issues { + for _, issue := range myIssues { if issue.Title == title { - createIssue = false + return nil } } - if createIssue { - stepv2.Cmd(ctx, - "gh", "issue", "create", - "--repo="+repoOrg+"/"+repoName, - "--body=Release details: https://github.com/"+upstreamOrg+"/"+upstreamProviderName+"/releases/tag/v"+version, - "--title="+title, - "--label="+"kind/enhancement", - ) - } + // Hide some special searchable words in the issue body via an HTML comment to help us find + // this issue later, also without requiring labels to be set up. + hiddenBody := fmt.Sprintf("", versionToken) + + stepv2.Cmd(ctx, + "gh", "issue", "create", + "--repo="+repoOrg+"/"+repoName, + "--body=Release details: https://github.com/"+upstreamOrg+"/"+upstreamProviderName+"/releases/tag/v"+version+"\n\n"+hiddenBody, + "--title="+title, + "--label="+"kind/enhancement", + ) + return nil }) + +type issue struct { + Title string `json:"title"` + Number int `json:"number"` +} + +func searchIssues(ctx context.Context, args ...string) ([]issue, error) { + cmdArgs := []string{"issue", "list", "--json=title,number"} + cmdArgs = append(cmdArgs, args...) + issueList := stepv2.Cmd(ctx, "gh", cmdArgs...) + var issues []issue + err := json.Unmarshal([]byte(issueList), &issues) + return issues, err +} From d86ffb9af9944ee1e2a695b0893c02f889179e30 Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Mon, 28 Oct 2024 12:04:11 +0000 Subject: [PATCH 2/5] Simplify issue deduplication search List issues by "pulumiupgradeproviderissue" then match on full title. If not found, list by author then match on full title. - Remove base64 encoded version number weirdness. Just leave the easier to understand token. - This still requires the title to be exactly accurate to our expectations. --- upgrade/steps_helpers.go | 54 +++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/upgrade/steps_helpers.go b/upgrade/steps_helpers.go index 12bd816..71fd706 100644 --- a/upgrade/steps_helpers.go +++ b/upgrade/steps_helpers.go @@ -2,7 +2,6 @@ package upgrade import ( "context" - "encoding/base64" "encoding/json" "fmt" "os" @@ -672,46 +671,46 @@ var getExpectedTargetFromIssues = stepv2.Func11E("From Issues", func(ctx context }, nil }) +const pulumiupgradeproviderissue = "pulumiupgradeproviderissue" + // Create an issue in the provider repo that signals an upgrade var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ctx context.Context, repoOrg, repoName, version string) error { upstreamProviderName := GetContext(ctx).UpstreamProviderName upstreamOrg := GetContext(ctx).UpstreamProviderOrg title := fmt.Sprintf("Upgrade %s to v%s", upstreamProviderName, version) - // Turn the version into a token that we can search for later. - versionToken := base64.RawStdEncoding.EncodeToString([]byte(version)) - - // Try to check if the issue already exists for the version via the token. - repoArg := fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName) - tokenIssues, err := searchIssues(ctx, repoArg, fmt.Sprintf("--search=%q", versionToken)) + var found bool + var err error + // Search through existing "pulumiupgradeproviderissue" issues to see if we've already created one for this version. + found, err = issueExistsForVersion(ctx, title, + fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), + fmt.Sprintf("--search=%q", pulumiupgradeproviderissue), + "--state=open") if err != nil { return err } - if len(tokenIssues) > 0 { + if found { return nil } - // Fall back to checking if the issue exists by the title for the time being. - myIssues, err := searchIssues(ctx, repoArg, + // Fall back to searching through the issues from the current user. + found, err = issueExistsForVersion(ctx, title, + fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), fmt.Sprintf("--search=%q", title), "--state=open", "--author=@me") - if err != nil { return err } - - // check for exact title match from search results - for _, issue := range myIssues { - if issue.Title == title { - return nil - } + if found { + return nil } - // Hide some special searchable words in the issue body via an HTML comment to help us find - // this issue later, also without requiring labels to be set up. - hiddenBody := fmt.Sprintf("", versionToken) + // We've not found an appropriate existing issue, so we'll create a new one. + + // Hide searchable token in the issue body via an HTML comment to help us find this issue later without requiring labels to be set up. + hiddenBody := fmt.Sprintf("", pulumiupgradeproviderissue) stepv2.Cmd(ctx, "gh", "issue", "create", @@ -729,6 +728,21 @@ type issue struct { Number int `json:"number"` } +func issueExistsForVersion(ctx context.Context, title string, searchArgs ...string) (bool, error) { + issues, err := searchIssues(ctx, searchArgs...) + if err != nil { + return false, err + } + + // check for exact title match from search results + for _, issue := range issues { + if issue.Title == title { + return true, nil + } + } + return false, nil +} + func searchIssues(ctx context.Context, args ...string) ([]issue, error) { cmdArgs := []string{"issue", "list", "--json=title,number"} cmdArgs = append(cmdArgs, args...) From db67693671605a159de5803eb02db7058a5863f7 Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Mon, 28 Oct 2024 14:36:11 +0000 Subject: [PATCH 3/5] Add GITHUB_OUTPUT for issue_created Purpose: when running in CI, we only want to attempt the upgrade if we've got a new version to upgrade to. - Extract upgradeIssueExits function. - If GITHUB_OUTPUT is available, append issue_created variable. --- upgrade/steps_helpers.go | 62 ++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/upgrade/steps_helpers.go b/upgrade/steps_helpers.go index 71fd706..c6f4ac9 100644 --- a/upgrade/steps_helpers.go +++ b/upgrade/steps_helpers.go @@ -680,35 +680,28 @@ var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ct upstreamOrg := GetContext(ctx).UpstreamProviderOrg title := fmt.Sprintf("Upgrade %s to v%s", upstreamProviderName, version) - var found bool - var err error - // Search through existing "pulumiupgradeproviderissue" issues to see if we've already created one for this version. - found, err = issueExistsForVersion(ctx, title, - fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), - fmt.Sprintf("--search=%q", pulumiupgradeproviderissue), - "--state=open") + issueAlreadyExists, err := upgradeIssueExits(ctx, title, repoOrg, repoName) if err != nil { return err } - if found { - return nil - } - // Fall back to searching through the issues from the current user. - found, err = issueExistsForVersion(ctx, title, - fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), - fmt.Sprintf("--search=%q", title), - "--state=open", - "--author=@me") - if err != nil { - return err + // Write issue_created=true to GITHUB_OUTPUT, if it exists for CI control flow. + if GITHUB_OUTPUT, found := os.LookupEnv("GITHUB_OUTPUT"); found { + f, err := os.OpenFile(GITHUB_OUTPUT, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + defer f.Close() + if _, err := f.WriteString(fmt.Sprintf("issue_created=%t\n", issueAlreadyExists)); err != nil { + return err + } } - if found { + + // We've found an appropriate existing issue, so we'll skip creating a new one. + if issueAlreadyExists { return nil } - // We've not found an appropriate existing issue, so we'll create a new one. - // Hide searchable token in the issue body via an HTML comment to help us find this issue later without requiring labels to be set up. hiddenBody := fmt.Sprintf("", pulumiupgradeproviderissue) @@ -723,6 +716,33 @@ var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ct return nil }) +func upgradeIssueExits(ctx context.Context, title, repoOrg, repoName string) (bool, error) { + var found bool + var err error + // Search through existing "pulumiupgradeproviderissue" issues to see if we've already created one for this version. + found, err = issueExistsForVersion(ctx, title, + fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), + fmt.Sprintf("--search=%q", pulumiupgradeproviderissue), + "--state=open") + if err != nil { + return false, err + } + if found { + return true, nil + } + + // Fall back to searching through the issues from the current user. + found, err = issueExistsForVersion(ctx, title, + fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), + fmt.Sprintf("--search=%q", title), + "--state=open", + "--author=@me") + if err != nil { + return false, err + } + return found, nil +} + type issue struct { Title string `json:"title"` Number int `json:"number"` From 2578591809f7d55416bc0310b7121efb8669d9c9 Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Wed, 30 Oct 2024 15:52:30 +0000 Subject: [PATCH 4/5] Improve upgrade issue information Explain how this issue was created and what it is used for. --- upgrade/steps_helpers.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/upgrade/steps_helpers.go b/upgrade/steps_helpers.go index c6f4ac9..088149d 100644 --- a/upgrade/steps_helpers.go +++ b/upgrade/steps_helpers.go @@ -671,7 +671,13 @@ var getExpectedTargetFromIssues = stepv2.Func11E("From Issues", func(ctx context }, nil }) -const pulumiupgradeproviderissue = "pulumiupgradeproviderissue" +// Hide searchable token in the issue body via an HTML comment to help us find this issue later without requiring labels to be set up. +const upgradeIssueBodyTemplate = ` + + +> [!NOTE] +> This issue was created automatically by the upgrade-provider tool and should be automatically closed by a subsequent upgrade pull request. +` // Create an issue in the provider repo that signals an upgrade var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ctx context.Context, @@ -692,7 +698,7 @@ var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ct return err } defer f.Close() - if _, err := f.WriteString(fmt.Sprintf("issue_created=%t\n", issueAlreadyExists)); err != nil { + if _, err := f.WriteString(fmt.Sprintf("latest_version=%s\n", version)); err != nil { return err } } @@ -702,13 +708,10 @@ var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ct return nil } - // Hide searchable token in the issue body via an HTML comment to help us find this issue later without requiring labels to be set up. - hiddenBody := fmt.Sprintf("", pulumiupgradeproviderissue) - stepv2.Cmd(ctx, "gh", "issue", "create", "--repo="+repoOrg+"/"+repoName, - "--body=Release details: https://github.com/"+upstreamOrg+"/"+upstreamProviderName+"/releases/tag/v"+version+"\n\n"+hiddenBody, + "--body=Release details: https://github.com/"+upstreamOrg+"/"+upstreamProviderName+"/releases/tag/v"+version+"\n"+upgradeIssueBodyTemplate, "--title="+title, "--label="+"kind/enhancement", ) @@ -722,7 +725,7 @@ func upgradeIssueExits(ctx context.Context, title, repoOrg, repoName string) (bo // Search through existing "pulumiupgradeproviderissue" issues to see if we've already created one for this version. found, err = issueExistsForVersion(ctx, title, fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), - fmt.Sprintf("--search=%q", pulumiupgradeproviderissue), + fmt.Sprintf("--search=%q", upgradeIssueBodyTemplate), "--state=open") if err != nil { return false, err From dff9aeb80dab9a213e7140c39dfb3e328ca4313c Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Wed, 30 Oct 2024 17:32:50 +0000 Subject: [PATCH 5/5] Default to current implementation until issues have migrated to the new format --- upgrade/steps_helpers.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/upgrade/steps_helpers.go b/upgrade/steps_helpers.go index 088149d..fb97886 100644 --- a/upgrade/steps_helpers.go +++ b/upgrade/steps_helpers.go @@ -722,11 +722,14 @@ var createUpstreamUpgradeIssue = stepv2.Func30E("Ensure Upstream Issue", func(ct func upgradeIssueExits(ctx context.Context, title, repoOrg, repoName string) (bool, error) { var found bool var err error - // Search through existing "pulumiupgradeproviderissue" issues to see if we've already created one for this version. + // TODO: Remove this after we've migrated all issues to the new format. + // https://github.com/pulumi/upgrade-provider/issues/284 + // Fall back to searching through the issues from the current user. found, err = issueExistsForVersion(ctx, title, fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), - fmt.Sprintf("--search=%q", upgradeIssueBodyTemplate), - "--state=open") + fmt.Sprintf("--search=%q", title), + "--state=open", + "--author=@me") if err != nil { return false, err } @@ -734,12 +737,11 @@ func upgradeIssueExits(ctx context.Context, title, repoOrg, repoName string) (bo return true, nil } - // Fall back to searching through the issues from the current user. + // Search through existing pulumiupgradeproviderissue issues to see if we've already created one for this version. found, err = issueExistsForVersion(ctx, title, fmt.Sprintf("--repo=%q", repoOrg+"/"+repoName), - fmt.Sprintf("--search=%q", title), - "--state=open", - "--author=@me") + fmt.Sprintf("--search=%q", upgradeIssueBodyTemplate), + "--state=open") if err != nil { return false, err }