From db60ee5a3d04e178a42a1bfbd7b99e099743ee5a Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Wed, 17 Jan 2024 13:53:48 -0800 Subject: [PATCH] Migrate generate downstream to go (#9747) * Migrate generate downstream to go * Push to scratch path even if nothing to commit * Apply suggestions from code review * Fix for new RequestCall method --- .ci/magician/cmd/generate_comment.go | 6 +- .ci/magician/cmd/generate_downstream.go | 352 ++++++++++++++++++++++++ .ci/magician/cmd/interfaces.go | 4 + .ci/magician/cmd/mock_github_test.go | 10 + .ci/magician/github/get.go | 24 +- .ci/magician/github/set.go | 17 +- .ci/magician/provider/version.go | 3 +- .ci/magician/source/repo.go | 31 ++- 8 files changed, 431 insertions(+), 16 deletions(-) create mode 100644 .ci/magician/cmd/generate_downstream.go diff --git a/.ci/magician/cmd/generate_comment.go b/.ci/magician/cmd/generate_comment.go index 2684f3068aa5..0e15d20b389c 100644 --- a/.ci/magician/cmd/generate_comment.go +++ b/.ci/magician/cmd/generate_comment.go @@ -149,11 +149,7 @@ func execGenerateComment(env map[string]string, gh GithubClient, rnr ExecRunner, env["OLD_REF"] = oldBranch env["NEW_REF"] = newBranch - for _, repo := range []struct { - Title string - Path string - Version provider.Version - }{ + for _, repo := range []*source.Repo{ { Title: "TPG", Path: tpgLocalPath, diff --git a/.ci/magician/cmd/generate_downstream.go b/.ci/magician/cmd/generate_downstream.go new file mode 100644 index 000000000000..148a29e9d931 --- /dev/null +++ b/.ci/magician/cmd/generate_downstream.go @@ -0,0 +1,352 @@ +package cmd + +import ( + "fmt" + "magician/exec" + "magician/github" + "magician/provider" + "magician/source" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var changelogExp = regexp.MustCompile("(?s)```release-note.*?```") + +var gdEnvironmentVariables = [...]string{ + "BASE_BRANCH", + "GITHUB_TOKEN", + "GOPATH", +} + +var generateDownstreamCmd = &cobra.Command{ + Use: "generate-downstream", + Short: "Run generate downstream", + Long: `This command runs after pull requests are merged to generate corresponding changes in downstream repos. + + It expects the following arguments: + 1. Command, either head, base, or downstream + 2. Name of the downstream repo, either terraform, terraform-google-conversion, or tf-oics + 3. Version of the downstream + 4. Commit SHA of the squashed merge commit + + The following environment variables should be set: +` + listGDEnvironmentVariables(), + Run: func(cmd *cobra.Command, args []string) { + env := make(map[string]string, len(gdEnvironmentVariables)) + for _, ev := range gdEnvironmentVariables { + val, ok := os.LookupEnv(ev) + if !ok { + fmt.Printf("Did not provide %s environment variable\n", ev) + os.Exit(1) + } + env[ev] = val + } + + gh := github.NewClient() + rnr, err := exec.NewRunner() + if err != nil { + fmt.Println("Error creating a runner: ", err) + os.Exit(1) + } + ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr) + + if len(args) != 4 { + fmt.Printf("Wrong number of arguments %d, expected 4\n", len(args)) + os.Exit(1) + } + + execGenerateDownstream(env["BASE_BRANCH"], args[0], args[1], args[2], args[3], gh, rnr, ctlr) + }, +} + +func listGDEnvironmentVariables() string { + var result string + for i, ev := range gdEnvironmentVariables { + result += fmt.Sprintf("\t%2d. %s\n", i+1, ev) + } + return result +} + +func execGenerateDownstream(baseBranch, command, repo, version, ref string, gh GithubClient, rnr ExecRunner, ctlr *source.Controller) { + if baseBranch == "" { + baseBranch = "main" + } + + mmLocalPath := filepath.Join(rnr.GetCWD(), "..", "..") + mmCopyPath := filepath.Join(mmLocalPath, "..", fmt.Sprintf("mm-%s-%s-%s", repo, version, command)) + if _, err := rnr.Run("cp", []string{"-rp", mmLocalPath, mmCopyPath}, nil); err != nil { + fmt.Println("Error copying magic modules: ", err) + os.Exit(1) + } + mmRepo := &source.Repo{ + Name: "magic-modules", + Path: mmCopyPath, + } + + downstreamRepo, scratchRepo, commitMessage, err := cloneRepo(mmRepo, baseBranch, repo, version, command, ref, rnr, ctlr) + if err != nil { + fmt.Println("Error cloning repo: ", err) + os.Exit(1) + } + + if err := rnr.PushDir(mmCopyPath); err != nil { + fmt.Println("Error changing directory to copied magic modules: ", err) + os.Exit(1) + } + + if err := setGitConfig(rnr); err != nil { + fmt.Println("Error setting config: ", err) + os.Exit(1) + } + + if err := runMake(downstreamRepo, command, rnr); err != nil { + fmt.Println("Error running make: ", err) + os.Exit(1) + } + + if command == "downstream" { + pullRequest, err := getPullRequest(baseBranch, ref, gh) + if err != nil { + fmt.Println("Error getting pull request: ", err) + os.Exit(1) + } + + if err := pushCommit(downstreamRepo, scratchRepo, pullRequest, baseBranch, commitMessage, ref, gh, rnr, ctlr); err != nil { + fmt.Println("Error pushing commit: ", err) + os.Exit(1) + } + + if err := mergePullRequest(downstreamRepo, scratchRepo, pullRequest, rnr, gh); err != nil { + fmt.Println("Error merging pull request: ", err) + os.Exit(1) + } + } else { + if _, err := rnr.Run("git", []string{"push", ctlr.URL(scratchRepo), scratchRepo.Branch, "-f"}, nil); err != nil { + fmt.Println("Error pushing to scratch repo: ", err) + os.Exit(1) + } + } +} + +func cloneRepo(mmRepo *source.Repo, baseBranch, repo, version, command, ref string, rnr ExecRunner, ctlr *source.Controller) (*source.Repo, *source.Repo, string, error) { + downstreamRepo := &source.Repo{ + Title: repo, + Branch: baseBranch, + } + switch repo { + case "terraform": + if version == "ga" { + downstreamRepo.Name = "terraform-provider-google" + downstreamRepo.Version = provider.GA + } else if version == "beta" { + downstreamRepo.Name = "terraform-provider-google-beta" + downstreamRepo.Version = provider.Beta + } else { + return nil, nil, "", fmt.Errorf("unrecognized version %s", version) + } + downstreamRepo.Owner = "hashicorp" + case "terraform-google-conversion": + downstreamRepo.Name = "terraform-google-conversion" + downstreamRepo.Owner = "GoogleCloudPlatform" + case "tf-oics": + if downstreamRepo.Branch == "main" { + downstreamRepo.Branch = "master" + } + downstreamRepo.Name = "docs-examples" + downstreamRepo.Owner = "terraform-google-modules" + case "tf-cloud-docs": + fmt.Println(repo, " is no longer available.") + return nil, nil, "", nil + default: + return nil, nil, "", fmt.Errorf("unrecognized repo %s", repo) + } + ctlr.SetPath(downstreamRepo) + if err := ctlr.Clone(downstreamRepo); err != nil { + return nil, nil, "", err + } + scratchRepo := &source.Repo{ + Name: downstreamRepo.Name, + Owner: "modular-magician", + Path: downstreamRepo.Path, + Version: downstreamRepo.Version, + } + var commitMessage string + switch command { + case "head": + scratchRepo.Branch = "auto-pr-" + ref + commitMessage = fmt.Sprintf("New generated code for MM PR %s.", ref) + case "base": + // In this case, there is guaranteed to be a merge commit, + // and the *left* side of it is the old main branch. + // the *right* side of it is the code to be merged. + if err := ctlr.Checkout(mmRepo, "HEAD~"); err != nil { + return nil, nil, "", err + } + scratchRepo.Branch = fmt.Sprintf("auto-pr-%s-old", ref) + commitMessage = fmt.Sprintf("Old generated code for MM PR %s.", ref) + case "downstream": + scratchRepo.Branch = "downstream-pr-" + ref + originalMessage, err := rnr.Run("git", []string{"log", "-1", "--pretty=%B", ref}, nil) + if err != nil { + return nil, nil, "", err + } + commitMessage = fmt.Sprintf("%s\n[upstream:%s]", originalMessage, ref) + } + return downstreamRepo, scratchRepo, commitMessage, nil +} + +func setGitConfig(rnr ExecRunner) error { + if _, err := rnr.Run("git", []string{"config", "--local", "user.name", "Modular Magician"}, nil); err != nil { + return err + } + if _, err := rnr.Run("git", []string{"config", "--local", "user.email", "magic-modules@google.com"}, nil); err != nil { + return err + } + return nil +} + +func runMake(downstreamRepo *source.Repo, command string, rnr ExecRunner) error { + switch downstreamRepo.Title { + case "terraform-google-conversion": + if _, err := rnr.Run("make", []string{"clean-tgc", "OUTPUT_PATH=" + downstreamRepo.Path}, nil); err != nil { + return err + } + if _, err := rnr.Run("make", []string{"tgc", "OUTPUT_PATH=" + downstreamRepo.Path}, nil); err != nil { + return err + } + if command == "downstream" { + if err := rnr.PushDir(downstreamRepo.Path); err != nil { + return err + } + if _, err := rnr.Run("go", []string{"get", "-d", "github.com/hashicorp/terraform-provider-google-beta@" + downstreamRepo.Branch}, nil); err != nil { + return err + } + if _, err := rnr.Run("go", []string{"mod", "tidy"}, nil); err != nil { + return err + } + if _, err := rnr.Run("make", []string{"build"}, nil); err != nil { + fmt.Println("Error building tgc: ", err) + } + if err := rnr.PopDir(); err != nil { + return err + } + } + case "tf-oics": + if _, err := rnr.Run("make", []string{"tf-oics", "OUTPUT_PATH=" + downstreamRepo.Path}, nil); err != nil { + return err + } + case "terraform": + if _, err := rnr.Run("make", []string{"clean-provider", "OUTPUT_PATH=" + downstreamRepo.Path}, nil); err != nil { + return err + } + if _, err := rnr.Run("make", []string{"provider", "OUTPUT_PATH=" + downstreamRepo.Path, fmt.Sprintf("VERSION=%s", downstreamRepo.Version)}, nil); err != nil { + return err + } + } + return nil +} + +func getPullRequest(baseBranch, ref string, gh GithubClient) (*github.PullRequest, error) { + prs, err := gh.GetPullRequests("closed", baseBranch, "updated", "desc") + if err != nil { + return nil, err + } + for _, pr := range prs { + if pr.MergeCommitSha == ref { + return &pr, nil + } + } + return nil, fmt.Errorf("no pr found with merge commit sha %s and base branch %s", ref, baseBranch) +} + +func pushCommit(downstreamRepo, scratchRepo *source.Repo, pullRequest *github.PullRequest, baseBranch, commitMessage, ref string, gh GithubClient, rnr ExecRunner, ctlr *source.Controller) error { + if err := rnr.PushDir(scratchRepo.Path); err != nil { + return err + } + if err := setGitConfig(rnr); err != nil { + return err + } + + if _, err := rnr.Run("git", []string{"add", "."}, nil); err != nil { + return err + } + if _, err := rnr.Run("git", []string{"checkout", "-b", scratchRepo.Branch}, nil); err != nil { + return err + } + + if _, err := rnr.Run("git", []string{"commit", "--signoff", "-m", commitMessage}, nil); err != nil { + if strings.Contains(err.Error(), "nothing to commit, working tree clean") { + if _, err := rnr.Run("git", []string{"push", ctlr.URL(scratchRepo), scratchRepo.Branch, "-f"}, nil); err != nil { + return err + } + return nil + } + return err + } + + if downstreamRepo.Title == "terraform" { + // Add the changelog entry. + rnr.Mkdir(".changelog") + if err := rnr.WriteFile(filepath.Join(".changelog", fmt.Sprintf("%d.txt", pullRequest.Number)), strings.Join(changelogExp.FindAllString(pullRequest.Body, -1), "\n")); err != nil { + return err + } + if _, err := rnr.Run("git", []string{"add", "."}, nil); err != nil { + return err + } + if _, err := rnr.Run("git", []string{"commit", "--signoff", "--amend", "--no-edit"}, nil); err != nil { + return err + } + } + + if _, err := rnr.Run("git", []string{"push", ctlr.URL(scratchRepo), scratchRepo.Branch, "-f"}, nil); err != nil { + return err + } + + return nil +} + +func mergePullRequest(downstreamRepo, scratchRepo *source.Repo, pullRequest *github.PullRequest, rnr ExecRunner, gh GithubClient) error { + fmt.Printf(`Base: %s:%s +Head: %s:%s +`, downstreamRepo.Owner, downstreamRepo.Branch, scratchRepo.Owner, scratchRepo.Branch) + newPRURL, err := rnr.Run("hub", []string{ + "pull-request", + "-b", + fmt.Sprintf("%s:%s", + downstreamRepo.Owner, + downstreamRepo.Branch), + "-h", + fmt.Sprintf("%s:%s", + scratchRepo.Owner, + scratchRepo.Branch), + "-m", + pullRequest.Title, + "-m", + pullRequest.Body, + "-m", + "Derived from " + pullRequest.HTMLUrl, + }, nil) + if err != nil { + return err + } + fmt.Println("Created PR ", newPRURL) + newPRURLParts := strings.Split(newPRURL, "/") + newPRNumber := strings.TrimSuffix(newPRURLParts[len(newPRURLParts)-1], "\n") + + // Wait a few seconds, then merge the PR. + time.Sleep(5 * time.Second) + fmt.Println("Merging PR ", newPRURL) + if err := gh.MergePullRequest(downstreamRepo.Owner, downstreamRepo.Name, newPRNumber); err != nil { + return err + } + return nil +} + +func init() { + rootCmd.AddCommand(generateDownstreamCmd) +} diff --git a/.ci/magician/cmd/interfaces.go b/.ci/magician/cmd/interfaces.go index b9b6c34ff1cb..879f4a70a5c8 100644 --- a/.ci/magician/cmd/interfaces.go +++ b/.ci/magician/cmd/interfaces.go @@ -21,10 +21,12 @@ import ( type GithubClient interface { GetPullRequest(prNumber string) (github.PullRequest, error) + GetPullRequests(state, base, sort, direction string) ([]github.PullRequest, error) GetPullRequestRequestedReviewers(prNumber string) ([]github.User, error) GetPullRequestPreviousReviewers(prNumber string) ([]github.User, error) GetUserType(user string) github.UserType GetTeamMembers(organization, team string) ([]github.User, error) + MergePullRequest(owner, repo, prNumber string) error PostBuildStatus(prNumber, title, state, targetURL, commitSha string) error PostComment(prNumber, comment string) error RequestPullRequestReviewer(prNumber, assignee string) error @@ -42,9 +44,11 @@ type CloudbuildClient interface { type ExecRunner interface { GetCWD() string Copy(src, dest string) error + Mkdir(path string) error RemoveAll(path string) error PushDir(path string) error PopDir() error + WriteFile(name, data string) error Run(name string, args []string, env map[string]string) (string, error) MustRun(name string, args []string, env map[string]string) string } diff --git a/.ci/magician/cmd/mock_github_test.go b/.ci/magician/cmd/mock_github_test.go index 9b2e7e261a52..69eb80eee99f 100644 --- a/.ci/magician/cmd/mock_github_test.go +++ b/.ci/magician/cmd/mock_github_test.go @@ -31,6 +31,11 @@ func (m *mockGithub) GetPullRequest(prNumber string) (github.PullRequest, error) return m.pullRequest, nil } +func (m *mockGithub) GetPullRequests(state, base, sort, direction string) ([]github.PullRequest, error) { + m.calledMethods["GetPullRequests"] = append(m.calledMethods["GetPullRequests"], []any{state, base, sort, direction}) + return []github.PullRequest{m.pullRequest}, nil +} + func (m *mockGithub) GetUserType(user string) github.UserType { m.calledMethods["GetUserType"] = append(m.calledMethods["GetUserType"], []any{user}) return m.userType @@ -80,3 +85,8 @@ func (m *mockGithub) CreateWorkflowDispatchEvent(workflowFileName string, inputs m.calledMethods["CreateWorkflowDispatchEvent"] = append(m.calledMethods["CreateWorkflowDispatchEvent"], []any{workflowFileName, inputs}) return nil } + +func (m *mockGithub) MergePullRequest(owner, repo, prNumber string) error { + m.calledMethods["MergePullRequest"] = append(m.calledMethods["MergePullRequest"], []any{owner, repo, prNumber}) + return nil +} diff --git a/.ci/magician/github/get.go b/.ci/magician/github/get.go index 83ce8a1d19ac..4b1409460639 100644 --- a/.ci/magician/github/get.go +++ b/.ci/magician/github/get.go @@ -29,8 +29,13 @@ type Label struct { } type PullRequest struct { - User User `json:"user"` - Labels []Label `json:"labels"` + HTMLUrl string `json:"html_url"` + Number int `json:"number"` + Title string `json:"title"` + User User `json:"user"` + Body string `json:"body"` + Labels []Label `json:"labels"` + MergeCommitSha string `json:"merge_commit_sha"` } func (gh *Client) GetPullRequest(prNumber string) (PullRequest, error) { @@ -39,11 +44,18 @@ func (gh *Client) GetPullRequest(prNumber string) (PullRequest, error) { var pullRequest PullRequest err := utils.RequestCall(url, "GET", gh.token, &pullRequest, nil) - if err != nil { - return pullRequest, err - } - return pullRequest, nil + return pullRequest, err +} + +func (gh *Client) GetPullRequests(state, base, sort, direction string) ([]PullRequest, error) { + url := fmt.Sprintf("https://api.github.com/repos/GoogleCloudPlatform/magic-modules/pulls?state=%s&base=%s&sort=%s&direction=%s", state, base, sort, direction) + + var pullRequests []PullRequest + + err := utils.RequestCall(url, "GET", gh.token, &pullRequests, nil) + + return pullRequests, err } func (gh *Client) GetPullRequestRequestedReviewers(prNumber string) ([]User, error) { diff --git a/.ci/magician/github/set.go b/.ci/magician/github/set.go index eedf0a61f625..698781c6d667 100644 --- a/.ci/magician/github/set.go +++ b/.ci/magician/github/set.go @@ -112,7 +112,22 @@ func (gh *Client) CreateWorkflowDispatchEvent(workflowFileName string, inputs ma return fmt.Errorf("failed to create workflow dispatch event: %s", err) } - fmt.Printf("Successfully created workflow dispatch event for %s with inputs %v", workflowFileName, inputs) + fmt.Printf("Successfully created workflow dispatch event for %s with inputs %v\n", workflowFileName, inputs) + + return nil +} + +func (gh *Client) MergePullRequest(owner, repo, prNumber string) error { + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/pulls/%s/merge", owner, repo, prNumber) + err := utils.RequestCall(url, "PUT", gh.token, nil, map[string]any{ + "merge_method": "squash", + }) + + if err != nil { + return fmt.Errorf("failed to merge pull request: %s", err) + } + + fmt.Printf("Successfully merged pull request %s\n", prNumber) return nil } diff --git a/.ci/magician/provider/version.go b/.ci/magician/provider/version.go index 7c98f3817e29..c8c65fc8e82b 100644 --- a/.ci/magician/provider/version.go +++ b/.ci/magician/provider/version.go @@ -3,7 +3,8 @@ package provider type Version int const ( - GA Version = iota + None Version = iota + GA Beta ) diff --git a/.ci/magician/source/repo.go b/.ci/magician/source/repo.go index 20b7a4f52ec4..2d62d531431f 100644 --- a/.ci/magician/source/repo.go +++ b/.ci/magician/source/repo.go @@ -2,6 +2,7 @@ package source import ( "fmt" + "magician/provider" "path/filepath" "strings" ) @@ -10,8 +11,10 @@ type Repo struct { Name string // Name in GitHub (e.g. magic-modules) Title string // Title for display (e.g. Magic Modules) Branch string // Branch to clone, optional + Owner string // Owner of repo, optional Path string // local Path once cloned, including Name - DiffCanFail bool // whether to allow the command to continue if cloning or diffing the repo fails + Version provider.Version + DiffCanFail bool // whether to allow the command to continue if cloning or diffing the repo fails } type Controller struct { @@ -37,12 +40,24 @@ func NewController(goPath, username, token string, rnr Runner) *Controller { } func (gc Controller) SetPath(repo *Repo) { - repo.Path = filepath.Join(gc.goPath, "src", "github.com", gc.username, repo.Name) + owner := repo.Owner + if owner == "" { + owner = gc.username + } + repo.Path = filepath.Join(gc.goPath, "src", "github.com", owner, repo.Name) +} + +func (gc Controller) URL(repo *Repo) string { + owner := repo.Owner + if owner == "" { + owner = gc.username + } + return fmt.Sprintf("https://%s:%s@github.com/%s/%s", gc.username, gc.token, owner, repo.Name) } func (gc Controller) Clone(repo *Repo) error { + url := gc.URL(repo) var err error - url := fmt.Sprintf("https://%s:%s@github.com/%s/%s", gc.username, gc.token, gc.username, repo.Name) if repo.Branch == "" { _, err = gc.rnr.Run("git", []string{"clone", url, repo.Path}, nil) } else { @@ -56,6 +71,16 @@ func (gc Controller) Clone(repo *Repo) error { return err } +func (gc Controller) Checkout(repo *Repo, ref string) error { + if err := gc.rnr.PushDir(repo.Path); err != nil { + return err + } + if _, err := gc.rnr.Run("git", []string{"checkout", ref}, nil); err != nil { + return err + } + return gc.rnr.PopDir() +} + func (gc Controller) Fetch(repo *Repo, branch string) error { if err := gc.rnr.PushDir(repo.Path); err != nil { return err