From 4dfe570720956741a6f1eaff28ef768c4e8c5e6f Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 13 Oct 2023 22:29:40 -0300 Subject: [PATCH 01/27] change repo files --- repository/repository.go | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/repository/repository.go b/repository/repository.go index 60fd4e2a..8f7a6e8f 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -2,8 +2,11 @@ package repository import ( "context" + "encoding/json" "errors" "fmt" + "io/ioutil" + "net/http" "os" "os/exec" "strings" @@ -13,6 +16,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/google/go-github/v39/github" "github.com/rancher/ecm-distro-tools/types" + "github.com/sirupsen/logrus" "golang.org/x/oauth2" ) @@ -21,6 +25,8 @@ const ( emptyReleaseNote = "```release-note\r\n\r\n```" noneReleaseNote = "```release-note\r\nNONE\r\n```" httpTimeout = time.Second * 10 + ghContentURL = "https://raw.githubusercontent.com" + baseURLGHApi = "https://api.github.com" ) // repoToOrg associates repo to org. @@ -456,3 +462,78 @@ To find more information on specific steps, please see documentation [here](http - [ ] QA: Final validation of above PR and tracked through the linked ticket - [ ] PJM: Close the milestone in GitHub. ` + +// Partial Body from commit api, fell free to add more fields if needed +type Author struct { + Name string `json:"name"` + Email string `json:"email"` + Date string `json:"date"` +} + +type Commit struct { + Author Author `json:"author"` + Committer Author `json:"committer"` + Message string `json:"message"` + URL string `json:"url"` +} + +type CommitResponse struct { + SHA string `json:"sha"` + NodeID string `json:"node_id"` + Commit Commit `json:"commit"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + CommentsURL string `json:"comments_url"` +} + +func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commit, error) { + apiUrl := fmt.Sprintf(baseURLGHApi+"/repos/%s/%s/commits/%s", owner, repo, commitHash) + + response, err := httpClient.Get(apiUrl) + if err != nil { + logrus.Debug("Error making request:", err) + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + logrus.Debug("Failed to fetch commit information. Status code:", response.StatusCode) + return nil, err + } + data, err := ioutil.ReadAll(response.Body) + if err != nil { + logrus.Debug("Error reading response body:", err) + return nil, err + } + + var commitResponse CommitResponse + + err = json.Unmarshal([]byte(data), &commitResponse) + if err != nil { + logrus.Debug("Error unmarshaling JSON:", err) + return nil, err + } + + return &commitResponse.Commit, nil +} + +func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpClient *http.Client) ([]byte, error) { + rawURL := fmt.Sprintf(ghContentURL+"/%s/%s/%s/%s", owner, repo, commitHash, filePath) + + response, err := http.Get(rawURL) + if err != nil { + logrus.Debug("Error fetching raw file:", err) + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + logrus.Debug("Failed to fetch raw file. Status code:", response.StatusCode) + return nil, err + } + + data, err := ioutil.ReadAll(response.Body) + if err != nil { + logrus.Debug("Error reading response body:", err) + return nil, err + } + return data, nil +} From e0620ec7271d28c5905863da4101a9249060a221 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Mon, 16 Oct 2023 22:29:45 -0300 Subject: [PATCH 02/27] func to check all deps and command --- cmd/rancher_release/check_rancher_rc_deps.go | 70 ++++++++++++++++++++ cmd/rancher_release/main.go | 1 + release/rancher/rancher.go | 64 +++++++++++++++++- 3 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 cmd/rancher_release/check_rancher_rc_deps.go diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go new file mode 100644 index 00000000..64fe0aa0 --- /dev/null +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -0,0 +1,70 @@ +package main + +import ( + "errors" + + "github.com/rancher/ecm-distro-tools/release/rancher" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +func checkRancherRCDepsCommand() *cli.Command { + return &cli.Command{ + Name: "check-rancher-rc-deps", + Usage: "check if rancher commit contains the necessary values for a final rc", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "commit", + Aliases: []string{"c"}, + Usage: "last commit for a final rc", + Required: true, + }, + &cli.StringFlag{ + Name: "release-title", + Aliases: []string{"rt"}, + Usage: "release title from a given release process", + Required: false, + }, + &cli.StringFlag{ + Name: "org", + Aliases: []string{"o"}, + Usage: "organization name, e.g rancher,suse", + Required: false, + }, + &cli.StringFlag{ + Name: "repo", + Aliases: []string{"r"}, + Usage: "repository from process", + Required: false, + }, + &cli.StringFlag{ + Name: "files", + Aliases: []string{"f"}, + Usage: "files to be checked", + Required: false, + }, + }, + Action: checkRancherRCDeps, + } +} + +func checkRancherRCDeps(c *cli.Context) error { + rcReleaseTitle := c.String("release-title") + rcCommit := c.String("commit") + rcOrg := c.String("org") + rcRepo := c.String("repo") + rcFiles := c.String("files") + + if rcFiles == "" { + return errors.New("'files' is required, e.g --files Dockerfile.dapper,go.mod ") + } + if rcCommit == "" && rcReleaseTitle == "" { + return errors.New("'commit' or 'release-title' are required") + } + if rcOrg == "" { + return errors.New("'org' is required") + } + logrus.Debug("commit: " + rcCommit) + + return rancher.CheckRancherFinalRCDeps(rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) +} diff --git a/cmd/rancher_release/main.go b/cmd/rancher_release/main.go index 2f40522c..fbdac7fd 100644 --- a/cmd/rancher_release/main.go +++ b/cmd/rancher_release/main.go @@ -31,6 +31,7 @@ func main() { checkRancherImageCommand(), setKDMBranchReferencesCommand(), setChartsBranchReferencesCommand(), + checkRancherRCDepsCommand(), } app.Flags = rootFlags diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 6d2354be..1fa00025 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net/http" + "regexp" "strconv" "strings" "time" @@ -90,6 +91,10 @@ git commit --all --signoff -m "update chart branch references to {{ .NewBranch } if [ "${DRY_RUN}" = false ]; then git push --set-upstream origin ${BRANCH_NAME} fi` + partialFinalRCCommitMessage = "last commit for final rc" + releaseTitleRegex = `^Pre-release v2\.7\.[0-9]{1,100}-rc[1-9][0-9]{0,1}$` + rancherRepo = "rancher" + rancherOrg = rancherRepo ) type SetBranchReferencesArgs struct { @@ -264,8 +269,7 @@ func SetChartBranchReferences(ctx context.Context, forkPath, rancherBaseBranch, } func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchName, forkOwner string, ghClient *github.Client) error { - const repo = "rancher" - org, err := repository.OrgFromRepo(repo) + org, err := repository.OrgFromRepo(rancherRepo) if err != nil { return err } @@ -275,7 +279,61 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa Head: github.String(forkOwner + ":" + branchName), MaintainerCanModify: github.Bool(true), } - _, _, err = ghClient.PullRequests.Create(ctx, org, repo, pull) + _, _, err = ghClient.PullRequests.Create(ctx, org, rancherRepo, pull) return err } + +func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) error { + var matchCommitMessage bool + var existsReleaseTitle bool + var badFiles bool + + httpClient := ecmHTTP.NewClient(time.Second * 15) + + if repo == "" { + repo = rancherRepo + } + if commitHash != "" { + commitData, err := repository.CommitInfo(org, repo, commitHash, &httpClient) + if err != nil { + return err + } + matchCommitMessage = strings.Contains(commitData.Message, partialFinalRCCommitMessage) + + } + if releaseTitle != "" { + innerExistsReleaseTitle, err := regexp.MatchString(releaseTitleRegex, releaseTitle) + if err != nil { + return err + } + existsReleaseTitle = innerExistsReleaseTitle + } + + if matchCommitMessage || existsReleaseTitle { + for _, filePath := range strings.Split(files, ",") { + content, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) + if err != nil { + return err + } + + if regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`).Match(content) { + badFiles = true + logrus.Info("error: " + filePath + " contains dev dependencies") + } + if regexp.MustCompile(`-rc[0-9]+`).Match(content) { + badFiles = true + logrus.Info("error: " + filePath + " contains rc tags") + } + } + if badFiles { + return errors.New("Check failed, some files don't match the expected dependencies for a final release candidate") + } + + logrus.Info("Check completed successfully") + return nil + } + + logrus.Info("Skipped check") + return nil +} From 54fbdda686db3edf95c6aad85b983785200d67d5 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Tue, 17 Oct 2023 17:29:14 -0300 Subject: [PATCH 03/27] add test --- release/rancher/rancher_test.go | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/release/rancher/rancher_test.go b/release/rancher/rancher_test.go index 8b1baffa..52657c22 100644 --- a/release/rancher/rancher_test.go +++ b/release/rancher/rancher_test.go @@ -1,10 +1,15 @@ package rancher import ( + "errors" + "io/ioutil" "net/http" "net/http/httptest" "reflect" + "strings" "testing" + + "github.com/stretchr/testify/assert" ) func TestNonMirroredRCImages(t *testing.T) { @@ -59,3 +64,48 @@ func TestRancherHelmChartVersions(t *testing.T) { t.Errorf("expected %v, got %v", expectedVersions, versions) } } + +func TestCheckRancherFinalRCDeps(t *testing.T) { + org := "testOrg" + repo := "testRepo" + commitHash := "testHash" + releaseTitle := "Pre-release v2.7.1-rc1" + files := "file1,file2" + + t.Run("Valid CheckRancherFinalRCDeps", func(t *testing.T) { + httpClient := &MockHTTPClient{Response: "dev-v2.7.1"} + err := CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files, &httpClient) + assert.NoError(t, err) + }) + + t.Run("Invalid CheckRancherFinalRCDeps with Dev Dependency", func(t *testing.T) { + httpClient := &MockHTTPClient{Response: "dev-v3.0"} + err := CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files, &httpClient) + expectedError := errors.New("Check failed, some files don't match the expected dependencies for a final release candidate") + assert.EqualError(t, err, expectedError.Error()) + }) + + t.Run("Invalid CheckRancherFinalRCDeps with RC Tag", func(t *testing.T) { + httpClient := &MockHTTPClient{Response: "-rc1"} + err := CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files, &httpClient) + expectedError := errors.New("Check failed, some files don't match the expected dependencies for a final release candidate") + assert.EqualError(t, err, expectedError.Error()) + }) + + t.Run("Skipped CheckRancherFinalRCDeps", func(t *testing.T) { + httpClient := &MockHTTPClient{Response: "dev-v2.7.1"} + err := CheckRancherFinalRCDeps("", "", "", "", "", &httpClient) + assert.NoError(t, err) + }) +} + +type MockHTTPClient struct { + Response string +} + +func (c *MockHTTPClient) Get(url string) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(strings.NewReader(c.Response)), + }, nil +} From ea323c580cd94b7b30a704b89b71bdb0057246a3 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 19 Oct 2023 18:06:09 -0300 Subject: [PATCH 04/27] last adjusts --- cmd/rancher_release/check_rancher_rc_deps.go | 15 +++--- release/rancher/rancher.go | 14 ++++-- release/rancher/rancher_test.go | 50 -------------------- repository/repository.go | 3 +- 4 files changed, 18 insertions(+), 64 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 64fe0aa0..15ce5711 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -11,7 +11,7 @@ import ( func checkRancherRCDepsCommand() *cli.Command { return &cli.Command{ Name: "check-rancher-rc-deps", - Usage: "check if rancher commit contains the necessary values for a final rc", + Usage: "check if the Rancher version specified by the commit does not contain development dependencies and rc tags", Flags: []cli.Flag{ &cli.StringFlag{ Name: "commit", @@ -28,13 +28,13 @@ func checkRancherRCDepsCommand() *cli.Command { &cli.StringFlag{ Name: "org", Aliases: []string{"o"}, - Usage: "organization name, e.g rancher,suse", + Usage: "organization name", Required: false, }, &cli.StringFlag{ Name: "repo", Aliases: []string{"r"}, - Usage: "repository from process", + Usage: "rancher repository from process", Required: false, }, &cli.StringFlag{ @@ -55,16 +55,17 @@ func checkRancherRCDeps(c *cli.Context) error { rcRepo := c.String("repo") rcFiles := c.String("files") - if rcFiles == "" { - return errors.New("'files' is required, e.g --files Dockerfile.dapper,go.mod ") - } if rcCommit == "" && rcReleaseTitle == "" { return errors.New("'commit' or 'release-title' are required") } + if rcFiles == "" { + return errors.New("'files' is required, e.g, --files Dockerfile.dapper,go.mod") + } if rcOrg == "" { return errors.New("'org' is required") } - logrus.Debug("commit: " + rcCommit) + logrus.Debugf("organization: %s, repository: %s, commit: %s, release title: %s, files: %s", + rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) return rancher.CheckRancherFinalRCDeps(rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) } diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 1fa00025..d3a9e0ec 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -285,9 +285,13 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa } func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) error { - var matchCommitMessage bool - var existsReleaseTitle bool - var badFiles bool + var ( + devDependencyPattern = regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) + rcTagPattern = regexp.MustCompile(`-rc[0-9]+`) + matchCommitMessage bool + existsReleaseTitle bool + badFiles bool + ) httpClient := ecmHTTP.NewClient(time.Second * 15) @@ -317,11 +321,11 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) return err } - if regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`).Match(content) { + if devDependencyPattern.Match(content) { badFiles = true logrus.Info("error: " + filePath + " contains dev dependencies") } - if regexp.MustCompile(`-rc[0-9]+`).Match(content) { + if rcTagPattern.Match(content) { badFiles = true logrus.Info("error: " + filePath + " contains rc tags") } diff --git a/release/rancher/rancher_test.go b/release/rancher/rancher_test.go index 52657c22..8b1baffa 100644 --- a/release/rancher/rancher_test.go +++ b/release/rancher/rancher_test.go @@ -1,15 +1,10 @@ package rancher import ( - "errors" - "io/ioutil" "net/http" "net/http/httptest" "reflect" - "strings" "testing" - - "github.com/stretchr/testify/assert" ) func TestNonMirroredRCImages(t *testing.T) { @@ -64,48 +59,3 @@ func TestRancherHelmChartVersions(t *testing.T) { t.Errorf("expected %v, got %v", expectedVersions, versions) } } - -func TestCheckRancherFinalRCDeps(t *testing.T) { - org := "testOrg" - repo := "testRepo" - commitHash := "testHash" - releaseTitle := "Pre-release v2.7.1-rc1" - files := "file1,file2" - - t.Run("Valid CheckRancherFinalRCDeps", func(t *testing.T) { - httpClient := &MockHTTPClient{Response: "dev-v2.7.1"} - err := CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files, &httpClient) - assert.NoError(t, err) - }) - - t.Run("Invalid CheckRancherFinalRCDeps with Dev Dependency", func(t *testing.T) { - httpClient := &MockHTTPClient{Response: "dev-v3.0"} - err := CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files, &httpClient) - expectedError := errors.New("Check failed, some files don't match the expected dependencies for a final release candidate") - assert.EqualError(t, err, expectedError.Error()) - }) - - t.Run("Invalid CheckRancherFinalRCDeps with RC Tag", func(t *testing.T) { - httpClient := &MockHTTPClient{Response: "-rc1"} - err := CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files, &httpClient) - expectedError := errors.New("Check failed, some files don't match the expected dependencies for a final release candidate") - assert.EqualError(t, err, expectedError.Error()) - }) - - t.Run("Skipped CheckRancherFinalRCDeps", func(t *testing.T) { - httpClient := &MockHTTPClient{Response: "dev-v2.7.1"} - err := CheckRancherFinalRCDeps("", "", "", "", "", &httpClient) - assert.NoError(t, err) - }) -} - -type MockHTTPClient struct { - Response string -} - -func (c *MockHTTPClient) Get(url string) (*http.Response, error) { - return &http.Response{ - StatusCode: 200, - Body: ioutil.NopCloser(strings.NewReader(c.Response)), - }, nil -} diff --git a/repository/repository.go b/repository/repository.go index 8f7a6e8f..084f8cf5 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -463,7 +463,6 @@ To find more information on specific steps, please see documentation [here](http - [ ] PJM: Close the milestone in GitHub. ` -// Partial Body from commit api, fell free to add more fields if needed type Author struct { Name string `json:"name"` Email string `json:"email"` @@ -529,11 +528,11 @@ func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpCl logrus.Debug("Failed to fetch raw file. Status code:", response.StatusCode) return nil, err } - data, err := ioutil.ReadAll(response.Body) if err != nil { logrus.Debug("Error reading response body:", err) return nil, err } + return data, nil } From 0c968b8946953e93e19182fc6a5fdc9157c652b4 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 19 Oct 2023 18:57:48 -0300 Subject: [PATCH 05/27] improve code --- release/rancher/rancher.go | 6 +++--- repository/repository.go | 15 +++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index d3a9e0ec..fcc38314 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -331,13 +331,13 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) } } if badFiles { - return errors.New("Check failed, some files don't match the expected dependencies for a final release candidate") + return errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } - logrus.Info("Check completed successfully") + logrus.Info("check completed successfully") return nil } - logrus.Info("Skipped check") + logrus.Info("skipped check") return nil } diff --git a/repository/repository.go b/repository/repository.go index 084f8cf5..a0ae9acd 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "os/exec" + "strconv" "strings" "time" @@ -495,12 +496,11 @@ func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commi } defer response.Body.Close() if response.StatusCode != http.StatusOK { - logrus.Debug("Failed to fetch commit information. Status code:", response.StatusCode) - return nil, err + return nil, errors.New("failed to fetch commit information. status code: " + strconv.Itoa(response.StatusCode)) } data, err := ioutil.ReadAll(response.Body) if err != nil { - logrus.Debug("Error reading response body:", err) + logrus.Debug("error reading response body:", err) return nil, err } @@ -508,7 +508,7 @@ func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commi err = json.Unmarshal([]byte(data), &commitResponse) if err != nil { - logrus.Debug("Error unmarshaling JSON:", err) + logrus.Debug("error unmarshaling JSON:", err) return nil, err } @@ -520,17 +520,16 @@ func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpCl response, err := http.Get(rawURL) if err != nil { - logrus.Debug("Error fetching raw file:", err) + logrus.Debug("error fetching raw file:", err) return nil, err } defer response.Body.Close() if response.StatusCode != http.StatusOK { - logrus.Debug("Failed to fetch raw file. Status code:", response.StatusCode) - return nil, err + return nil, errors.New("failed to fetch raw file. status code: " + strconv.Itoa(response.StatusCode)) } data, err := ioutil.ReadAll(response.Body) if err != nil { - logrus.Debug("Error reading response body:", err) + logrus.Debug("error reading response body:", err) return nil, err } From df575f1c21d8aa67c3d200adb11b99a57b1e309a Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 19 Oct 2023 19:19:49 -0300 Subject: [PATCH 06/27] change requires --- cmd/rancher_release/check_rancher_rc_deps.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 15ce5711..a38612a6 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -17,7 +17,7 @@ func checkRancherRCDepsCommand() *cli.Command { Name: "commit", Aliases: []string{"c"}, Usage: "last commit for a final rc", - Required: true, + Required: false, }, &cli.StringFlag{ Name: "release-title", @@ -29,7 +29,7 @@ func checkRancherRCDepsCommand() *cli.Command { Name: "org", Aliases: []string{"o"}, Usage: "organization name", - Required: false, + Required: true, }, &cli.StringFlag{ Name: "repo", @@ -41,7 +41,7 @@ func checkRancherRCDepsCommand() *cli.Command { Name: "files", Aliases: []string{"f"}, Usage: "files to be checked", - Required: false, + Required: true, }, }, Action: checkRancherRCDeps, From 98cc14d4ec3b82a9199152b7093e75a006fa5ccb Mon Sep 17 00:00:00 2001 From: Johnatas Date: Mon, 23 Oct 2023 13:10:52 -0300 Subject: [PATCH 07/27] remove necessity of repo and org Signed-off-by: Johnatas --- cmd/rancher_release/check_rancher_rc_deps.go | 7 ++----- release/rancher/rancher.go | 3 +++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index a38612a6..56c02f2a 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -11,7 +11,7 @@ import ( func checkRancherRCDepsCommand() *cli.Command { return &cli.Command{ Name: "check-rancher-rc-deps", - Usage: "check if the Rancher version specified by the commit does not contain development dependencies and rc tags", + Usage: "check if the Rancher version specified by the commit or pre-release title does not contain development dependencies and rc tags", Flags: []cli.Flag{ &cli.StringFlag{ Name: "commit", @@ -29,7 +29,7 @@ func checkRancherRCDepsCommand() *cli.Command { Name: "org", Aliases: []string{"o"}, Usage: "organization name", - Required: true, + Required: false, }, &cli.StringFlag{ Name: "repo", @@ -61,9 +61,6 @@ func checkRancherRCDeps(c *cli.Context) error { if rcFiles == "" { return errors.New("'files' is required, e.g, --files Dockerfile.dapper,go.mod") } - if rcOrg == "" { - return errors.New("'org' is required") - } logrus.Debugf("organization: %s, repository: %s, commit: %s, release title: %s, files: %s", rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 20083465..b47d5bf0 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -297,6 +297,9 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) if repo == "" { repo = rancherRepo } + if org == "" { + org = rancherOrg + } if commitHash != "" { commitData, err := repository.CommitInfo(org, repo, commitHash, &httpClient) if err != nil { From b2d1773504d15340388f3475a7adf430dbbdd6db Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 26 Oct 2023 16:07:40 -0300 Subject: [PATCH 08/27] more improves Signed-off-by: Johnatas --- cmd/rancher_release/check_rancher_rc_deps.go | 4 +- release/rancher/rancher.go | 52 ++++++++++++++------ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 56c02f2a..0b9497ee 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -21,7 +21,7 @@ func checkRancherRCDepsCommand() *cli.Command { }, &cli.StringFlag{ Name: "release-title", - Aliases: []string{"rt"}, + Aliases: []string{"t"}, Usage: "release title from a given release process", Required: false, }, @@ -34,7 +34,7 @@ func checkRancherRCDepsCommand() *cli.Command { &cli.StringFlag{ Name: "repo", Aliases: []string{"r"}, - Usage: "rancher repository from process", + Usage: "rancher repository", Required: false, }, &cli.StringFlag{ diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index b47d5bf0..04a43ced 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "errors" + "fmt" "io" "net/http" "regexp" @@ -90,10 +91,11 @@ git commit --all --signoff -m "update chart branch references to {{ .NewBranch } if [ "${DRY_RUN}" = false ]; then git push --set-upstream origin ${BRANCH_NAME} fi` - partialFinalRCCommitMessage = "last commit for final rc" - releaseTitleRegex = `^Pre-release v2\.7\.[0-9]{1,100}-rc[1-9][0-9]{0,1}$` - rancherRepo = "rancher" - rancherOrg = rancherRepo +) + +const ( + rancherRepo = "rancher" + rancherOrg = rancherRepo ) type SetBranchReferencesArgs struct { @@ -284,14 +286,19 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa } func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) error { + const ( + releaseTitleRegex = `^Pre-release v2\.7\.[0-9]{1,100}-rc[1-9][0-9]{0,1}$` + partialFinalRCCommitMessage = "last commit for final rc" + ) var ( - devDependencyPattern = regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) - rcTagPattern = regexp.MustCompile(`-rc[0-9]+`) - matchCommitMessage bool - existsReleaseTitle bool - badFiles bool + matchCommitMessage bool + existsReleaseTitle bool + badFiles bool ) + devDependencyPattern := regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) + rcTagPattern := regexp.MustCompile(`-rc[0-9]+`) + httpClient := ecmHTTP.NewClient(time.Second * 15) if repo == "" { @@ -323,13 +330,28 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) return err } - if devDependencyPattern.Match(content) { - badFiles = true - logrus.Info("error: " + filePath + " contains dev dependencies") + scanner := bufio.NewScanner(strings.NewReader(string(content))) + lineNum := 1 + + for scanner.Scan() { + line := scanner.Text() + lineByte := []byte(line) + + if devDependencyPattern.Match(lineByte) { + badFiles = true + logMessage := fmt.Sprintf("error: %s contains dev dependencies (line %d) - %s", filePath, lineNum, line) + logrus.Info(logMessage) + } + if rcTagPattern.Match(lineByte) { + badFiles = true + logMessage := fmt.Sprintf("error: %s contains rc tags (line %d) - %s", filePath, lineNum, line) + logrus.Info(logMessage) + } + + lineNum++ } - if rcTagPattern.Match(content) { - badFiles = true - logrus.Info("error: " + filePath + " contains rc tags") + if err := scanner.Err(); err != nil { + return err } } if badFiles { From b7444fe656bb7eb26affe180e7a349a058947028 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 27 Oct 2023 19:43:26 -0300 Subject: [PATCH 09/27] more adjusts Signed-off-by: Johnatas --- release/rancher/rancher.go | 6 +++--- repository/repository.go | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 04a43ced..8925b21b 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -334,17 +334,17 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) lineNum := 1 for scanner.Scan() { - line := scanner.Text() + line := strings.TrimSpace(scanner.Text()) lineByte := []byte(line) if devDependencyPattern.Match(lineByte) { badFiles = true - logMessage := fmt.Sprintf("error: %s contains dev dependencies (line %d) - %s", filePath, lineNum, line) + logMessage := fmt.Sprintf("file: %s, line: %d, content: '%s' contains dev dependencies", filePath, lineNum, line) logrus.Info(logMessage) } if rcTagPattern.Match(lineByte) { badFiles = true - logMessage := fmt.Sprintf("error: %s contains rc tags (line %d) - %s", filePath, lineNum, line) + logMessage := fmt.Sprintf("file: %s, line: %d, content: '%s' contains rc tags", filePath, lineNum, line) logrus.Info(logMessage) } diff --git a/repository/repository.go b/repository/repository.go index d4dc6c81..5679f337 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -11,6 +11,7 @@ import ( "os/exec" "strconv" "strings" + "sync" "time" "github.com/go-git/go-git/v5" @@ -27,7 +28,7 @@ const ( noneReleaseNote = "```release-note\r\nNONE\r\n```" httpTimeout = time.Second * 10 ghContentURL = "https://raw.githubusercontent.com" - baseURLGHApi = "https://api.github.com" + ghAPIURL = "https://api.github.com" ) // repoToOrg associates repo to org. @@ -488,7 +489,9 @@ type CommitResponse struct { } func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commit, error) { - apiUrl := fmt.Sprintf(baseURLGHApi+"/repos/%s/%s/commits/%s", owner, repo, commitHash) + var commitResponseMutex sync.Mutex + + apiUrl := fmt.Sprintf(ghAPIURL+"/repos/%s/%s/commits/%s", owner, repo, commitHash) response, err := httpClient.Get(apiUrl) if err != nil { @@ -499,15 +502,12 @@ func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commi if response.StatusCode != http.StatusOK { return nil, errors.New("failed to fetch commit information. status code: " + strconv.Itoa(response.StatusCode)) } - data, err := ioutil.ReadAll(response.Body) - if err != nil { - logrus.Debug("error reading response body:", err) - return nil, err - } var commitResponse CommitResponse - err = json.Unmarshal([]byte(data), &commitResponse) + commitResponseMutex.Lock() + err = json.NewDecoder(response.Body).Decode(&commitResponse) + commitResponseMutex.Unlock() if err != nil { logrus.Debug("error unmarshaling JSON:", err) return nil, err From e0cf846f7fa705e0e534bfd5dfec972988940a98 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 27 Oct 2023 20:25:39 -0300 Subject: [PATCH 10/27] removing logrus Signed-off-by: Johnatas --- cmd/rancher_release/check_rancher_rc_deps.go | 10 +++++++- release/rancher/rancher.go | 26 +++++++++++--------- repository/repository.go | 5 ---- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 0b9497ee..d22f18d3 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -2,6 +2,7 @@ package main import ( "errors" + "fmt" "github.com/rancher/ecm-distro-tools/release/rancher" "github.com/sirupsen/logrus" @@ -64,5 +65,12 @@ func checkRancherRCDeps(c *cli.Context) error { logrus.Debugf("organization: %s, repository: %s, commit: %s, release title: %s, files: %s", rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) - return rancher.CheckRancherFinalRCDeps(rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) + output, err := rancher.CheckRancherFinalRCDeps(rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) + if err != nil { + return err + } + + fmt.Println(output) + + return nil } diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 8925b21b..f48b5f90 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -285,7 +285,7 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa return err } -func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) error { +func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) (string, error) { const ( releaseTitleRegex = `^Pre-release v2\.7\.[0-9]{1,100}-rc[1-9][0-9]{0,1}$` partialFinalRCCommitMessage = "last commit for final rc" @@ -294,6 +294,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) matchCommitMessage bool existsReleaseTitle bool badFiles bool + output string ) devDependencyPattern := regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) @@ -310,7 +311,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) if commitHash != "" { commitData, err := repository.CommitInfo(org, repo, commitHash, &httpClient) if err != nil { - return err + return "", err } matchCommitMessage = strings.Contains(commitData.Message, partialFinalRCCommitMessage) @@ -318,7 +319,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) if releaseTitle != "" { innerExistsReleaseTitle, err := regexp.MatchString(releaseTitleRegex, releaseTitle) if err != nil { - return err + return "", err } existsReleaseTitle = innerExistsReleaseTitle } @@ -327,7 +328,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) for _, filePath := range strings.Split(files, ",") { content, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) if err != nil { - return err + return "", err } scanner := bufio.NewScanner(strings.NewReader(string(content))) @@ -340,28 +341,29 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) if devDependencyPattern.Match(lineByte) { badFiles = true logMessage := fmt.Sprintf("file: %s, line: %d, content: '%s' contains dev dependencies", filePath, lineNum, line) - logrus.Info(logMessage) + output += logMessage + "\n" } if rcTagPattern.Match(lineByte) { badFiles = true logMessage := fmt.Sprintf("file: %s, line: %d, content: '%s' contains rc tags", filePath, lineNum, line) - logrus.Info(logMessage) + output += logMessage + "\n" } lineNum++ } if err := scanner.Err(); err != nil { - return err + return "", err } } if badFiles { - return errors.New("check failed, some files don't match the expected dependencies for a final release candidate") + fmt.Println(output) + return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } - logrus.Info("check completed successfully") - return nil + output += "check completed successfully" + "\n" + return output, nil } - logrus.Info("skipped check") - return nil + output += "skipped check" + return output, nil } diff --git a/repository/repository.go b/repository/repository.go index 5679f337..b7fc366f 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -18,7 +18,6 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/google/go-github/v39/github" "github.com/rancher/ecm-distro-tools/types" - "github.com/sirupsen/logrus" "golang.org/x/oauth2" ) @@ -495,7 +494,6 @@ func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commi response, err := httpClient.Get(apiUrl) if err != nil { - logrus.Debug("Error making request:", err) return nil, err } defer response.Body.Close() @@ -509,7 +507,6 @@ func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commi err = json.NewDecoder(response.Body).Decode(&commitResponse) commitResponseMutex.Unlock() if err != nil { - logrus.Debug("error unmarshaling JSON:", err) return nil, err } @@ -521,7 +518,6 @@ func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpCl response, err := http.Get(rawURL) if err != nil { - logrus.Debug("error fetching raw file:", err) return nil, err } defer response.Body.Close() @@ -530,7 +526,6 @@ func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpCl } data, err := ioutil.ReadAll(response.Body) if err != nil { - logrus.Debug("error reading response body:", err) return nil, err } From b4a2ec26499da632ab7fa37de86077da4505b374 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 27 Oct 2023 20:27:36 -0300 Subject: [PATCH 11/27] adjust for message Signed-off-by: Johnatas --- release/rancher/rancher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index f48b5f90..ee018fcc 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -360,7 +360,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } - output += "check completed successfully" + "\n" + output += "check completed successfully \n" return output, nil } From 7fabcf8c1a1ed1a409863faf11d63b64739969c9 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 27 Oct 2023 20:28:57 -0300 Subject: [PATCH 12/27] adjust for message Signed-off-by: Johnatas --- release/rancher/rancher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index ee018fcc..2905ebae 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -360,7 +360,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } - output += "check completed successfully \n" + output += "check completed successfully" return output, nil } From 81ff6cf01b906f34a40c6c93df95aae6a476127b Mon Sep 17 00:00:00 2001 From: Johnatas Date: Mon, 30 Oct 2023 19:06:36 -0300 Subject: [PATCH 13/27] add template tag Signed-off-by: Johnatas --- cmd/rancher_release/check_rancher_rc_deps.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index d22f18d3..b546fa32 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -44,6 +44,12 @@ func checkRancherRCDepsCommand() *cli.Command { Usage: "files to be checked", Required: true, }, + &cli.BoolFlag{ + Name: "template", + Aliases: []string{"m"}, + Usage: "export as MD template", + Required: false, + }, }, Action: checkRancherRCDeps, } @@ -55,6 +61,7 @@ func checkRancherRCDeps(c *cli.Context) error { rcOrg := c.String("org") rcRepo := c.String("repo") rcFiles := c.String("files") + toTemplate := c.String("template") if rcCommit == "" && rcReleaseTitle == "" { return errors.New("'commit' or 'release-title' are required") From f6bd50dd34e555371c60298d36bd5d217beee22f Mon Sep 17 00:00:00 2001 From: Johnatas Date: Tue, 31 Oct 2023 12:59:35 -0300 Subject: [PATCH 14/27] add template Signed-off-by: Johnatas --- cmd/rancher_release/check_rancher_rc_deps.go | 10 +- release/rancher/rancher.go | 165 +++++++++++++++++-- 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index b546fa32..5667079d 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -45,9 +45,9 @@ func checkRancherRCDepsCommand() *cli.Command { Required: true, }, &cli.BoolFlag{ - Name: "template", - Aliases: []string{"m"}, - Usage: "export as MD template", + Name: "for-ci", + Aliases: []string{"p"}, + Usage: "instead export a md template run a check raising a error if contains rc tags and dev deps", Required: false, }, }, @@ -61,7 +61,7 @@ func checkRancherRCDeps(c *cli.Context) error { rcOrg := c.String("org") rcRepo := c.String("repo") rcFiles := c.String("files") - toTemplate := c.String("template") + forCi := c.Bool("for-ci") if rcCommit == "" && rcReleaseTitle == "" { return errors.New("'commit' or 'release-title' are required") @@ -72,7 +72,7 @@ func checkRancherRCDeps(c *cli.Context) error { logrus.Debugf("organization: %s, repository: %s, commit: %s, release title: %s, files: %s", rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) - output, err := rancher.CheckRancherFinalRCDeps(rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) + output, err := rancher.CheckRancherRCDeps(forCi, rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) if err != nil { return err } diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 2905ebae..4589a638 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -2,14 +2,17 @@ package rancher import ( "bufio" + "bytes" "context" "errors" "fmt" "io" "net/http" + "os" "regexp" "strconv" "strings" + "text/template" "time" "github.com/google/go-github/v39/github" @@ -285,7 +288,22 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa return err } -func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) (string, error) { +type LineContent struct { + Line int + File string + Content string + Tag string +} + +type Content struct { + RancherImages []LineContent + FilesWithRC []LineContent + MinFilesWithRC []LineContent + FilesWithDev []LineContent + ChartsKDM []LineContent +} + +func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files string) (string, error) { const ( releaseTitleRegex = `^Pre-release v2\.7\.[0-9]{1,100}-rc[1-9][0-9]{0,1}$` partialFinalRCCommitMessage = "last commit for final rc" @@ -295,6 +313,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) existsReleaseTitle bool badFiles bool output string + content Content ) devDependencyPattern := regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) @@ -308,6 +327,7 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) if org == "" { org = rancherOrg } + if commitHash != "" { commitData, err := repository.CommitInfo(org, repo, commitHash, &httpClient) if err != nil { @@ -324,29 +344,46 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) existsReleaseTitle = innerExistsReleaseTitle } + err := writeRancherImagesDeps(&content) + if err != err { + return "", err + } + if matchCommitMessage || existsReleaseTitle { for _, filePath := range strings.Split(files, ",") { - content, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) + repoContent, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) if err != nil { return "", err } - scanner := bufio.NewScanner(strings.NewReader(string(content))) + scanner := bufio.NewScanner(strings.NewReader(string(repoContent))) lineNum := 1 for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) lineByte := []byte(line) + writeMinVersionComponentsFromDockerfile(&content, lineNum, line, filePath) + if devDependencyPattern.Match(lineByte) { badFiles = true - logMessage := fmt.Sprintf("file: %s, line: %d, content: '%s' contains dev dependencies", filePath, lineNum, line) - output += logMessage + "\n" + lineContent := LineContent{ + File: filePath, + Line: lineNum, + Content: line, + Tag: "-rc", + } + content.FilesWithRC = append(content.FilesWithRC, lineContent) } if rcTagPattern.Match(lineByte) { badFiles = true - logMessage := fmt.Sprintf("file: %s, line: %d, content: '%s' contains rc tags", filePath, lineNum, line) - output += logMessage + "\n" + lineContent := LineContent{ + File: filePath, + Line: lineNum, + Content: line, + Tag: "dev-", + } + content.FilesWithDev = append(content.FilesWithDev, lineContent) } lineNum++ @@ -355,15 +392,121 @@ func CheckRancherFinalRCDeps(org, repo, commitHash, releaseTitle, files string) return "", err } } - if badFiles { - fmt.Println(output) + if forCi && badFiles { return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } - output += "check completed successfully" - return output, nil + tmpl := template.New("rancher-release-rc-dev-deps") + tmpl = template.Must(tmpl.Parse(checkRCDevDeps)) + buff := bytes.NewBuffer(nil) + err := tmpl.ExecuteTemplate(buff, "componentsFile", content) + if err != nil { + return "", err + } + + return buff.String(), nil } output += "skipped check" return output, nil } + +func writeRancherImagesDeps(content *Content) error { + imageFiles := []string{"./bin/rancher-images.txt", "./bin/rancher-windows-images.txt"} + + for _, file := range imageFiles { + lines, err := readLines(file) + if err != nil { + fmt.Println("Error reading file:", err) + return err + } + lineNumber := 1 + for _, line := range lines { + var lineContent LineContent + if strings.Contains(line, "-rc") { + lineContent = LineContent{ + Line: lineNumber, + File: line, + Content: "", + Tag: "-rc", + } + } + if strings.Contains(line, "dev-") { + lineContent = LineContent{Line: lineNumber, File: line, Content: "", Tag: "dev-"} + } + content.RancherImages = append(content.RancherImages, lineContent) + } + } + return nil +} + +func writeMinVersionComponentsFromDockerfile(content *Content, lineNumber int, line, filePath string) { + if strings.Contains(filePath, "./package/Dockerfile") { + matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) + if len(matches) == 2 && strings.Contains(line, "-rc") { + lineContent := LineContent{Line: lineNumber, File: filePath, Content: line, Tag: "min"} + content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) + } + } +} + +func readLines(filePath string) ([]string, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, scanner.Err() +} + +const checkRCDevDeps = `{{- define "componentsFile" -}} +# Images with -rc +{{range .Content.RancherImages}} +* {{ .File }} +{{- end}} + +rancher/backup-restore-operator v4.0.0-rc1 +rancher/fleet v0.9.0-rc.5 +rancher/fleet-agent v0.9.0-rc.5 +rancher/rancher v2.8.0-rc3 +rancher/rancher-agent v2.8.0-rc3 +rancher/system-agent v0.3.4-rc1-suc + +# Components with -rc +{{range .Content.FilesWithRC}} +* {{ .Line }} {{ .Tag }} +{{- end}} + +LI_VERSION v2.8.0-rc1 +RANCHER_WEBHOOK_VERSION 103.0.0+up0.4.0-rc9 +AKS-OPERATOR v1.2.0-rc4 +DYNAMICLISTENER v0.3.6-rc3-deadlock-fix-revert +EKS-OPERATOR v1.3.0-rc3 +GKE-OPERATOR v1.2.0-rc2 +RKE v1.5.0-rc5 +DASHBOARD_UI_VERSION v2.8.0-rc3 +FLEET_VERSION 103.1.0+up0.9.0-rc.5 +SYSTEM_AGENT_VERSION v0.3.4-rc1 +UI_VERSION 2.8.0-rc3 +RKE v1.5.0-rc9 + +# Min version components with -rc +{{range .Content.MinFilesWithRC}} +{{ .Line }} {{ .Tag }} +{{- end}} + +CSP_ADAPTER_MIN_VERSION 103.0.0+up3.0.0-rc1 +FLEET_MIN_VERSION 103.1.0+up0.9.0-rc.3 + +# Components with dev- +{{range .Content.FilesWithRC}} +* {{ .Line }} {{ .Tag }} +{{- end}} + +{{ end }}` From 4bcb215209a026712f16eb7086412fd3fa8b6a20 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Wed, 1 Nov 2023 15:17:14 -0300 Subject: [PATCH 15/27] add more changes Signed-off-by: Johnatas --- cmd/rancher_release/check_rancher_rc_deps.go | 2 +- release/rancher/rancher.go | 116 +++++++------------ 2 files changed, 41 insertions(+), 77 deletions(-) diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 5667079d..9831d19e 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -47,7 +47,7 @@ func checkRancherRCDepsCommand() *cli.Command { &cli.BoolFlag{ Name: "for-ci", Aliases: []string{"p"}, - Usage: "instead export a md template run a check raising a error if contains rc tags and dev deps", + Usage: "export a md template also check raising a error if contains rc tags and dev deps", Required: false, }, }, diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 4589a638..eedc4b60 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -288,19 +288,17 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa return err } -type LineContent struct { +type ContentLine struct { Line int File string Content string - Tag string } type Content struct { - RancherImages []LineContent - FilesWithRC []LineContent - MinFilesWithRC []LineContent - FilesWithDev []LineContent - ChartsKDM []LineContent + RancherImages []ContentLine + FilesWithRC []ContentLine + MinFilesWithRC []ContentLine + FilesWithDev []ContentLine } func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files string) (string, error) { @@ -312,7 +310,6 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s matchCommitMessage bool existsReleaseTitle bool badFiles bool - output string content Content ) @@ -344,8 +341,9 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s existsReleaseTitle = innerExistsReleaseTitle } + //should return data if executed in rancher project root path err := writeRancherImagesDeps(&content) - if err != err { + if err != nil { return "", err } @@ -367,34 +365,20 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s if devDependencyPattern.Match(lineByte) { badFiles = true - lineContent := LineContent{ - File: filePath, - Line: lineNum, - Content: line, - Tag: "-rc", - } + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} content.FilesWithRC = append(content.FilesWithRC, lineContent) } if rcTagPattern.Match(lineByte) { badFiles = true - lineContent := LineContent{ - File: filePath, - Line: lineNum, - Content: line, - Tag: "dev-", - } + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} content.FilesWithDev = append(content.FilesWithDev, lineContent) } - lineNum++ } if err := scanner.Err(); err != nil { return "", err } } - if forCi && badFiles { - return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") - } tmpl := template.New("rancher-release-rc-dev-deps") tmpl = template.Must(tmpl.Parse(checkRCDevDeps)) @@ -403,12 +387,17 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s if err != nil { return "", err } + output := buff.String() - return buff.String(), nil + if forCi && badFiles { + fmt.Println(output) + return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") + } + + return output, nil } - output += "skipped check" - return output, nil + return "skipped check", nil } func writeRancherImagesDeps(content *Content) error { @@ -417,39 +406,36 @@ func writeRancherImagesDeps(content *Content) error { for _, file := range imageFiles { lines, err := readLines(file) if err != nil { - fmt.Println("Error reading file:", err) - return err + continue } lineNumber := 1 for _, line := range lines { - var lineContent LineContent - if strings.Contains(line, "-rc") { - lineContent = LineContent{ - Line: lineNumber, - File: line, - Content: "", - Tag: "-rc", - } - } - if strings.Contains(line, "dev-") { - lineContent = LineContent{Line: lineNumber, File: line, Content: "", Tag: "dev-"} + if strings.Contains(line, "-rc") || strings.Contains(line, "dev-") { + lineContent := ContentLine{Line: lineNumber, File: file, Content: formatContentLine(line)} + content.RancherImages = append(content.RancherImages, lineContent) } - content.RancherImages = append(content.RancherImages, lineContent) + lineNumber++ } } return nil } func writeMinVersionComponentsFromDockerfile(content *Content, lineNumber int, line, filePath string) { - if strings.Contains(filePath, "./package/Dockerfile") { + if strings.Contains(filePath, "/package/Dockerfile") { matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) if len(matches) == 2 && strings.Contains(line, "-rc") { - lineContent := LineContent{Line: lineNumber, File: filePath, Content: line, Tag: "min"} + lineContent := ContentLine{Line: lineNumber, File: filePath, Content: formatContentLine(line)} content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) } } } +func formatContentLine(line string) string { + re := regexp.MustCompile(`\s+`) + line = re.ReplaceAllString(line, " ") + return strings.TrimSpace(line) +} + func readLines(filePath string) ([]string, error) { file, err := os.Open(filePath) if err != nil { @@ -466,47 +452,25 @@ func readLines(filePath string) ([]string, error) { } const checkRCDevDeps = `{{- define "componentsFile" -}} +{{- if .RancherImages }} # Images with -rc -{{range .Content.RancherImages}} -* {{ .File }} +{{range .RancherImages}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} {{- end}} - -rancher/backup-restore-operator v4.0.0-rc1 -rancher/fleet v0.9.0-rc.5 -rancher/fleet-agent v0.9.0-rc.5 -rancher/rancher v2.8.0-rc3 -rancher/rancher-agent v2.8.0-rc3 -rancher/system-agent v0.3.4-rc1-suc # Components with -rc -{{range .Content.FilesWithRC}} -* {{ .Line }} {{ .Tag }} +{{range .FilesWithRC}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) {{- end}} -LI_VERSION v2.8.0-rc1 -RANCHER_WEBHOOK_VERSION 103.0.0+up0.4.0-rc9 -AKS-OPERATOR v1.2.0-rc4 -DYNAMICLISTENER v0.3.6-rc3-deadlock-fix-revert -EKS-OPERATOR v1.3.0-rc3 -GKE-OPERATOR v1.2.0-rc2 -RKE v1.5.0-rc5 -DASHBOARD_UI_VERSION v2.8.0-rc3 -FLEET_VERSION 103.1.0+up0.9.0-rc.5 -SYSTEM_AGENT_VERSION v0.3.4-rc1 -UI_VERSION 2.8.0-rc3 -RKE v1.5.0-rc9 - # Min version components with -rc -{{range .Content.MinFilesWithRC}} -{{ .Line }} {{ .Tag }} +{{range .MinFilesWithRC}} +* {{ .Content }} {{- end}} -CSP_ADAPTER_MIN_VERSION 103.0.0+up3.0.0-rc1 -FLEET_MIN_VERSION 103.1.0+up0.9.0-rc.3 - # Components with dev- -{{range .Content.FilesWithRC}} -* {{ .Line }} {{ .Tag }} +{{range .FilesWithRC}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) {{- end}} - {{ end }}` From aae58999c5a4dca7147fa6877536f84939088922 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Tue, 14 Nov 2023 18:16:30 -0300 Subject: [PATCH 16/27] using regex for rancher txt files Signed-off-by: Johnatas --- release/rancher/rancher.go | 110 ++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 57 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 37946c3b..2cb1214f 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -107,6 +107,30 @@ if [ "${DRY_RUN}" = false ]; then fi` ) +const templateCheckRCDevDeps = `{{- define "componentsFile" -}} +{{- if .RancherImages }} +# Images with -rc +{{range .RancherImages}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} +{{- end}} + +# Components with -rc +{{range .FilesWithRC}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} + +# Min version components with -rc +{{range .MinFilesWithRC}} +* {{ .Content }} +{{- end}} + +# Components with dev- +{{range .FilesWithRC}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} +{{ end }}` + const ( rancherRepo = "rancher" rancherOrg = rancherRepo @@ -363,7 +387,7 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s } //should return data if executed in rancher project root path - err := writeRancherImagesDeps(&content) + err := writeRancherImagesDeps(&content, rcTagPattern) if err != nil { return "", err } @@ -379,16 +403,23 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s lineNum := 1 for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - lineByte := []byte(line) - - writeMinVersionComponentsFromDockerfile(&content, lineNum, line, filePath) + lineByte, line := formatLineByte(scanner.Text()) if devDependencyPattern.Match(lineByte) { badFiles = true lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} content.FilesWithRC = append(content.FilesWithRC, lineContent) } + if strings.Contains(filePath, "/package/Dockerfile") { + if !strings.Contains(line, "_VERSION") { + continue + } + matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) + if len(matches) == 2 && strings.Contains(line, "-rc") { + lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)} + content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) + } + } if rcTagPattern.Match(lineByte) { badFiles = true lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} @@ -402,7 +433,7 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s } tmpl := template.New("rancher-release-rc-dev-deps") - tmpl = template.Must(tmpl.Parse(checkRCDevDeps)) + tmpl = template.Must(tmpl.Parse(templateCheckRCDevDeps)) buff := bytes.NewBuffer(nil) err := tmpl.ExecuteTemplate(buff, "componentsFile", content) if err != nil { @@ -421,34 +452,33 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s return "skipped check", nil } -func writeRancherImagesDeps(content *Content) error { +func writeRancherImagesDeps(content *Content, rcTagPattern *regexp.Regexp) error { + // files below were generated in build time into rancher path imageFiles := []string{"./bin/rancher-images.txt", "./bin/rancher-windows-images.txt"} for _, file := range imageFiles { - lines, err := readLines(file) + filePath, err := os.Open(file) if err != nil { continue } + defer filePath.Close() + + scanner := bufio.NewScanner(filePath) lineNumber := 1 - for _, line := range lines { - if strings.Contains(line, "-rc") || strings.Contains(line, "dev-") { + for scanner.Scan() { + lineByte, line := formatLineByte(scanner.Text()) + + if rcTagPattern.Match(lineByte) { lineContent := ContentLine{Line: lineNumber, File: file, Content: formatContentLine(line)} content.RancherImages = append(content.RancherImages, lineContent) } lineNumber++ } - } - return nil -} - -func writeMinVersionComponentsFromDockerfile(content *Content, lineNumber int, line, filePath string) { - if strings.Contains(filePath, "/package/Dockerfile") { - matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) - if len(matches) == 2 && strings.Contains(line, "-rc") { - lineContent := ContentLine{Line: lineNumber, File: filePath, Content: formatContentLine(line)} - content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) + if err := scanner.Err(); err != nil { + return err } } + return nil } func formatContentLine(line string) string { @@ -457,41 +487,7 @@ func formatContentLine(line string) string { return strings.TrimSpace(line) } -func readLines(filePath string) ([]string, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - - var lines []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - return lines, scanner.Err() +func formatLineByte(line string) ([]byte, string) { + line = strings.TrimSpace(line) + return []byte(line), line } - -const checkRCDevDeps = `{{- define "componentsFile" -}} -{{- if .RancherImages }} -# Images with -rc -{{range .RancherImages}} -* {{ .Content }} ({{ .File }}, line {{ .Line }}) -{{- end}} -{{- end}} - -# Components with -rc -{{range .FilesWithRC}} -* {{ .Content }} ({{ .File }}, line {{ .Line }}) -{{- end}} - -# Min version components with -rc -{{range .MinFilesWithRC}} -* {{ .Content }} -{{- end}} - -# Components with dev- -{{range .FilesWithRC}} -* {{ .Content }} ({{ .File }}, line {{ .Line }}) -{{- end}} -{{ end }}` From e3b512e6a8c5c16f4ecfb2bb2515f9edd69c9110 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Wed, 15 Nov 2023 00:07:51 -0300 Subject: [PATCH 17/27] add docs Signed-off-by: Johnatas --- cmd/rancher_release/README.md | 64 +++++++++++ cmd/rancher_release/check_rancher_rc_deps.go | 21 +--- release/rancher/rancher.go | 107 ++++++++----------- 3 files changed, 114 insertions(+), 78 deletions(-) diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md index ce9459b3..14ea2546 100644 --- a/cmd/rancher_release/README.md +++ b/cmd/rancher_release/README.md @@ -142,6 +142,70 @@ rancher_release label-issues -t v2.8.1-rc1 --dry-run # [Waiting for RC] -> [To Test] ``` +### check-rancher-rc-deps + +This command checks Rancher by the commit hash in the selected files, verifying if they contain 'rc' and 'dev' dependencies. It generates an MD-formatted file print that can be used as a release description. If necessary, the command can generate an error if these dependencies are found, ideal for use in CI pipelines. When executed in the root of the Rancher project, it checks the `rancher-images.txt` and `rancher-windows-images.txt` files. + +The pattern of files to be checked includes: +- `pkg/settings/setting.go` +- `package/Dockerfile` +- `scripts/package-env` +- `Dockerfile.dapper` +- `go.mod` +- `pkg/apis/go.mod` +- `pkg/client/go.mod` + +| **Flag** | **Description** | **Required** | +| ---------------- | ----------------------------------------------------------------------------------------------------- | ------------ | +| `commit`, `c` | Required commit to find the Rancher project reference that will be executed | TRUE | +| `org`, `o` | Reference organization of the commit | FALSE | +| `repo`, `r` | Reference repository of the commit | FALSE | +| `files`, `f` | List of files to be checked by the command, they are mandatory | TRUE | +| `for-ci`, `p` | With this flag, it's possible to return an error if any of the files contain 'rc' tags or 'dev' dependencies, ideal for use in integration pipelines | FALSE | + +**Examples** + +``` +rancher_release check-rancher-rc-deps -c -f Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env +``` + +``` +# Images with -rc + +* rancher/backup-restore-operator v4.0.0-rc1 (./bin/rancher-images.txt, line 1) +* rancher/rancher v2.8.0-rc3 (./bin/rancher-windows-images.txt, line 1) +* rancher/rancher-agent v2.8.0-rc3 (./bin/rancher-windows-images.txt, line 2) +* rancher/system-agent v0.3.4-rc1-suc (./bin/rancher-windows-images.txt, line 3) + +# Components with -rc + +* ENV CATTLE_KDM_BRANCH=dev-v2.8 (Dockerfile.dapper, line 16) +* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ARG CATTLE_KDM_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* KDMBranch = NewSetting("kdm-branch", "dev-v2.8") (/pkg/settings/setting.go, line 84) +* ChartDefaultBranch = NewSetting("chart-default-branch", "dev-v2.8") (/pkg/settings/setting.go, line 116) +* SYSTEM_CHART_DEFAULT_BRANCH=${SYSTEM_CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 5) +* CHART_DEFAULT_BRANCH=${CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 7) + +# Min version components with -rc + +* ENV CATTLE_FLEET_MIN_VERSION=103.1.0+up0.9.0-rc.3 +* ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1 + +# Components with dev- + +* ENV CATTLE_KDM_BRANCH=dev-v2.8 (Dockerfile.dapper, line 16) +* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ARG CATTLE_KDM_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* KDMBranch = NewSetting("kdm-branch", "dev-v2.8") (/pkg/settings/setting.go, line 84) +* ChartDefaultBranch = NewSetting("chart-default-branch", "dev-v2.8") (/pkg/settings/setting.go, line 116) +* SYSTEM_CHART_DEFAULT_BRANCH=${SYSTEM_CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 5) +* CHART_DEFAULT_BRANCH=${CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 7) + +``` + ## Contributions - File Issue with details of the problem, feature request, etc. diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 9831d19e..cb21eca7 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -2,7 +2,6 @@ package main import ( "errors" - "fmt" "github.com/rancher/ecm-distro-tools/release/rancher" "github.com/sirupsen/logrus" @@ -20,12 +19,6 @@ func checkRancherRCDepsCommand() *cli.Command { Usage: "last commit for a final rc", Required: false, }, - &cli.StringFlag{ - Name: "release-title", - Aliases: []string{"t"}, - Usage: "release title from a given release process", - Required: false, - }, &cli.StringFlag{ Name: "org", Aliases: []string{"o"}, @@ -56,28 +49,24 @@ func checkRancherRCDepsCommand() *cli.Command { } func checkRancherRCDeps(c *cli.Context) error { - rcReleaseTitle := c.String("release-title") rcCommit := c.String("commit") rcOrg := c.String("org") rcRepo := c.String("repo") rcFiles := c.String("files") forCi := c.Bool("for-ci") - if rcCommit == "" && rcReleaseTitle == "" { - return errors.New("'commit' or 'release-title' are required") + if rcCommit == "" { + return errors.New("'commit hash' are required") } if rcFiles == "" { return errors.New("'files' is required, e.g, --files Dockerfile.dapper,go.mod") } - logrus.Debugf("organization: %s, repository: %s, commit: %s, release title: %s, files: %s", - rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) + logrus.Debugf("organization: %s, repository: %s, commit: %s, files: %s", + rcOrg, rcRepo, rcCommit, rcFiles) - output, err := rancher.CheckRancherRCDeps(forCi, rcOrg, rcRepo, rcCommit, rcReleaseTitle, rcFiles) + err := rancher.CheckRancherRCDeps(forCi, rcOrg, rcRepo, rcCommit, rcFiles) if err != nil { return err } - - fmt.Println(output) - return nil } diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 2cb1214f..4cb65e0a 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -346,14 +346,10 @@ type Content struct { FilesWithDev []ContentLine } -func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files string) (string, error) { - const ( - releaseTitleRegex = `^Pre-release v2\.7\.[0-9]{1,100}-rc[1-9][0-9]{0,1}$` - partialFinalRCCommitMessage = "last commit for final rc" - ) +func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { + const partialFinalRCCommitMessage = "commit for final rc" var ( matchCommitMessage bool - existsReleaseTitle bool badFiles bool content Content ) @@ -369,87 +365,74 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, releaseTitle, files s if org == "" { org = rancherOrg } - if commitHash != "" { commitData, err := repository.CommitInfo(org, repo, commitHash, &httpClient) if err != nil { - return "", err + return err } matchCommitMessage = strings.Contains(commitData.Message, partialFinalRCCommitMessage) - - } - if releaseTitle != "" { - innerExistsReleaseTitle, err := regexp.MatchString(releaseTitleRegex, releaseTitle) - if err != nil { - return "", err - } - existsReleaseTitle = innerExistsReleaseTitle } //should return data if executed in rancher project root path err := writeRancherImagesDeps(&content, rcTagPattern) if err != nil { - return "", err + return err } - if matchCommitMessage || existsReleaseTitle { - for _, filePath := range strings.Split(files, ",") { - repoContent, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) - if err != nil { - return "", err - } + for _, filePath := range strings.Split(files, ",") { + repoContent, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) + if err != nil { + return err + } - scanner := bufio.NewScanner(strings.NewReader(string(repoContent))) - lineNum := 1 + scanner := bufio.NewScanner(strings.NewReader(string(repoContent))) + lineNum := 1 - for scanner.Scan() { - lineByte, line := formatLineByte(scanner.Text()) + for scanner.Scan() { + lineByte, line := formatLineByte(scanner.Text()) - if devDependencyPattern.Match(lineByte) { - badFiles = true - lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} - content.FilesWithRC = append(content.FilesWithRC, lineContent) - } - if strings.Contains(filePath, "/package/Dockerfile") { - if !strings.Contains(line, "_VERSION") { - continue - } - matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) - if len(matches) == 2 && strings.Contains(line, "-rc") { - lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)} - content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) - } + if devDependencyPattern.Match(lineByte) { + badFiles = true + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} + content.FilesWithRC = append(content.FilesWithRC, lineContent) + } + if strings.Contains(filePath, "/package/Dockerfile") { + if !strings.Contains(line, "_VERSION") { + continue } - if rcTagPattern.Match(lineByte) { - badFiles = true - lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} - content.FilesWithDev = append(content.FilesWithDev, lineContent) + matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) + if len(matches) == 2 && strings.Contains(line, "-rc") { + lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)} + content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) } - lineNum++ } - if err := scanner.Err(); err != nil { - return "", err + if rcTagPattern.Match(lineByte) { + badFiles = true + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} + content.FilesWithDev = append(content.FilesWithDev, lineContent) } + lineNum++ } - - tmpl := template.New("rancher-release-rc-dev-deps") - tmpl = template.Must(tmpl.Parse(templateCheckRCDevDeps)) - buff := bytes.NewBuffer(nil) - err := tmpl.ExecuteTemplate(buff, "componentsFile", content) - if err != nil { - return "", err + if err := scanner.Err(); err != nil { + return err } - output := buff.String() + } - if forCi && badFiles { - fmt.Println(output) - return "", errors.New("check failed, some files don't match the expected dependencies for a final release candidate") - } + tmpl := template.New("rancher-release-rc-dev-deps") + tmpl = template.Must(tmpl.Parse(templateCheckRCDevDeps)) + buff := bytes.NewBuffer(nil) + err = tmpl.ExecuteTemplate(buff, "componentsFile", content) + if err != nil { + return err + } + + fmt.Println(buff.String()) - return output, nil + if forCi && matchCommitMessage && badFiles { + return errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } - return "skipped check", nil + return nil } func writeRancherImagesDeps(content *Content, rcTagPattern *regexp.Regexp) error { From 9422eacbc38c9693f8ef285b20793402302082a9 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 16 Nov 2023 10:07:33 -0300 Subject: [PATCH 18/27] Update rancher.go --- release/rancher/rancher.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 4cb65e0a..136de000 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -320,14 +320,12 @@ func SetChartBranchReferences(ctx context.Context, forkPath, rancherBaseBranch, } func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchName, forkOwner string, ghClient *github.Client) error { - pull := &github.NewPullRequest{ Title: github.String(title), Base: github.String(rancherBaseBranch), Head: github.String(forkOwner + ":" + branchName), MaintainerCanModify: github.Bool(true), } - _, _, err := ghClient.PullRequests.Create(ctx, "rancher", "rancher", pull) return err From 01b8712e11e17daa548facb5ab68effd548dcb75 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 17 Nov 2023 12:16:27 -0300 Subject: [PATCH 19/27] remove unsed commit validation Signed-off-by: Johnatas --- release/rancher/rancher.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 136de000..862e16dd 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -345,7 +345,6 @@ type Content struct { } func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { - const partialFinalRCCommitMessage = "commit for final rc" var ( matchCommitMessage bool badFiles bool @@ -363,13 +362,6 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { if org == "" { org = rancherOrg } - if commitHash != "" { - commitData, err := repository.CommitInfo(org, repo, commitHash, &httpClient) - if err != nil { - return err - } - matchCommitMessage = strings.Contains(commitData.Message, partialFinalRCCommitMessage) - } //should return data if executed in rancher project root path err := writeRancherImagesDeps(&content, rcTagPattern) From d47811b2f0b92182ae642ed7b68e0193d041a391 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 17 Nov 2023 16:52:07 -0300 Subject: [PATCH 20/27] split dev deps Signed-off-by: Johnatas --- release/rancher/rancher.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 862e16dd..a431db32 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -125,8 +125,13 @@ const templateCheckRCDevDeps = `{{- define "componentsFile" -}} * {{ .Content }} {{- end}} -# Components with dev- -{{range .FilesWithRC}} +# KDM References with dev branch +{{range .KDMWithDev}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} + +# Chart References with dev branch +{{range .ChartsWithDev}} * {{ .Content }} ({{ .File }}, line {{ .Line }}) {{- end}} {{ end }}` @@ -341,7 +346,9 @@ type Content struct { RancherImages []ContentLine FilesWithRC []ContentLine MinFilesWithRC []ContentLine - FilesWithDev []ContentLine + ChartsWithDev []ContentLine + KDMWithDev []ContentLine + WithDev []ContentLine } func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { @@ -384,7 +391,13 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { if devDependencyPattern.Match(lineByte) { badFiles = true lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} - content.FilesWithRC = append(content.FilesWithRC, lineContent) + lineContentLower := strings.ToLower(lineContent.Content) + if strings.Contains(lineContentLower, "chart") { + content.ChartsWithDev = append(content.ChartsWithDev, lineContent) + } + if strings.Contains(lineContentLower, "kdm") { + content.KDMWithDev = append(content.KDMWithDev, lineContent) + } } if strings.Contains(filePath, "/package/Dockerfile") { if !strings.Contains(line, "_VERSION") { @@ -399,7 +412,7 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { if rcTagPattern.Match(lineByte) { badFiles = true lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} - content.FilesWithDev = append(content.FilesWithDev, lineContent) + content.FilesWithRC = append(content.FilesWithRC, lineContent) } lineNum++ } From 9683343939253f8fe624fda56194b8423008352a Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 17 Nov 2023 17:50:57 -0300 Subject: [PATCH 21/27] update doc and requires Signed-off-by: Johnatas --- cmd/rancher_release/README.md | 40 ++++++++-------- cmd/rancher_release/check_rancher_rc_deps.go | 4 +- repository/repository.go | 50 -------------------- 3 files changed, 23 insertions(+), 71 deletions(-) diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md index 14ea2546..63e71387 100644 --- a/cmd/rancher_release/README.md +++ b/cmd/rancher_release/README.md @@ -170,40 +170,42 @@ rancher_release check-rancher-rc-deps -c -f Dockerfile.dapper,go.m ``` ``` -# Images with -rc - -* rancher/backup-restore-operator v4.0.0-rc1 (./bin/rancher-images.txt, line 1) -* rancher/rancher v2.8.0-rc3 (./bin/rancher-windows-images.txt, line 1) -* rancher/rancher-agent v2.8.0-rc3 (./bin/rancher-windows-images.txt, line 2) -* rancher/system-agent v0.3.4-rc1-suc (./bin/rancher-windows-images.txt, line 3) - # Components with -rc -* ENV CATTLE_KDM_BRANCH=dev-v2.8 (Dockerfile.dapper, line 16) -* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) -* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) -* ARG CATTLE_KDM_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) -* KDMBranch = NewSetting("kdm-branch", "dev-v2.8") (/pkg/settings/setting.go, line 84) -* ChartDefaultBranch = NewSetting("chart-default-branch", "dev-v2.8") (/pkg/settings/setting.go, line 116) -* SYSTEM_CHART_DEFAULT_BRANCH=${SYSTEM_CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 5) -* CHART_DEFAULT_BRANCH=${CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 7) +* github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.1.0-rc2 // needed for containers/image/v5 (go.mod, line 15) +* github.com/rancher/aks-operator v1.2.0-rc4 (go.mod, line 111) +* github.com/rancher/dynamiclistener v0.3.6-rc3-deadlock-fix-revert (go.mod, line 114) +* github.com/rancher/eks-operator v1.3.0-rc3 (go.mod, line 115) +* github.com/rancher/gke-operator v1.2.0-rc2 (go.mod, line 117) +* github.com/rancher/rke v1.5.0-rc5 (go.mod, line 124) +* github.com/opencontainers/image-spec v1.1.0-rc3 // indirect (go.mod, line 370) +* ENV CATTLE_RANCHER_WEBHOOK_VERSION=103.0.0+up0.4.0-rc9 (/package/Dockerfile, line 26) +* ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1 (/package/Dockerfile, line 27) +* ENV CATTLE_CLI_VERSION v2.8.0-rc1 (/package/Dockerfile, line 48) +* github.com/rancher/aks-operator v1.2.0-rc4 (/pkg/apis/go.mod, line 11) +* github.com/rancher/eks-operator v1.3.0-rc3 (/pkg/apis/go.mod, line 12) +* github.com/rancher/gke-operator v1.2.0-rc2 (/pkg/apis/go.mod, line 14) +* github.com/rancher/rke v1.5.0-rc5 (/pkg/apis/go.mod, line 16) +* ShellImage = NewSetting("shell-image", "rancher/shell:v0.1.21-rc1") (/pkg/settings/setting.go, line 121) # Min version components with -rc * ENV CATTLE_FLEET_MIN_VERSION=103.1.0+up0.9.0-rc.3 * ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1 -# Components with dev- +# KDM References with dev branch * ENV CATTLE_KDM_BRANCH=dev-v2.8 (Dockerfile.dapper, line 16) -* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) -* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) * ARG CATTLE_KDM_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) * KDMBranch = NewSetting("kdm-branch", "dev-v2.8") (/pkg/settings/setting.go, line 84) + +# Chart References with dev branch + +* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) * ChartDefaultBranch = NewSetting("chart-default-branch", "dev-v2.8") (/pkg/settings/setting.go, line 116) * SYSTEM_CHART_DEFAULT_BRANCH=${SYSTEM_CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 5) * CHART_DEFAULT_BRANCH=${CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 7) - ``` ## Contributions diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index cb21eca7..bb5e86d6 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -17,7 +17,7 @@ func checkRancherRCDepsCommand() *cli.Command { Name: "commit", Aliases: []string{"c"}, Usage: "last commit for a final rc", - Required: false, + Required: true, }, &cli.StringFlag{ Name: "org", @@ -40,7 +40,7 @@ func checkRancherRCDepsCommand() *cli.Command { &cli.BoolFlag{ Name: "for-ci", Aliases: []string{"p"}, - Usage: "export a md template also check raising a error if contains rc tags and dev deps", + Usage: "export a md template also check raising an error if contains rc tags and dev deps", Required: false, }, }, diff --git a/repository/repository.go b/repository/repository.go index 8ebe0e6b..62386dd9 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -2,7 +2,6 @@ package repository import ( "context" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -11,7 +10,6 @@ import ( "os/exec" "strconv" "strings" - "sync" "time" "github.com/go-git/go-git/v5" @@ -431,54 +429,6 @@ To find more information on specific steps, please see documentation [here](http - [ ] PJM: Close the milestone in GitHub. ` -type Author struct { - Name string `json:"name"` - Email string `json:"email"` - Date string `json:"date"` -} - -type Commit struct { - Author Author `json:"author"` - Committer Author `json:"committer"` - Message string `json:"message"` - URL string `json:"url"` -} - -type CommitResponse struct { - SHA string `json:"sha"` - NodeID string `json:"node_id"` - Commit Commit `json:"commit"` - URL string `json:"url"` - HTMLURL string `json:"html_url"` - CommentsURL string `json:"comments_url"` -} - -func CommitInfo(owner, repo, commitHash string, httpClient *http.Client) (*Commit, error) { - var commitResponseMutex sync.Mutex - - apiUrl := fmt.Sprintf(ghAPIURL+"/repos/%s/%s/commits/%s", owner, repo, commitHash) - - response, err := httpClient.Get(apiUrl) - if err != nil { - return nil, err - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return nil, errors.New("failed to fetch commit information. status code: " + strconv.Itoa(response.StatusCode)) - } - - var commitResponse CommitResponse - - commitResponseMutex.Lock() - err = json.NewDecoder(response.Body).Decode(&commitResponse) - commitResponseMutex.Unlock() - if err != nil { - return nil, err - } - - return &commitResponse.Commit, nil -} - func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpClient *http.Client) ([]byte, error) { rawURL := fmt.Sprintf(ghContentURL+"/%s/%s/%s/%s", owner, repo, commitHash, filePath) From 28b4198f5d4a8c155c50ff839e994cebedcb8d52 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Fri, 17 Nov 2023 17:54:26 -0300 Subject: [PATCH 22/27] remove unsed api url Signed-off-by: Johnatas --- release/rancher/rancher.go | 1 - repository/repository.go | 1 - 2 files changed, 2 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index a431db32..8c03331d 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -348,7 +348,6 @@ type Content struct { MinFilesWithRC []ContentLine ChartsWithDev []ContentLine KDMWithDev []ContentLine - WithDev []ContentLine } func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { diff --git a/repository/repository.go b/repository/repository.go index 62386dd9..20d1af23 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -25,7 +25,6 @@ const ( noneReleaseNote = "```release-note\r\nNONE\r\n```" httpTimeout = time.Second * 10 ghContentURL = "https://raw.githubusercontent.com" - ghAPIURL = "https://api.github.com" ) // stripBackportTag returns a string with a prefix backport tag removed From 1616d52c8bcf3fa203554ee93a9a0bed45255933 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 23 Nov 2023 18:39:01 -0300 Subject: [PATCH 23/27] more fixes Signed-off-by: Johnatas --- cmd/rancher_release/README.md | 16 ++- cmd/rancher_release/check_rancher_rc_deps.go | 23 +++- release/rancher/rancher.go | 112 ++++++++----------- repository/repository.go | 4 + 4 files changed, 77 insertions(+), 78 deletions(-) diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md index 63e71387..34793699 100644 --- a/cmd/rancher_release/README.md +++ b/cmd/rancher_release/README.md @@ -144,7 +144,7 @@ rancher_release label-issues -t v2.8.1-rc1 --dry-run ### check-rancher-rc-deps -This command checks Rancher by the commit hash in the selected files, verifying if they contain 'rc' and 'dev' dependencies. It generates an MD-formatted file print that can be used as a release description. If necessary, the command can generate an error if these dependencies are found, ideal for use in CI pipelines. When executed in the root of the Rancher project, it checks the `rancher-images.txt` and `rancher-windows-images.txt` files. +This command checks Rancher verifying if contains 'rc' and 'dev' dependencies for some files, this command could be used locally or remotely by commit hash. It generates an MD-formatted file print that can be used as a release description. If necessary, the command can generate an error if these dependencies are found, ideal for use in CI pipelines. The pattern of files to be checked includes: - `pkg/settings/setting.go` @@ -157,14 +157,18 @@ The pattern of files to be checked includes: | **Flag** | **Description** | **Required** | | ---------------- | ----------------------------------------------------------------------------------------------------- | ------------ | -| `commit`, `c` | Required commit to find the Rancher project reference that will be executed | TRUE | -| `org`, `o` | Reference organization of the commit | FALSE | -| `repo`, `r` | Reference repository of the commit | FALSE | -| `files`, `f` | List of files to be checked by the command, they are mandatory | TRUE | +| `commit`, `c` | Commit used to get all files during the check, required for remote execution | FALSE | +| `org`, `o` | Reference organization of the commit, as default `rancher` | FALSE | +| `repo`, `r` | Reference repository of the commit, as default `rancher` | FALSE | +| `files`, `f` | List of files to be checked by the command, required for remote execution | TRUE | | `for-ci`, `p` | With this flag, it's possible to return an error if any of the files contain 'rc' tags or 'dev' dependencies, ideal for use in integration pipelines | FALSE | **Examples** - +LOCAL +``` +rancher_release check-rancher-rc-deps +``` +REMOTE ``` rancher_release check-rancher-rc-deps -c -f Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env ``` diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index bb5e86d6..91a9af2a 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "github.com/rancher/ecm-distro-tools/release/rancher" @@ -17,25 +18,27 @@ func checkRancherRCDepsCommand() *cli.Command { Name: "commit", Aliases: []string{"c"}, Usage: "last commit for a final rc", - Required: true, + Required: false, }, &cli.StringFlag{ Name: "org", Aliases: []string{"o"}, Usage: "organization name", Required: false, + Value: "rancher", }, &cli.StringFlag{ Name: "repo", Aliases: []string{"r"}, Usage: "rancher repository", Required: false, + Value: "rancher", }, &cli.StringFlag{ Name: "files", Aliases: []string{"f"}, - Usage: "files to be checked", - Required: true, + Usage: "files to be checked if remotely", + Required: false, }, &cli.BoolFlag{ Name: "for-ci", @@ -49,6 +52,10 @@ func checkRancherRCDepsCommand() *cli.Command { } func checkRancherRCDeps(c *cli.Context) error { + const files = "/bin/rancher-images.txt,/bin/rancher-windows-images.txt,Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env" + + var local bool + rcCommit := c.String("commit") rcOrg := c.String("org") rcRepo := c.String("repo") @@ -56,15 +63,19 @@ func checkRancherRCDeps(c *cli.Context) error { forCi := c.Bool("for-ci") if rcCommit == "" { - return errors.New("'commit hash' are required") + if rcFiles != "" { + return errors.New("'commit hash' are required for remote operation") + } } if rcFiles == "" { - return errors.New("'files' is required, e.g, --files Dockerfile.dapper,go.mod") + rcFiles = files + local = true } + logrus.Debugf("organization: %s, repository: %s, commit: %s, files: %s", rcOrg, rcRepo, rcCommit, rcFiles) - err := rancher.CheckRancherRCDeps(forCi, rcOrg, rcRepo, rcCommit, rcFiles) + err := rancher.CheckRancherRCDeps(context.Background(), local, forCi, rcOrg, rcRepo, rcCommit, rcFiles) if err != nil { return err } diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 8c03331d..657ec4b5 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -108,12 +108,10 @@ fi` ) const templateCheckRCDevDeps = `{{- define "componentsFile" -}} -{{- if .RancherImages }} # Images with -rc {{range .RancherImages}} * {{ .Content }} ({{ .File }}, line {{ .Line }}) {{- end}} -{{- end}} # Components with -rc {{range .FilesWithRC}} @@ -136,11 +134,6 @@ const templateCheckRCDevDeps = `{{- define "componentsFile" -}} {{- end}} {{ end }}` -const ( - rancherRepo = "rancher" - rancherOrg = rancherRepo -) - type SetBranchReferencesArgs struct { RancherRepoPath string NewBranch string @@ -350,51 +343,51 @@ type Content struct { KDMWithDev []ContentLine } -func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { +func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commitHash, files string) error { var ( - matchCommitMessage bool - badFiles bool - content Content + content Content + badFiles bool ) devDependencyPattern := regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) rcTagPattern := regexp.MustCompile(`-rc[0-9]+`) - httpClient := ecmHTTP.NewClient(time.Second * 15) - - if repo == "" { - repo = rancherRepo - } - if org == "" { - org = rancherOrg - } - - //should return data if executed in rancher project root path - err := writeRancherImagesDeps(&content, rcTagPattern) - if err != nil { - return err - } + ghClient := repository.NewGithub(ctx, "") for _, filePath := range strings.Split(files, ",") { - repoContent, err := repository.ContentByFileNameAndCommit(org, repo, commitHash, filePath, &httpClient) - if err != nil { - return err + var scanner *bufio.Scanner + if local { + content, err := contentLocal(filePath) + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("file '%s' not found, skipping...", filePath) + continue + } + return err + } + defer content.Close() + scanner = bufio.NewScanner(content) + } else { + content, err := contentRemote(ctx, ghClient, org, repo, commitHash, filePath) + if err != nil { + return err + } + scanner = bufio.NewScanner(strings.NewReader(content)) } - scanner := bufio.NewScanner(strings.NewReader(string(repoContent))) lineNum := 1 for scanner.Scan() { - lineByte, line := formatLineByte(scanner.Text()) - - if devDependencyPattern.Match(lineByte) { - badFiles = true + line := scanner.Text() + if devDependencyPattern.MatchString(line) { lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} lineContentLower := strings.ToLower(lineContent.Content) if strings.Contains(lineContentLower, "chart") { + badFiles = true content.ChartsWithDev = append(content.ChartsWithDev, lineContent) } if strings.Contains(lineContentLower, "kdm") { + badFiles = true content.KDMWithDev = append(content.KDMWithDev, lineContent) } } @@ -404,11 +397,12 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { } matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) if len(matches) == 2 && strings.Contains(line, "-rc") { + badFiles = true lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)} content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) } } - if rcTagPattern.Match(lineByte) { + if rcTagPattern.MatchString(line) { badFiles = true lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} content.FilesWithRC = append(content.FilesWithRC, lineContent) @@ -423,47 +417,38 @@ func CheckRancherRCDeps(forCi bool, org, repo, commitHash, files string) error { tmpl := template.New("rancher-release-rc-dev-deps") tmpl = template.Must(tmpl.Parse(templateCheckRCDevDeps)) buff := bytes.NewBuffer(nil) - err = tmpl.ExecuteTemplate(buff, "componentsFile", content) + err := tmpl.ExecuteTemplate(buff, "componentsFile", content) if err != nil { return err } fmt.Println(buff.String()) - if forCi && matchCommitMessage && badFiles { + if forCi && badFiles { return errors.New("check failed, some files don't match the expected dependencies for a final release candidate") } return nil } -func writeRancherImagesDeps(content *Content, rcTagPattern *regexp.Regexp) error { - // files below were generated in build time into rancher path - imageFiles := []string{"./bin/rancher-images.txt", "./bin/rancher-windows-images.txt"} - - for _, file := range imageFiles { - filePath, err := os.Open(file) - if err != nil { - continue - } - defer filePath.Close() - - scanner := bufio.NewScanner(filePath) - lineNumber := 1 - for scanner.Scan() { - lineByte, line := formatLineByte(scanner.Text()) +func contentLocal(filePath string) (*os.File, error) { + repoContent, err := os.Open(filePath) + if err != nil { + return nil, err + } + return repoContent, nil +} - if rcTagPattern.Match(lineByte) { - lineContent := ContentLine{Line: lineNumber, File: file, Content: formatContentLine(line)} - content.RancherImages = append(content.RancherImages, lineContent) - } - lineNumber++ - } - if err := scanner.Err(); err != nil { - return err - } +func contentRemote(ctx context.Context, ghClient *github.Client, org, repo, commitHash, filePath string) (string, error) { + content, _, _, err := ghClient.Repositories.GetContents(ctx, org, repo, filePath, &github.RepositoryContentGetOptions{Ref: commitHash}) + if err != nil { + return "", err } - return nil + decodedContent, err := content.GetContent() + if err != nil { + return "", err + } + return decodedContent, nil } func formatContentLine(line string) string { @@ -471,8 +456,3 @@ func formatContentLine(line string) string { line = re.ReplaceAllString(line, " ") return strings.TrimSpace(line) } - -func formatLineByte(line string) ([]byte, string) { - line = strings.TrimSpace(line) - return []byte(line), line -} diff --git a/repository/repository.go b/repository/repository.go index 20d1af23..0c7fafce 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -54,6 +54,10 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { // NewGithub creates a value of type github.Client pointer // with the given context and Github token. func NewGithub(ctx context.Context, token string) *github.Client { + if token == "" { + return github.NewClient(nil) + } + ts := TokenSource{ AccessToken: token, } From a8d361fc1f338efd693c1d6bedbf9f7dc7b6607c Mon Sep 17 00:00:00 2001 From: Johnatas Date: Thu, 23 Nov 2023 18:45:49 -0300 Subject: [PATCH 24/27] remove func contents Signed-off-by: Johnatas --- repository/repository.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/repository/repository.go b/repository/repository.go index 0c7fafce..b15c04dc 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -4,11 +4,8 @@ import ( "context" "errors" "fmt" - "io/ioutil" - "net/http" "os" "os/exec" - "strconv" "strings" "time" @@ -431,22 +428,3 @@ To find more information on specific steps, please see documentation [here](http - [ ] QA: Final validation of above PR and tracked through the linked ticket - [ ] PJM: Close the milestone in GitHub. ` - -func ContentByFileNameAndCommit(owner, repo, commitHash, filePath string, httpClient *http.Client) ([]byte, error) { - rawURL := fmt.Sprintf(ghContentURL+"/%s/%s/%s/%s", owner, repo, commitHash, filePath) - - response, err := http.Get(rawURL) - if err != nil { - return nil, err - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return nil, errors.New("failed to fetch raw file. status code: " + strconv.Itoa(response.StatusCode)) - } - data, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - - return data, nil -} From 04eb84f0da56c551ac507f777194629226a4d6da Mon Sep 17 00:00:00 2001 From: Johnatas Date: Mon, 27 Nov 2023 20:27:00 -0300 Subject: [PATCH 25/27] more adjusts --- cmd/rancher_release/README.md | 2 +- cmd/rancher_release/check_rancher_rc_deps.go | 8 ++------ release/rancher/rancher.go | 11 +++++------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md index 34793699..ddf1d48e 100644 --- a/cmd/rancher_release/README.md +++ b/cmd/rancher_release/README.md @@ -160,7 +160,7 @@ The pattern of files to be checked includes: | `commit`, `c` | Commit used to get all files during the check, required for remote execution | FALSE | | `org`, `o` | Reference organization of the commit, as default `rancher` | FALSE | | `repo`, `r` | Reference repository of the commit, as default `rancher` | FALSE | -| `files`, `f` | List of files to be checked by the command, required for remote execution | TRUE | +| `files`, `f` | List of files to be checked by the command | FALSE | | `for-ci`, `p` | With this flag, it's possible to return an error if any of the files contain 'rc' tags or 'dev' dependencies, ideal for use in integration pipelines | FALSE | **Examples** diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go index 91a9af2a..4e2a541b 100644 --- a/cmd/rancher_release/check_rancher_rc_deps.go +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "github.com/rancher/ecm-distro-tools/release/rancher" "github.com/sirupsen/logrus" @@ -62,13 +61,10 @@ func checkRancherRCDeps(c *cli.Context) error { rcFiles := c.String("files") forCi := c.Bool("for-ci") - if rcCommit == "" { - if rcFiles != "" { - return errors.New("'commit hash' are required for remote operation") - } - } if rcFiles == "" { rcFiles = files + } + if rcCommit == "" { local = true } diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 657ec4b5..53a23f77 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -357,7 +357,7 @@ func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commi for _, filePath := range strings.Split(files, ",") { var scanner *bufio.Scanner if local { - content, err := contentLocal(filePath) + content, err := contentLocal("./" + filePath) if err != nil { if os.IsNotExist(err) { logrus.Debugf("file '%s' not found, skipping...", filePath) @@ -368,6 +368,9 @@ func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commi defer content.Close() scanner = bufio.NewScanner(content) } else { + if strings.Contains(filePath, "bin") { + continue + } content, err := contentRemote(ctx, ghClient, org, repo, commitHash, filePath) if err != nil { return err @@ -392,11 +395,7 @@ func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commi } } if strings.Contains(filePath, "/package/Dockerfile") { - if !strings.Contains(line, "_VERSION") { - continue - } - matches := regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).FindStringSubmatch(line) - if len(matches) == 2 && strings.Contains(line, "-rc") { + if regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).MatchString(line) && strings.Contains(line, "-rc") { badFiles = true lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)} content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) From 1b965e60f648694edeee93b4ba1138efa7c59bae Mon Sep 17 00:00:00 2001 From: Johnatas Date: Tue, 28 Nov 2023 10:49:04 -0300 Subject: [PATCH 26/27] add line and content to min version Signed-off-by: Johnatas --- release/rancher/rancher.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 53a23f77..5660716d 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -120,8 +120,8 @@ const templateCheckRCDevDeps = `{{- define "componentsFile" -}} # Min version components with -rc {{range .MinFilesWithRC}} -* {{ .Content }} -{{- end}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} # KDM References with dev branch {{range .KDMWithDev}} @@ -382,6 +382,12 @@ func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commi for scanner.Scan() { line := scanner.Text() + if strings.Contains(filePath, "bin/rancher") { + badFiles = true + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} + content.RancherImages = append(content.RancherImages, lineContent) + continue + } if devDependencyPattern.MatchString(line) { lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} lineContentLower := strings.ToLower(lineContent.Content) From 0fae92fbcfdbbedd4e60cfc7bf44995aa8882af0 Mon Sep 17 00:00:00 2001 From: Johnatas Date: Tue, 28 Nov 2023 10:51:41 -0300 Subject: [PATCH 27/27] add rancher images export in example --- cmd/rancher_release/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md index ddf1d48e..e5806522 100644 --- a/cmd/rancher_release/README.md +++ b/cmd/rancher_release/README.md @@ -174,6 +174,15 @@ rancher_release check-rancher-rc-deps -c -f Dockerfile.dapper,go.m ``` ``` +# Images with -rc + +* rancher/backup-restore-operator v4.0.0-rc1 (/bin/rancher-images.txt, line 1) +* rancher/fleet v0.9.0-rc.5 (/bin/rancher-images.txt, line 1) +* rancher/fleet-agent v0.9.0-rc.5 (/bin/rancher-images.txt, line 1) +* rancher/rancher v2.8.0-rc3 (/bin/rancher-windows-images.txt, line 1) +* rancher/rancher-agent v2.8.0-rc3 (/bin/rancher-windows-images.txt, line 1) +* rancher/system-agent v0.3.4-rc1-suc (/bin/rancher-windows-images.txt, line 1) + # Components with -rc * github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.1.0-rc2 // needed for containers/image/v5 (go.mod, line 15)