diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md index ce9459b3..e5806522 100644 --- a/cmd/rancher_release/README.md +++ b/cmd/rancher_release/README.md @@ -142,6 +142,85 @@ rancher_release label-issues -t v2.8.1-rc1 --dry-run # [Waiting for RC] -> [To Test] ``` +### check-rancher-rc-deps + +This command checks Rancher verifying if contains 'rc' and 'dev' dependencies for some files, this command could be used locally or remotely by commit hash. It generates an MD-formatted file print that can be used as a release description. If necessary, the command can generate an error if these dependencies are found, ideal for use in CI pipelines. + +The pattern of files to be checked includes: +- `pkg/settings/setting.go` +- `package/Dockerfile` +- `scripts/package-env` +- `Dockerfile.dapper` +- `go.mod` +- `pkg/apis/go.mod` +- `pkg/client/go.mod` + +| **Flag** | **Description** | **Required** | +| ---------------- | ----------------------------------------------------------------------------------------------------- | ------------ | +| `commit`, `c` | Commit used to get all files during the check, required for remote execution | FALSE | +| `org`, `o` | Reference organization of the commit, as default `rancher` | FALSE | +| `repo`, `r` | Reference repository of the commit, as default `rancher` | FALSE | +| `files`, `f` | List of files to be checked by the command | FALSE | +| `for-ci`, `p` | With this flag, it's possible to return an error if any of the files contain 'rc' tags or 'dev' dependencies, ideal for use in integration pipelines | FALSE | + +**Examples** +LOCAL +``` +rancher_release check-rancher-rc-deps +``` +REMOTE +``` +rancher_release check-rancher-rc-deps -c -f Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env +``` + +``` +# Images with -rc + +* rancher/backup-restore-operator v4.0.0-rc1 (/bin/rancher-images.txt, line 1) +* rancher/fleet v0.9.0-rc.5 (/bin/rancher-images.txt, line 1) +* rancher/fleet-agent v0.9.0-rc.5 (/bin/rancher-images.txt, line 1) +* rancher/rancher v2.8.0-rc3 (/bin/rancher-windows-images.txt, line 1) +* rancher/rancher-agent v2.8.0-rc3 (/bin/rancher-windows-images.txt, line 1) +* rancher/system-agent v0.3.4-rc1-suc (/bin/rancher-windows-images.txt, line 1) + +# Components with -rc + +* github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.1.0-rc2 // needed for containers/image/v5 (go.mod, line 15) +* github.com/rancher/aks-operator v1.2.0-rc4 (go.mod, line 111) +* github.com/rancher/dynamiclistener v0.3.6-rc3-deadlock-fix-revert (go.mod, line 114) +* github.com/rancher/eks-operator v1.3.0-rc3 (go.mod, line 115) +* github.com/rancher/gke-operator v1.2.0-rc2 (go.mod, line 117) +* github.com/rancher/rke v1.5.0-rc5 (go.mod, line 124) +* github.com/opencontainers/image-spec v1.1.0-rc3 // indirect (go.mod, line 370) +* ENV CATTLE_RANCHER_WEBHOOK_VERSION=103.0.0+up0.4.0-rc9 (/package/Dockerfile, line 26) +* ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1 (/package/Dockerfile, line 27) +* ENV CATTLE_CLI_VERSION v2.8.0-rc1 (/package/Dockerfile, line 48) +* github.com/rancher/aks-operator v1.2.0-rc4 (/pkg/apis/go.mod, line 11) +* github.com/rancher/eks-operator v1.3.0-rc3 (/pkg/apis/go.mod, line 12) +* github.com/rancher/gke-operator v1.2.0-rc2 (/pkg/apis/go.mod, line 14) +* github.com/rancher/rke v1.5.0-rc5 (/pkg/apis/go.mod, line 16) +* ShellImage = NewSetting("shell-image", "rancher/shell:v0.1.21-rc1") (/pkg/settings/setting.go, line 121) + +# Min version components with -rc + +* ENV CATTLE_FLEET_MIN_VERSION=103.1.0+up0.9.0-rc.3 +* ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1 + +# KDM References with dev branch + +* ENV CATTLE_KDM_BRANCH=dev-v2.8 (Dockerfile.dapper, line 16) +* ARG CATTLE_KDM_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* KDMBranch = NewSetting("kdm-branch", "dev-v2.8") (/pkg/settings/setting.go, line 84) + +# Chart References with dev branch + +* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1) +* ChartDefaultBranch = NewSetting("chart-default-branch", "dev-v2.8") (/pkg/settings/setting.go, line 116) +* SYSTEM_CHART_DEFAULT_BRANCH=${SYSTEM_CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 5) +* CHART_DEFAULT_BRANCH=${CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 7) +``` + ## Contributions - File Issue with details of the problem, feature request, etc. diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go new file mode 100644 index 00000000..4e2a541b --- /dev/null +++ b/cmd/rancher_release/check_rancher_rc_deps.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + + "github.com/rancher/ecm-distro-tools/release/rancher" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +func checkRancherRCDepsCommand() *cli.Command { + return &cli.Command{ + Name: "check-rancher-rc-deps", + Usage: "check if the Rancher version specified by the commit or pre-release title does not contain development dependencies and rc tags", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "commit", + Aliases: []string{"c"}, + Usage: "last commit for a final rc", + Required: false, + }, + &cli.StringFlag{ + Name: "org", + Aliases: []string{"o"}, + Usage: "organization name", + Required: false, + Value: "rancher", + }, + &cli.StringFlag{ + Name: "repo", + Aliases: []string{"r"}, + Usage: "rancher repository", + Required: false, + Value: "rancher", + }, + &cli.StringFlag{ + Name: "files", + Aliases: []string{"f"}, + Usage: "files to be checked if remotely", + Required: false, + }, + &cli.BoolFlag{ + Name: "for-ci", + Aliases: []string{"p"}, + Usage: "export a md template also check raising an error if contains rc tags and dev deps", + Required: false, + }, + }, + Action: checkRancherRCDeps, + } +} + +func checkRancherRCDeps(c *cli.Context) error { + const files = "/bin/rancher-images.txt,/bin/rancher-windows-images.txt,Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env" + + var local bool + + rcCommit := c.String("commit") + rcOrg := c.String("org") + rcRepo := c.String("repo") + rcFiles := c.String("files") + forCi := c.Bool("for-ci") + + if rcFiles == "" { + rcFiles = files + } + if rcCommit == "" { + local = true + } + + logrus.Debugf("organization: %s, repository: %s, commit: %s, files: %s", + rcOrg, rcRepo, rcCommit, rcFiles) + + err := rancher.CheckRancherRCDeps(context.Background(), local, forCi, rcOrg, rcRepo, rcCommit, rcFiles) + if err != nil { + return err + } + return nil +} diff --git a/cmd/rancher_release/main.go b/cmd/rancher_release/main.go index aa253249..d2bfd995 100644 --- a/cmd/rancher_release/main.go +++ b/cmd/rancher_release/main.go @@ -31,6 +31,7 @@ func main() { checkRancherImageCommand(), setKDMBranchReferencesCommand(), setChartsBranchReferencesCommand(), + checkRancherRCDepsCommand(), labelIssuesCommand(), } app.Flags = rootFlags diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index 817935fe..5660716d 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -2,12 +2,17 @@ package rancher import ( "bufio" + "bytes" "context" "errors" + "fmt" "io" "net/http" + "os" + "regexp" "strconv" "strings" + "text/template" "time" "github.com/google/go-github/v39/github" @@ -102,6 +107,33 @@ if [ "${DRY_RUN}" = false ]; then fi` ) +const templateCheckRCDevDeps = `{{- define "componentsFile" -}} +# Images with -rc +{{range .RancherImages}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} + +# Components with -rc +{{range .FilesWithRC}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} + +# Min version components with -rc +{{range .MinFilesWithRC}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} + +# KDM References with dev branch +{{range .KDMWithDev}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} + +# Chart References with dev branch +{{range .ChartsWithDev}} +* {{ .Content }} ({{ .File }}, line {{ .Line }}) +{{- end}} +{{ end }}` + type SetBranchReferencesArgs struct { RancherRepoPath string NewBranch string @@ -296,3 +328,136 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa return err } + +type ContentLine struct { + Line int + File string + Content string +} + +type Content struct { + RancherImages []ContentLine + FilesWithRC []ContentLine + MinFilesWithRC []ContentLine + ChartsWithDev []ContentLine + KDMWithDev []ContentLine +} + +func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commitHash, files string) error { + var ( + content Content + badFiles bool + ) + + devDependencyPattern := regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`) + rcTagPattern := regexp.MustCompile(`-rc[0-9]+`) + + ghClient := repository.NewGithub(ctx, "") + + for _, filePath := range strings.Split(files, ",") { + var scanner *bufio.Scanner + if local { + content, err := contentLocal("./" + filePath) + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("file '%s' not found, skipping...", filePath) + continue + } + return err + } + defer content.Close() + scanner = bufio.NewScanner(content) + } else { + if strings.Contains(filePath, "bin") { + continue + } + content, err := contentRemote(ctx, ghClient, org, repo, commitHash, filePath) + if err != nil { + return err + } + scanner = bufio.NewScanner(strings.NewReader(content)) + } + + lineNum := 1 + + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(filePath, "bin/rancher") { + badFiles = true + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} + content.RancherImages = append(content.RancherImages, lineContent) + continue + } + if devDependencyPattern.MatchString(line) { + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} + lineContentLower := strings.ToLower(lineContent.Content) + if strings.Contains(lineContentLower, "chart") { + badFiles = true + content.ChartsWithDev = append(content.ChartsWithDev, lineContent) + } + if strings.Contains(lineContentLower, "kdm") { + badFiles = true + content.KDMWithDev = append(content.KDMWithDev, lineContent) + } + } + if strings.Contains(filePath, "/package/Dockerfile") { + if regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).MatchString(line) && strings.Contains(line, "-rc") { + badFiles = true + lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)} + content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent) + } + } + if rcTagPattern.MatchString(line) { + badFiles = true + lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)} + content.FilesWithRC = append(content.FilesWithRC, lineContent) + } + lineNum++ + } + if err := scanner.Err(); err != nil { + return err + } + } + + tmpl := template.New("rancher-release-rc-dev-deps") + tmpl = template.Must(tmpl.Parse(templateCheckRCDevDeps)) + buff := bytes.NewBuffer(nil) + err := tmpl.ExecuteTemplate(buff, "componentsFile", content) + if err != nil { + return err + } + + fmt.Println(buff.String()) + + if forCi && badFiles { + return errors.New("check failed, some files don't match the expected dependencies for a final release candidate") + } + + return nil +} + +func contentLocal(filePath string) (*os.File, error) { + repoContent, err := os.Open(filePath) + if err != nil { + return nil, err + } + return repoContent, nil +} + +func contentRemote(ctx context.Context, ghClient *github.Client, org, repo, commitHash, filePath string) (string, error) { + content, _, _, err := ghClient.Repositories.GetContents(ctx, org, repo, filePath, &github.RepositoryContentGetOptions{Ref: commitHash}) + if err != nil { + return "", err + } + decodedContent, err := content.GetContent() + if err != nil { + return "", err + } + return decodedContent, nil +} + +func formatContentLine(line string) string { + re := regexp.MustCompile(`\s+`) + line = re.ReplaceAllString(line, " ") + return strings.TrimSpace(line) +} diff --git a/repository/repository.go b/repository/repository.go index 96b6d3d9..b15c04dc 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -21,6 +21,7 @@ const ( emptyReleaseNote = "```release-note\r\n\r\n```" noneReleaseNote = "```release-note\r\nNONE\r\n```" httpTimeout = time.Second * 10 + ghContentURL = "https://raw.githubusercontent.com" ) // stripBackportTag returns a string with a prefix backport tag removed @@ -50,6 +51,10 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { // NewGithub creates a value of type github.Client pointer // with the given context and Github token. func NewGithub(ctx context.Context, token string) *github.Client { + if token == "" { + return github.NewClient(nil) + } + ts := TokenSource{ AccessToken: token, }