Command to check Rancher dependencies for a RC (rancher#286)

* change repo files * func to check all deps and command * add test * last adjusts * improve code * change requires * remove necessity of repo and org Signed-off-by: Johnatas <[email protected]> * more improves Signed-off-by: Johnatas <[email protected]> * more adjusts Signed-off-by: Johnatas <[email protected]> * removing logrus Signed-off-by: Johnatas <[email protected]> * adjust for message Signed-off-by: Johnatas <[email protected]> * adjust for message Signed-off-by: Johnatas <[email protected]> * add template tag Signed-off-by: Johnatas <[email protected]> * add template Signed-off-by: Johnatas <[email protected]> * add more changes Signed-off-by: Johnatas <[email protected]> * using regex for rancher txt files Signed-off-by: Johnatas <[email protected]> * add docs Signed-off-by: Johnatas <[email protected]> * Update rancher.go * remove unsed commit validation Signed-off-by: Johnatas <[email protected]> * split dev deps Signed-off-by: Johnatas <[email protected]> * update doc and requires Signed-off-by: Johnatas <[email protected]> * remove unsed api url Signed-off-by: Johnatas <[email protected]> * more fixes Signed-off-by: Johnatas <[email protected]> * remove func contents Signed-off-by: Johnatas <[email protected]> * more adjusts * add line and content to min version Signed-off-by: Johnatas <[email protected]> * add rancher images export in example --------- Signed-off-by: Johnatas <[email protected]>
tashima42 · Dec 12, 2023 · 8685460 · 8685460
1 parent 5ce9bbc
commit 8685460
Show file tree

Hide file tree

Showing 5 changed files with 329 additions and 0 deletions.
diff --git a/cmd/rancher_release/README.md b/cmd/rancher_release/README.md
@@ -142,6 +142,85 @@ rancher_release label-issues -t v2.8.1-rc1 --dry-run
 #   [Waiting for RC] -> [To Test]
 ```
 
+### check-rancher-rc-deps
+
+This command checks Rancher verifying if contains 'rc' and 'dev' dependencies for some files, this command could be used locally or remotely by commit hash. It generates an MD-formatted file print that can be used as a release description. If necessary, the command can generate an error if these dependencies are found, ideal for use in CI pipelines. 
+
+The pattern of files to be checked includes:
+- `pkg/settings/setting.go`
+- `package/Dockerfile`
+- `scripts/package-env`
+- `Dockerfile.dapper`
+- `go.mod`
+- `pkg/apis/go.mod`
+- `pkg/client/go.mod`
+
+| **Flag**         | **Description**                                                                                       | **Required** |
+| ---------------- | ----------------------------------------------------------------------------------------------------- | ------------ |
+| `commit`, `c`    | Commit used to get all files during the check, required for remote execution                           | FALSE         |
+| `org`, `o`       | Reference organization of the commit, as default `rancher`                                                                                                                            | FALSE        |
+| `repo`, `r`      | Reference repository of the commit, as default `rancher`                                                                     | FALSE        |
+| `files`, `f`     | List of files to be checked by the command                                   | FALSE         |
+| `for-ci`, `p`    | With this flag, it's possible to return an error if any of the files contain 'rc' tags or 'dev' dependencies, ideal for use in integration pipelines | FALSE        |
+
+**Examples**
+LOCAL
+```
+rancher_release check-rancher-rc-deps
+```
+REMOTE
+```
+rancher_release check-rancher-rc-deps -c <HASH_COMMIT> -f Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env
+```
+
+```
+# Images with -rc
+
+* rancher/backup-restore-operator v4.0.0-rc1 (/bin/rancher-images.txt, line 1)
+* rancher/fleet v0.9.0-rc.5 (/bin/rancher-images.txt, line 1)
+* rancher/fleet-agent v0.9.0-rc.5 (/bin/rancher-images.txt, line 1)
+* rancher/rancher v2.8.0-rc3 (/bin/rancher-windows-images.txt, line 1)
+* rancher/rancher-agent v2.8.0-rc3 (/bin/rancher-windows-images.txt, line 1)
+* rancher/system-agent v0.3.4-rc1-suc (/bin/rancher-windows-images.txt, line 1)
+
+# Components with -rc
+
+* github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.1.0-rc2 // needed for containers/image/v5 (go.mod, line 15)
+* github.com/rancher/aks-operator v1.2.0-rc4 (go.mod, line 111)
+* github.com/rancher/dynamiclistener v0.3.6-rc3-deadlock-fix-revert (go.mod, line 114)
+* github.com/rancher/eks-operator v1.3.0-rc3 (go.mod, line 115)
+* github.com/rancher/gke-operator v1.2.0-rc2 (go.mod, line 117)
+* github.com/rancher/rke v1.5.0-rc5 (go.mod, line 124)
+* github.com/opencontainers/image-spec v1.1.0-rc3 // indirect (go.mod, line 370)
+* ENV CATTLE_RANCHER_WEBHOOK_VERSION=103.0.0+up0.4.0-rc9 (/package/Dockerfile, line 26)
+* ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1 (/package/Dockerfile, line 27)
+* ENV CATTLE_CLI_VERSION v2.8.0-rc1 (/package/Dockerfile, line 48)
+* github.com/rancher/aks-operator v1.2.0-rc4 (/pkg/apis/go.mod, line 11)
+* github.com/rancher/eks-operator v1.3.0-rc3 (/pkg/apis/go.mod, line 12)
+* github.com/rancher/gke-operator v1.2.0-rc2 (/pkg/apis/go.mod, line 14)
+* github.com/rancher/rke v1.5.0-rc5 (/pkg/apis/go.mod, line 16)
+* ShellImage = NewSetting("shell-image", "rancher/shell:v0.1.21-rc1") (/pkg/settings/setting.go, line 121)
+
+# Min version components with -rc
+
+* ENV CATTLE_FLEET_MIN_VERSION=103.1.0+up0.9.0-rc.3
+* ENV CATTLE_CSP_ADAPTER_MIN_VERSION=103.0.0+up3.0.0-rc1
+
+# KDM References with dev branch
+
+* ENV CATTLE_KDM_BRANCH=dev-v2.8 (Dockerfile.dapper, line 16)
+* ARG CATTLE_KDM_BRANCH=dev-v2.8 (/package/Dockerfile, line 1)
+* KDMBranch = NewSetting("kdm-branch", "dev-v2.8") (/pkg/settings/setting.go, line 84)
+
+# Chart References with dev branch
+
+* ARG SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1)
+* ARG CHART_DEFAULT_BRANCH=dev-v2.8 (/package/Dockerfile, line 1)
+* ChartDefaultBranch = NewSetting("chart-default-branch", "dev-v2.8") (/pkg/settings/setting.go, line 116)
+* SYSTEM_CHART_DEFAULT_BRANCH=${SYSTEM_CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 5)
+* CHART_DEFAULT_BRANCH=${CHART_DEFAULT_BRANCH:-"dev-v2.8"} (/scripts/package-env, line 7)
+```
+
 ## Contributions
 
 - File Issue with details of the problem, feature request, etc.

diff --git a/cmd/rancher_release/check_rancher_rc_deps.go b/cmd/rancher_release/check_rancher_rc_deps.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+
+	"github.com/rancher/ecm-distro-tools/release/rancher"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+func checkRancherRCDepsCommand() *cli.Command {
+	return &cli.Command{
+		Name:  "check-rancher-rc-deps",
+		Usage: "check if the Rancher version specified by the commit or pre-release title does not contain development dependencies and rc tags",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "commit",
+				Aliases:  []string{"c"},
+				Usage:    "last commit for a final rc",
+				Required: false,
+			},
+			&cli.StringFlag{
+				Name:     "org",
+				Aliases:  []string{"o"},
+				Usage:    "organization name",
+				Required: false,
+				Value:    "rancher",
+			},
+			&cli.StringFlag{
+				Name:     "repo",
+				Aliases:  []string{"r"},
+				Usage:    "rancher repository",
+				Required: false,
+				Value:    "rancher",
+			},
+			&cli.StringFlag{
+				Name:     "files",
+				Aliases:  []string{"f"},
+				Usage:    "files to be checked if remotely",
+				Required: false,
+			},
+			&cli.BoolFlag{
+				Name:     "for-ci",
+				Aliases:  []string{"p"},
+				Usage:    "export a md template also check raising an error if contains rc tags and dev deps",
+				Required: false,
+			},
+		},
+		Action: checkRancherRCDeps,
+	}
+}
+
+func checkRancherRCDeps(c *cli.Context) error {
+	const files = "/bin/rancher-images.txt,/bin/rancher-windows-images.txt,Dockerfile.dapper,go.mod,/package/Dockerfile,/pkg/apis/go.mod,/pkg/settings/setting.go,/scripts/package-env"
+
+	var local bool
+
+	rcCommit := c.String("commit")
+	rcOrg := c.String("org")
+	rcRepo := c.String("repo")
+	rcFiles := c.String("files")
+	forCi := c.Bool("for-ci")
+
+	if rcFiles == "" {
+		rcFiles = files
+	}
+	if rcCommit == "" {
+		local = true
+	}
+
+	logrus.Debugf("organization: %s, repository: %s, commit: %s, files: %s",
+		rcOrg, rcRepo, rcCommit, rcFiles)
+
+	err := rancher.CheckRancherRCDeps(context.Background(), local, forCi, rcOrg, rcRepo, rcCommit, rcFiles)
+	if err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/cmd/rancher_release/main.go b/cmd/rancher_release/main.go
@@ -31,6 +31,7 @@ func main() {
 		checkRancherImageCommand(),
 		setKDMBranchReferencesCommand(),
 		setChartsBranchReferencesCommand(),
+		checkRancherRCDepsCommand(),
 		labelIssuesCommand(),
 	}
 	app.Flags = rootFlags

diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go
@@ -2,12 +2,17 @@ package rancher
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"regexp"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/google/go-github/v39/github"
@@ -102,6 +107,33 @@ if [ "${DRY_RUN}" = false ]; then
 fi`
 )
 
+const templateCheckRCDevDeps = `{{- define "componentsFile" -}}
+# Images with -rc
+{{range .RancherImages}}
+* {{ .Content }} ({{ .File }}, line {{ .Line }})
+{{- end}}
+
+# Components with -rc
+{{range .FilesWithRC}}
+* {{ .Content }} ({{ .File }}, line {{ .Line }})
+{{- end}}
+
+# Min version components with -rc
+{{range .MinFilesWithRC}}
+* {{ .Content }} ({{ .File }}, line {{ .Line }})
+{{- end}} 
+
+# KDM References with dev branch
+{{range .KDMWithDev}}
+* {{ .Content }} ({{ .File }}, line {{ .Line }})
+{{- end}}
+
+# Chart References with dev branch
+{{range .ChartsWithDev}}
+* {{ .Content }} ({{ .File }}, line {{ .Line }})
+{{- end}}
+{{ end }}`
+
 type SetBranchReferencesArgs struct {
 	RancherRepoPath   string
 	NewBranch         string
@@ -296,3 +328,136 @@ func createPRFromRancher(ctx context.Context, rancherBaseBranch, title, branchNa
 
 	return err
 }
+
+type ContentLine struct {
+	Line    int
+	File    string
+	Content string
+}
+
+type Content struct {
+	RancherImages  []ContentLine
+	FilesWithRC    []ContentLine
+	MinFilesWithRC []ContentLine
+	ChartsWithDev  []ContentLine
+	KDMWithDev     []ContentLine
+}
+
+func CheckRancherRCDeps(ctx context.Context, local, forCi bool, org, repo, commitHash, files string) error {
+	var (
+		content  Content
+		badFiles bool
+	)
+
+	devDependencyPattern := regexp.MustCompile(`dev-v[0-9]+\.[0-9]+`)
+	rcTagPattern := regexp.MustCompile(`-rc[0-9]+`)
+
+	ghClient := repository.NewGithub(ctx, "")
+
+	for _, filePath := range strings.Split(files, ",") {
+		var scanner *bufio.Scanner
+		if local {
+			content, err := contentLocal("./" + filePath)
+			if err != nil {
+				if os.IsNotExist(err) {
+					logrus.Debugf("file '%s' not found, skipping...", filePath)
+					continue
+				}
+				return err
+			}
+			defer content.Close()
+			scanner = bufio.NewScanner(content)
+		} else {
+			if strings.Contains(filePath, "bin") {
+				continue
+			}
+			content, err := contentRemote(ctx, ghClient, org, repo, commitHash, filePath)
+			if err != nil {
+				return err
+			}
+			scanner = bufio.NewScanner(strings.NewReader(content))
+		}
+
+		lineNum := 1
+
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.Contains(filePath, "bin/rancher") {
+				badFiles = true
+				lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)}
+				content.RancherImages = append(content.RancherImages, lineContent)
+				continue
+			}
+			if devDependencyPattern.MatchString(line) {
+				lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)}
+				lineContentLower := strings.ToLower(lineContent.Content)
+				if strings.Contains(lineContentLower, "chart") {
+					badFiles = true
+					content.ChartsWithDev = append(content.ChartsWithDev, lineContent)
+				}
+				if strings.Contains(lineContentLower, "kdm") {
+					badFiles = true
+					content.KDMWithDev = append(content.KDMWithDev, lineContent)
+				}
+			}
+			if strings.Contains(filePath, "/package/Dockerfile") {
+				if regexp.MustCompile(`CATTLE_(\S+)_MIN_VERSION`).MatchString(line) && strings.Contains(line, "-rc") {
+					badFiles = true
+					lineContent := ContentLine{Line: lineNum, File: filePath, Content: formatContentLine(line)}
+					content.MinFilesWithRC = append(content.MinFilesWithRC, lineContent)
+				}
+			}
+			if rcTagPattern.MatchString(line) {
+				badFiles = true
+				lineContent := ContentLine{File: filePath, Line: lineNum, Content: formatContentLine(line)}
+				content.FilesWithRC = append(content.FilesWithRC, lineContent)
+			}
+			lineNum++
+		}
+		if err := scanner.Err(); err != nil {
+			return err
+		}
+	}
+
+	tmpl := template.New("rancher-release-rc-dev-deps")
+	tmpl = template.Must(tmpl.Parse(templateCheckRCDevDeps))
+	buff := bytes.NewBuffer(nil)
+	err := tmpl.ExecuteTemplate(buff, "componentsFile", content)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(buff.String())
+
+	if forCi && badFiles {
+		return errors.New("check failed, some files don't match the expected dependencies for a final release candidate")
+	}
+
+	return nil
+}
+
+func contentLocal(filePath string) (*os.File, error) {
+	repoContent, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	return repoContent, nil
+}
+
+func contentRemote(ctx context.Context, ghClient *github.Client, org, repo, commitHash, filePath string) (string, error) {
+	content, _, _, err := ghClient.Repositories.GetContents(ctx, org, repo, filePath, &github.RepositoryContentGetOptions{Ref: commitHash})
+	if err != nil {
+		return "", err
+	}
+	decodedContent, err := content.GetContent()
+	if err != nil {
+		return "", err
+	}
+	return decodedContent, nil
+}
+
+func formatContentLine(line string) string {
+	re := regexp.MustCompile(`\s+`)
+	line = re.ReplaceAllString(line, " ")
+	return strings.TrimSpace(line)
+}
diff --git a/repository/repository.go b/repository/repository.go
@@ -21,6 +21,7 @@ const (
 	emptyReleaseNote   = "```release-note\r\n\r\n```"
 	noneReleaseNote    = "```release-note\r\nNONE\r\n```"
 	httpTimeout        = time.Second * 10
+	ghContentURL       = "https://raw.githubusercontent.com"
 )
 
 // stripBackportTag returns a string with a prefix backport tag removed
@@ -50,6 +51,10 @@ func (t *TokenSource) Token() (*oauth2.Token, error) {
 // NewGithub creates a value of type github.Client pointer
 // with the given context and Github token.
 func NewGithub(ctx context.Context, token string) *github.Client {
+	if token == "" {
+		return github.NewClient(nil)
+	}
+
 	ts := TokenSource{
 		AccessToken: token,
 	}