Helm 3 Support (#296)

Helm 3 Support

- Added support for Helm 3
- Removed support for Helm 2
- For backwards compatibility of some existing Fabrikate definitions, the [stable](https://github.com/helm/charts/tree/master/stable) is automatically added as repository to the host Helm 3 repositories list during `fab install`

Co-authored-by: Michael Perel <[email protected]>
Co-authored-by: Evan Louie <[email protected]>

README.md

@@ -30,7 +30,7 @@ First, install the latest `fab` cli on your local machine from
 appropriate binary and placing `fab` in your path. The `fab` cli tool, `helm`,
 and `git` are the only tools you need to have installed.
 
-**NOTE** Currently fabrikate does not support Helm 3. Please use helm 2.16.1 (stable) instead when running fabrikate.
+**NOTE** Fabrikate supports Helm 3, do not use Helm 2.
 
 Let's walk through building an example Fabrikate definition to see how it works
 in practice. First off, let's create a directory for our cluster definition:

azure-pipelines.yml

@@ -15,6 +15,7 @@ variables:
   GOPATH: "$(system.defaultWorkingDirectory)/gopath" # Go workspace path
   modulePath: "$(GOPATH)/src/github.com/$(build.repository.name)" # Path to the module's code
   GO111MODULE: "on"
+  linterTimeout: "5m"
 
 steps:
   - script: |
@@ -30,16 +31,15 @@ steps:
     displayName: "Set up Go workspace"
 
   - script: |
-      HELM_URL=https://storage.googleapis.com/kubernetes-helm
-      HELM_TGZ=helm-v2.13.1-linux-amd64.tar.gz
+      HELM_URL=https://get.helm.sh
+      HELM_TGZ=helm-v3.1.2-linux-amd64.tar.gz
       wget -q ${HELM_URL}/${HELM_TGZ}
       tar xzfv ${HELM_TGZ}
       PATH=`pwd`/linux-amd64/:$PATH
-      helm init --client-only
     displayName: "Install helm"
 
   - script: |
-      curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.21.0
+      curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.25.0
     displayName: Install linter
 
   - script: |
@@ -49,7 +49,7 @@ steps:
     displayName: "Get Go dependencies"
 
   - script: |
-      golangci-lint run
+      golangci-lint run --timeout $(linterTimeout)
     workingDirectory: "$(modulePath)"
     displayName: "Lint"
 

cmd/generate_test.go

@@ -22,10 +22,10 @@ func TestGenerateJSON(t *testing.T) {
 	assert.Nil(t, err)
 
 	expectedLengths := map[string]int{
-		"elasticsearch":         14495,
-		"elasticsearch-curator": 2394,
-		"fluentd-elasticsearch": 20203,
-		"kibana":                1595,
+		"elasticsearch":         14477,
+		"elasticsearch-curator": 2390,
+		"fluentd-elasticsearch": 20230,
+		"kibana":                1590,
 		"static":                188,
 	}
 
@@ -39,8 +39,8 @@ func TestGenerateYAML(t *testing.T) {
 
 	expectedLengths := map[string]int{
 		"prometheus-grafana": 125,
-		"grafana":            8575,
-		"prometheus":         21401,
+		"grafana":            8552,
+		"prometheus":         28363,
 	}
 
 	assert.Nil(t, err)

cmd/install.go

@@ -24,8 +24,11 @@ func Install(path string) (err error) {
 		logger.Info(emoji.Sprintf(":mag: Using %s: %s", tool, path))
 	}
 
-	logger.Info(emoji.Sprintf(":point_right: Initializing Helm"))
-	if output, err := exec.Command("helm", "init", "--client-only").CombinedOutput(); err != nil {
+	// By default, Helm 2 used to have the "stable" repo. This is assumed
+	// to exist in Fabrikate as a non-http repo.
+	// See timeline/deprecation schedule: https://github.com/helm/charts
+	logger.Info(emoji.Sprintf(":point_right: Adding stable repository"))
+	if output, err := exec.Command("helm", "repo", "add", "stable", "https://kubernetes-charts.storage.googleapis.com").CombinedOutput(); err != nil {
 		logger.Error(emoji.Sprintf(":no_entry_sign: %s: %s", err, output))
 		return err
 	}

generators/helm.go

@@ -19,9 +19,8 @@ import (
 	"github.com/otiai10/copy"
 	"github.com/timfpark/yaml"
 
-	"k8s.io/helm/pkg/helm/environment"
-	"k8s.io/helm/pkg/helm/helmpath"
-	"k8s.io/helm/pkg/repo"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/repo"
 )
 
 // HelmGenerator provides 'helm generate' generator functionality to Fabrikate
@@ -167,7 +166,7 @@ func (hg *HelmGenerator) Generate(component *core.Component) (manifest string, e
 	overriddenValuesFileName := fmt.Sprintf("%s.yaml", randomString.String())
 	absOverriddenPath := path.Join(os.TempDir(), overriddenValuesFileName)
 	defer os.Remove(absOverriddenPath)
-	
+
 	logger.Debug(emoji.Sprintf(":pencil: Writing config %s to %s\n", configYaml, absOverriddenPath))
 	if err = ioutil.WriteFile(absOverriddenPath, configYaml, 0777); err != nil {
 		return "", err
@@ -185,7 +184,7 @@ func (hg *HelmGenerator) Generate(component *core.Component) (manifest string, e
 		return "", err
 	}
 	logger.Info(emoji.Sprintf(":memo: Running `helm template` on template '%s'", chartPath))
-	output, err := exec.Command("helm", "template", chartPath, "--values", absOverriddenPath, "--name", component.Name, "--namespace", namespace).CombinedOutput()
+	output, err := exec.Command("helm", "template", component.Name, chartPath, "--values", absOverriddenPath, "--namespace", namespace).CombinedOutput()
 	if err != nil {
 		logger.Error(fmt.Sprintf("helm template failed with:\n%s: %s", err, output))
 		return "", err
@@ -286,10 +285,13 @@ var hd = helmDownloader{}
 // places it in `into`. If `version` is blank, latest is automatically fetched.
 // -- `into` will be the dir containing Chart.yaml
 // The function will first look to leverage an existing Helm repo from the
-// repository file at $HELM_HOME/repositories.yaml.  If it fails to find
+// repository file at $HELM_REPOSITORY_CONFIG. If it fails to find
 // a repo there, it will add a temporary helm repo, fetch from it, and then remove
-// the temporary repo. This is a to get around a limitation in Helm 2.
+// the temporary repo. This was originally designed to get around a limitation
+// in Helm 2.
 // see: https://github.com/helm/helm/issues/4527
+// Now that Fabrikate uses Helm 3, downloading the chart before running
+// "helm template" is not necessary, so this approach could be redesigned in the future.
 func (hd *helmDownloader) downloadChart(repo, chart, version, into string) (err error) {
 	repoName, err := getRepoName(repo)
 	if err != nil {
@@ -321,7 +323,11 @@ func (hd *helmDownloader) downloadChart(repo, chart, version, into string) (err
 
 	// Fetch chart to random temp dir
 	chartName := fmt.Sprintf("%s/%s", repoName, chart)
-	randomDir := path.Join(os.TempDir(), repoName)
+	randomDir, err := ioutil.TempDir("", repoName)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(randomDir)
 	downloadVersion := "latest"
 	if version != "" {
 		downloadVersion = version
@@ -363,8 +369,8 @@ func updateHelmChartDep(chartPath string) (err error) {
 		Condition  string
 	}
 
-	// Contents of requirements.yaml for a helm chart
-	type helmRequirements struct {
+	// Contents of requirements.yaml or dependencies in Chart.yaml
+	type helmDependencies struct {
 		Dependencies []helmDependency
 	}
 
@@ -377,24 +383,29 @@ func updateHelmChartDep(chartPath string) (err error) {
 		absChartPath = asAbs
 	}
 
-	// Parse chart dependency repositories and add them if not present
-	requirementsYamlPath := path.Join(absChartPath, "requirements.yaml")
+	// Parse chart dependency repositories and add them if not present.
+	// For both api versions v1 and v2, if requirements.yaml has dependencies
+	// Chart.yaml's dependencies will be ignored.
+	dependenciesYamlPath := path.Join(absChartPath, "requirements.yaml")
+	if _, err := os.Stat(dependenciesYamlPath); err != nil {
+		dependenciesYamlPath = path.Join(absChartPath, "Chart.yaml")
+	}
 	addedDepRepoList := []string{}
-	if _, err := os.Stat(requirementsYamlPath); err == nil {
-		logger.Info(fmt.Sprintf("requirements.yaml found at '%s', ensuring repositories exist on helm client", requirementsYamlPath))
+	if _, err := os.Stat(dependenciesYamlPath); err == nil {
+		logger.Info(fmt.Sprintf("'%s' found at '%s', ensuring repositories exist on helm client", filepath.Base(dependenciesYamlPath), dependenciesYamlPath))
 
-		bytes, err := ioutil.ReadFile(requirementsYamlPath)
+		bytes, err := ioutil.ReadFile(dependenciesYamlPath)
 		if err != nil {
 			return err
 		}
 
-		requirementsYaml := helmRequirements{}
-		if err = yaml.Unmarshal(bytes, &requirementsYaml); err != nil {
+		dependenciesYaml := helmDependencies{}
+		if err = yaml.Unmarshal(bytes, &dependenciesYaml); err != nil {
 			return err
 		}
 
 		// Add each dependency repo with a temp name
-		for _, dep := range requirementsYaml.Dependencies {
+		for _, dep := range dependenciesYaml.Dependencies {
 			currentRepo, err := getRepoName(dep.Repository)
 			if err == nil {
 				logger.Info(emoji.Sprintf(":pencil: Helm dependency repo already present: %v", currentRepo))
@@ -425,16 +436,9 @@ func updateHelmChartDep(chartPath string) (err error) {
 		}
 	}
 
-	// Update dependencies -- Attempt twice; may fail the first time if running on
-	// a newly initialized ~/.helm directory because `helm serve` is typically
-	// not running and helm will attempt to fetch/cache all helm repositories
-	// during first run
 	logger.Info(emoji.Sprintf(":helicopter: Updating helm chart's dependencies for chart in '%s'", absChartPath))
 	if _, err := exec.Command("helm", "dependency", "update", chartPath).CombinedOutput(); err != nil {
-		if attempt2, err := exec.Command("helm", "dependency", "update", chartPath).CombinedOutput(); err != nil {
-			logger.Warn(emoji.Sprintf(":no_entry_sign: Updating chart dependencies failed for chart in '%s'; run `helm dependency update %s` for more error details.\n%s: %s", absChartPath, absChartPath, err, attempt2))
-			return err
-		}
+		return err
 	}
 
 	// Cleanup temp dependency repositories
@@ -455,19 +459,15 @@ func updateHelmChartDep(chartPath string) (err error) {
 // getRepoName returns the repo name for the provided url
 func getRepoName(url string) (string, error) {
 	logger.Info(emoji.Sprintf(":eyes: Looking for repo %v", url))
-	helmHome := os.Getenv(environment.HomeEnvVar)
-	if helmHome == "" {
-		helmHome = environment.DefaultHelmHome
-	}
-	a := helmpath.Home(helmHome)
-	f, err := repo.LoadRepositoriesFile(a.RepositoryFile())
+	helmEnvs := cli.New()
+	repoConfig := helmEnvs.RepositoryConfig
+	f, err := repo.LoadFile(repoConfig)
 	if err != nil {
 		return "", err
 	}
 	if len(f.Repositories) == 0 {
 		return "", fmt.Errorf("no repositories to show")
 	}
-
 	for _, re := range f.Repositories {
 		if strings.EqualFold(re.URL, url) {
 			logger.Info(emoji.Sprintf(":green_heart: %v matches repo %v", url, re.Name))

go.mod

@@ -3,23 +3,16 @@ module github.com/microsoft/fabrikate
 go 1.12
 
 require (
-	github.com/Masterminds/semver v1.4.2 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
-	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/google/go-github/v28 v28.0.1
 	github.com/google/uuid v1.1.1
 	github.com/kyokomi/emoji v2.1.0+incompatible
-	github.com/onsi/ginkgo v1.9.0 // indirect
-	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/otiai10/copy v1.0.1
 	github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect
 	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
+	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.4.0
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.1
 	github.com/timfpark/conjungo v1.0.1
 	github.com/timfpark/yaml v0.0.0-20190612232118-2e9e29c9df01
-	k8s.io/apimachinery v0.0.0-20190831074630-461753078381 // indirect
-	k8s.io/client-go v11.0.0+incompatible // indirect
-	k8s.io/helm v2.14.3+incompatible
+	helm.sh/helm/v3 v3.2.0
 )