diff --git a/cmd/generate.go b/cmd/generate.go index ee8759c..f4c808c 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -63,7 +63,7 @@ func Generate(startPath string, environments []string, validate bool) (component case "helm": generator = &generators.HelmGenerator{} case "static": - generator = &generators.StaticGenerator{} + generator = &generators.StaticGenerator{ StartPath: startPath} } return component.Generate(generator) diff --git a/cmd/generate_test.go b/cmd/generate_test.go index 3702417..5f66539 100644 --- a/cmd/generate_test.go +++ b/cmd/generate_test.go @@ -50,6 +50,21 @@ func TestGenerateYAML(t *testing.T) { checkComponentLengthsAgainstExpected(t, components, expectedLengths) } +func TestGenerateStaticRemoteYAML(t *testing.T) { + components, err := Generate("../testdata/generate-remote-static", []string{"common"}, false) + + expectedLengths := map[string]int{ + "keyvault-flexvolume": 5, + "keyvault-sub": 1372, + } + + assert.Nil(t, err) + assert.Equal(t, 2, len(components)) + + checkComponentLengthsAgainstExpected(t, components, expectedLengths) +} + + func TestGenerateWithHooks(t *testing.T) { _, err := Generate("../testdata/generate-hooks", []string{"prod"}, false) diff --git a/core/component.go b/core/component.go index e344924..6212c4e 100644 --- a/core/component.go +++ b/core/component.go @@ -4,9 +4,11 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/ioutil" "os" "os/exec" + "net/http" "path" "path/filepath" "sort" @@ -190,6 +192,44 @@ func (c *Component) afterInstall() (err error) { return c.ExecuteHook("after-install") } +// InstallRemoteStaticComponent installs a component by downloading the remote resource manifest +func (c *Component) InstallRemoteStaticComponent(componentPath string) (err error) { + if !IsValidRemoteComponentConfig(*c) { + return nil; + } + + componentsPath := path.Join(componentPath, "components/", c.Name) + + if err := os.MkdirAll(componentsPath, 0777); err != nil { + return err + } + + response, err := http.Get(c.Source); + + if err != nil { + return err + } + + // Write the downloaded resource manifest file + defer response.Body.Close() + out, err := os.Create(path.Join(componentsPath, c.Name + ".yaml")) + + if err != nil { + logger.Error(emoji.Sprintf(":no_entry_sign: Error occurred in install for component '%s'\nError: %s", c.Name, err)) + return err + } + + defer out.Close() + _, err = io.Copy(out, response.Body) + + if (err != nil) { + logger.Error(emoji.Sprintf(":no_entry_sign: Error occurred in writing manifest file for component '%s'\nError: %s", c.Name, err)) + return err + } + + return nil; +} + // InstallComponent installs the component (if needed) utilizing its Method. func (c *Component) InstallComponent(componentPath string) (err error) { if (c.ComponentType == "component" || len(c.ComponentType) == 0) && c.Method == "git" { @@ -208,6 +248,8 @@ func (c *Component) InstallComponent(componentPath string) (err error) { if err = CloneRepo(c.Source, c.Version, subcomponentPath, c.Branch); err != nil { return err } + } else if IsValidRemoteComponentConfig(*c) { + return c.InstallRemoteStaticComponent(componentPath) } return nil @@ -494,3 +536,46 @@ func (c *Component) GetAccessTokens() (tokens map[string]string, err error) { } return tokens, err } + +// GetStaticComponentPath returns the static path if a component is of type 'static', if not, it returns an empty string +func (c *Component) GetStaticComponentPath(startPath string)(componentPath string){ + if c.ComponentType != "static" { + return "" + } + + if IsValidRemoteComponentConfig(*c) { + return path.Join(startPath, "components", c.Name) + } + + return path.Join(c.PhysicalPath, c.Path) +} + + +// CreateDirectory a directory in the given path and reports no error if the directory already exists +func CreateDirectory(cmdDir string, dirPath string) (componentPath string, err error) { + + cmd := exec.Command("sh", "-c", "mkdir -p " + dirPath) + cmd.Dir = cmdDir + + output, err := cmd.CombinedOutput() + + if err != nil { + logger.Error(emoji.Sprintf(":no_entry_sign: Error occurred in creating directory for component\n")) + return "", err + } + if len(output) > 0 { + outstring := emoji.Sprintf(":mag_right: Completed creating directory for component\n") + logger.Trace(strings.TrimSpace(outstring)) + } + + return path.Join(cmdDir, dirPath), nil +} + +// IsValidRemoteComponentConfig checks if the given component configuration is valid for a remote component +func IsValidRemoteComponentConfig(c Component) (bool){ + return ( + (c.ComponentType == "static" && + c.Method == "http")) && + (strings.HasSuffix(c.Source, "yaml") || + strings.HasSuffix(c.Source, "yml")) +} diff --git a/core/component_test.go b/core/component_test.go index fd3a328..530eded 100644 --- a/core/component_test.go +++ b/core/component_test.go @@ -116,3 +116,73 @@ func TestWriteComponent(t *testing.T) { err = component.Write() assert.Nil(t, err) } + +func TestValidRemoteComponentConfig(t *testing.T) { + component := Component{ + ComponentType: "static", + Method: "http", + Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml", + } + + isValid := IsValidRemoteComponentConfig(component) + assert.True(t, isValid, "Component is remote static component.") +} + +func TestInvalidRemoteComponentSource(t *testing.T) { + component := Component{ + ComponentType: "static", + Method: "http", + Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer", + } + + isValid := IsValidRemoteComponentConfig(component) + assert.True(t, !isValid, "Component is remote static component.") +} + +func TestValidRemoteComponentURL(t *testing.T) { + component := Component{ + ComponentType: "static", + Method: "http", + Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml", + } + + isValid := IsValidRemoteComponentConfig(component) + assert.True(t, isValid, "Component is remote static component.") + + component.Source = "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yml" + isValid = IsValidRemoteComponentConfig(component) + assert.True(t, isValid, "Component is remote static component.") +} + +func TestInvalidValidRemoteComponentConfig(t *testing.T) { + + componentTypes := [3]string{"component", "helm", "static"} + methods := [3]string{"git", "helm", "local"} + component := Component{ + Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml", + } + + for _, componentType := range componentTypes { + for _, method := range methods { + component.ComponentType = componentType + component.Method = method + isValid := IsValidRemoteComponentConfig(component) + assert.True(t, !isValid, "Component is not a remote static component.") + } + } +} + + +func TestGetStaticComponentPath(t *testing.T) { + component := Component{ + Name: "kv-flexvol", + ComponentType: "static", + Method: "http", + Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml", + } + + expectedComponentPath := "../testdata/generate-remote-static/components/kv-flexvol" + componentPath := component.GetStaticComponentPath("../testdata/generate-remote-static") + + assert.Equal(t, expectedComponentPath, componentPath) +} \ No newline at end of file diff --git a/docs/component.md b/docs/component.md index 767df4f..32bc7c6 100644 --- a/docs/component.md +++ b/docs/component.md @@ -16,8 +16,10 @@ component with the following schema: - if `type: helm`: the component will use `helm template` to materialize the component using the specified config under `config` as the `values.yaml` file. - - if `type: static`: the component holds raw kubernetes manifest files in + - if `type: static`: + Option 1: the component holds raw kubernetes manifest files in `path`, these manifests will be copied to the generated output. + Option 2: when using `method: http` and `source: url`, the manifest file (.yaml) is downloaded and installed. Example: `source: https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml` - `method`: The method by which this component is sourced. Currently, only `git`, `helm`, and `local` are supported values. @@ -28,6 +30,7 @@ component with the following schema: `helm repo add foo && helm fetch foo/` - if `method: local`: Tells fabrikate to use the host filesystem as a means to find the component. + - if `method: http` and `type: static`: Tells fabrikate to download the manifest file (.yaml) from the `source`. - `source`: The source for this component. @@ -36,6 +39,7 @@ component with the following schema: - if `method: helm`: A URL to a helm repository (the url you would call `helm repo add` on). - if `method: local`: A local path to specify a local filesystem component. + - if `method: http` and `type: static`: A URL for a yaml file. - `path`: For some components, like ones generated with `helm`, the desired target of the component might not be located at the root of the repo. Path @@ -48,6 +52,7 @@ component with the following schema: `source`. - if `method: local`: the subdirectory on host filesystem where the component is located. + - if `method: http`: a path does not need to be specified - `version`: For git `method` components, this specifies a specific commit SHA hash that the component should be locked to, enabling you to lock the diff --git a/generators/static.go b/generators/static.go index e32c8b0..dc6cad5 100644 --- a/generators/static.go +++ b/generators/static.go @@ -11,13 +11,15 @@ import ( ) // StaticGenerator uses a static directory of resource manifests to create a rolled up multi-part manifest. -type StaticGenerator struct{} +type StaticGenerator struct{ + StartPath string +} // Generate iterates a static directory of resource manifests and creates a multi-part manifest. func (sg *StaticGenerator) Generate(component *core.Component) (manifest string, err error) { logger.Info(emoji.Sprintf(":truck: Generating component '%s' statically from path %s", component.Name, component.Path)) - staticPath := path.Join(component.PhysicalPath, component.Path) + staticPath := component.GetStaticComponentPath(sg.StartPath) staticFiles, err := ioutil.ReadDir(staticPath) if err != nil { logger.Error(fmt.Sprintf("error reading from directory %s", staticPath)) @@ -43,4 +45,4 @@ func (sg *StaticGenerator) Generate(component *core.Component) (manifest string, // Currently is a no-op, but could be extended to support remote static content (see #155) func (sg *StaticGenerator) Install(component *core.Component) (err error) { return nil -} +} \ No newline at end of file diff --git a/testdata/generate-remote-static/component.yaml b/testdata/generate-remote-static/component.yaml new file mode 100644 index 0000000..13d756c --- /dev/null +++ b/testdata/generate-remote-static/component.yaml @@ -0,0 +1,9 @@ +name: keyvault-flexvolume +type: static +path: ./manifests +subcomponents: + - name: "keyvault-sub" + source: https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml + method: http + type: static + path: "./tmp/keyvault-sub" \ No newline at end of file diff --git a/testdata/generate-remote-static/components/keyvault-sub/keyvault-sub.yaml b/testdata/generate-remote-static/components/keyvault-sub/keyvault-sub.yaml new file mode 100644 index 0000000..a053eb3 --- /dev/null +++ b/testdata/generate-remote-static/components/keyvault-sub/keyvault-sub.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kv +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + app: keyvault-flexvolume + name: keyvault-flexvolume + namespace: kv +spec: + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: keyvault-flexvolume + spec: + tolerations: + containers: + - name: flexvol-driver-installer + image: "mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v0.0.13" + imagePullPolicy: Always + resources: + requests: + cpu: 50m + memory: 100Mi + limits: + cpu: 50m + memory: 100Mi + env: + # if you have used flex before on your cluster, use same directory + # set TARGET_DIR env var and mount the same directory to to the container + - name: TARGET_DIR + value: "/etc/kubernetes/volumeplugins" + volumeMounts: + - mountPath: "/etc/kubernetes/volumeplugins" + name: volplugins + volumes: + - hostPath: + # Modify this directory if your nodes are using a different one + # default is "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" + # below is Azure default + path: "/etc/kubernetes/volumeplugins" + name: volplugins + nodeSelector: + beta.kubernetes.io/os: linux diff --git a/testdata/generate-remote-static/manifests/keyvault-flexvolume.yaml b/testdata/generate-remote-static/manifests/keyvault-flexvolume.yaml new file mode 100644 index 0000000..e69de29