#249 Support remote urls for static components (#254)

* #249 Support remote urls for static components

* Incorporate feedback

* Add documentation for specifying a remote url for a component manifest file

* Add error handling for writing component file

* Add test files

* Rename method to http instead of remote-url

cmd/generate.go

@@ -63,7 +63,7 @@ func Generate(startPath string, environments []string, validate bool) (component
 		case "helm":
 			generator = &generators.HelmGenerator{}
 		case "static":
-			generator = &generators.StaticGenerator{}
+			generator = &generators.StaticGenerator{ StartPath: startPath}
 		}
 
 		return component.Generate(generator)

cmd/generate_test.go

@@ -50,6 +50,21 @@ func TestGenerateYAML(t *testing.T) {
 	checkComponentLengthsAgainstExpected(t, components, expectedLengths)
 }
 
+func TestGenerateStaticRemoteYAML(t *testing.T) {
+	components, err := Generate("../testdata/generate-remote-static", []string{"common"}, false)
+
+	expectedLengths := map[string]int{
+		"keyvault-flexvolume": 5,
+		"keyvault-sub":  1372,
+	}
+
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(components))
+
+	checkComponentLengthsAgainstExpected(t, components, expectedLengths)
+}
+
+
 func TestGenerateWithHooks(t *testing.T) {
 	_, err := Generate("../testdata/generate-hooks", []string{"prod"}, false)
 

core/component.go

@@ -4,9 +4,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"net/http"
 	"path"
 	"path/filepath"
 	"sort"
@@ -190,6 +192,44 @@ func (c *Component) afterInstall() (err error) {
 	return c.ExecuteHook("after-install")
 }
 
+// InstallRemoteStaticComponent installs a component by downloading the remote resource manifest
+func (c *Component) InstallRemoteStaticComponent(componentPath string) (err error) {
+	if !IsValidRemoteComponentConfig(*c) {
+		return nil;
+	}
+
+	componentsPath := path.Join(componentPath, "components/", c.Name)
+
+	if err := os.MkdirAll(componentsPath, 0777); err != nil {
+		return err
+	}
+
+	response, err := http.Get(c.Source);
+
+	if err != nil {
+		return err
+	}
+
+	// Write the downloaded resource manifest file
+	defer response.Body.Close()
+	out, err := os.Create(path.Join(componentsPath, c.Name + ".yaml"))
+
+	if err != nil {
+		logger.Error(emoji.Sprintf(":no_entry_sign: Error occurred in install for component '%s'\nError: %s", c.Name, err))
+		return err
+	}
+
+	defer out.Close()
+	_, err = io.Copy(out, response.Body)
+
+	if (err != nil) {
+		logger.Error(emoji.Sprintf(":no_entry_sign: Error occurred in writing manifest file for component '%s'\nError: %s", c.Name, err))
+		return err
+	}
+
+	return nil;
+}
+
 // InstallComponent installs the component (if needed) utilizing its Method.
 func (c *Component) InstallComponent(componentPath string) (err error) {
 	if (c.ComponentType == "component" || len(c.ComponentType) == 0) && c.Method == "git" {
@@ -208,6 +248,8 @@ func (c *Component) InstallComponent(componentPath string) (err error) {
 		if err = CloneRepo(c.Source, c.Version, subcomponentPath, c.Branch); err != nil {
 			return err
 		}
+	} else if IsValidRemoteComponentConfig(*c) {
+		return c.InstallRemoteStaticComponent(componentPath)
 	}
 
 	return nil
@@ -494,3 +536,46 @@ func (c *Component) GetAccessTokens() (tokens map[string]string, err error) {
 	}
 	return tokens, err
 }
+
+// GetStaticComponentPath returns the static path if a component is of type 'static', if not, it returns an empty string
+func (c *Component) GetStaticComponentPath(startPath string)(componentPath string){
+	if c.ComponentType != "static" {
+		return ""
+	}
+
+	if IsValidRemoteComponentConfig(*c) {
+		return path.Join(startPath, "components", c.Name)
+	}
+
+	return path.Join(c.PhysicalPath, c.Path)
+}
+
+
+// CreateDirectory a directory in the given path and reports no error if the directory already exists
+func CreateDirectory(cmdDir string, dirPath string) (componentPath string, err error) {
+
+	cmd := exec.Command("sh", "-c", "mkdir -p " + dirPath)
+	cmd.Dir = cmdDir
+
+	output, err := cmd.CombinedOutput()
+
+	if err != nil {
+		logger.Error(emoji.Sprintf(":no_entry_sign: Error occurred in creating directory for component\n"))
+		return "", err
+	}
+	if len(output) > 0 {
+		outstring := emoji.Sprintf(":mag_right: Completed creating directory for component\n")
+		logger.Trace(strings.TrimSpace(outstring))
+	}
+
+	return path.Join(cmdDir, dirPath), nil
+}
+
+// IsValidRemoteComponentConfig checks if the given component configuration is valid for a remote component
+func IsValidRemoteComponentConfig(c Component) (bool){
+	return (
+		(c.ComponentType == "static" &&
+		 c.Method == "http")) &&
+		(strings.HasSuffix(c.Source, "yaml") ||
+		 strings.HasSuffix(c.Source, "yml"))
+}

core/component_test.go

@@ -116,3 +116,73 @@ func TestWriteComponent(t *testing.T) {
 	err = component.Write()
 	assert.Nil(t, err)
 }
+
+func TestValidRemoteComponentConfig(t *testing.T) {
+	component := Component{
+		ComponentType: "static",
+		Method: "http",
+		Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml",
+	}
+
+	isValid := IsValidRemoteComponentConfig(component)
+	assert.True(t, isValid, "Component is remote static component.")
+}
+
+func TestInvalidRemoteComponentSource(t *testing.T) {
+	component := Component{
+		ComponentType: "static",
+		Method: "http",
+		Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer",
+	}
+
+	isValid := IsValidRemoteComponentConfig(component)
+	assert.True(t, !isValid, "Component is remote static component.")
+}
+
+func TestValidRemoteComponentURL(t *testing.T) {
+	component := Component{
+		ComponentType: "static",
+		Method: "http",
+		Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml",
+	}
+
+	isValid := IsValidRemoteComponentConfig(component)
+	assert.True(t, isValid, "Component is remote static component.")
+
+	component.Source = "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yml"
+	isValid = IsValidRemoteComponentConfig(component)
+	assert.True(t, isValid, "Component is remote static component.")
+}
+
+func TestInvalidValidRemoteComponentConfig(t *testing.T) {
+
+	componentTypes := [3]string{"component", "helm", "static"}
+	methods := [3]string{"git", "helm", "local"}
+	component := Component{
+		Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml",
+	}
+
+	for _, componentType := range componentTypes {
+		for _, method := range methods {
+			component.ComponentType  = componentType
+			component.Method = method
+			isValid := IsValidRemoteComponentConfig(component)
+			assert.True(t, !isValid, "Component is not a remote static component.")
+		}
+	}
+}
+
+
+func TestGetStaticComponentPath(t *testing.T) {
+	component := Component{
+		Name: "kv-flexvol",
+		ComponentType: "static",
+		Method: "http",
+		Source: "https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml",
+	}
+
+	expectedComponentPath := "../testdata/generate-remote-static/components/kv-flexvol"
+	componentPath := component.GetStaticComponentPath("../testdata/generate-remote-static")
+
+	assert.Equal(t, expectedComponentPath, componentPath)
+}

docs/component.md

@@ -16,8 +16,10 @@ component with the following schema:
   - if `type: helm`: the component will use `helm template` to materialize the
     component using the specified config under `config` as the `values.yaml`
     file.
-  - if `type: static`: the component holds raw kubernetes manifest files in
+  - if `type: static`:
+    Option 1: the component holds raw kubernetes manifest files in
     `path`, these manifests will be copied to the generated output.
+    Option 2: when using `method: http` and `source: url`, the manifest file (.yaml) is downloaded and installed. Example: `source: https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml`
 
 - `method`: The method by which this component is sourced. Currently, only
   `git`, `helm`, and `local` are supported values.
@@ -28,6 +30,7 @@ component with the following schema:
     `helm repo add foo <my_helm_repository> && helm fetch foo/<path>`
   - if `method: local`: Tells fabrikate to use the host filesystem as a means to
     find the component.
+  - if `method: http` and `type: static`: Tells fabrikate to download the manifest file (.yaml) from the `source`.
 
 - `source`: The source for this component.
 
@@ -36,6 +39,7 @@ component with the following schema:
   - if `method: helm`: A URL to a helm repository (the url you would call
     `helm repo add` on).
   - if `method: local`: A local path to specify a local filesystem component.
+  - if `method: http` and `type: static`: A URL for a yaml file.
 
 - `path`: For some components, like ones generated with `helm`, the desired
   target of the component might not be located at the root of the repo. Path
@@ -48,6 +52,7 @@ component with the following schema:
     `source`.
   - if `method: local`: the subdirectory on host filesystem where the component
     is located.
+  - if `method: http`: a path does not need to be specified
 
 - `version`: For git `method` components, this specifies a specific commit SHA
   hash that the component should be locked to, enabling you to lock the

generators/static.go

@@ -11,13 +11,15 @@ import (
 )
 
 // StaticGenerator uses a static directory of resource manifests to create a rolled up multi-part manifest.
-type StaticGenerator struct{}
+type StaticGenerator struct{
+	StartPath string
+}
 
 // Generate iterates a static directory of resource manifests and creates a multi-part manifest.
 func (sg *StaticGenerator) Generate(component *core.Component) (manifest string, err error) {
 	logger.Info(emoji.Sprintf(":truck: Generating component '%s' statically from path %s", component.Name, component.Path))
 
-	staticPath := path.Join(component.PhysicalPath, component.Path)
+	staticPath := component.GetStaticComponentPath(sg.StartPath)
 	staticFiles, err := ioutil.ReadDir(staticPath)
 	if err != nil {
 		logger.Error(fmt.Sprintf("error reading from directory %s", staticPath))
@@ -43,4 +45,4 @@ func (sg *StaticGenerator) Generate(component *core.Component) (manifest string,
 // Currently is a no-op, but could be extended to support remote static content (see #155)
 func (sg *StaticGenerator) Install(component *core.Component) (err error) {
 	return nil
-}
+}

testdata/generate-remote-static/component.yaml

@@ -0,0 +1,9 @@
+name: keyvault-flexvolume
+type: static
+path: ./manifests
+subcomponents:
+  - name: "keyvault-sub"
+    source: https://raw.githubusercontent.com/Azure/kubernetes-keyvault-flexvol/master/deployment/kv-flexvol-installer.yaml
+    method: http
+    type: static
+    path: "./tmp/keyvault-sub"

testdata/generate-remote-static/components/keyvault-sub/keyvault-sub.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kv
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    app: keyvault-flexvolume
+  name: keyvault-flexvolume
+  namespace: kv
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: keyvault-flexvolume
+    spec:
+      tolerations:
+      containers:
+      - name: flexvol-driver-installer
+        image: "mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v0.0.13"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 50m
+            memory: 100Mi
+          limits:
+            cpu: 50m
+            memory: 100Mi
+        env:
+          # if you have used flex before on your cluster, use same directory
+          # set TARGET_DIR env var and mount the same directory to to the container
+        - name: TARGET_DIR
+          value: "/etc/kubernetes/volumeplugins"
+        volumeMounts:
+        - mountPath: "/etc/kubernetes/volumeplugins"
+          name: volplugins
+      volumes:
+      - hostPath:
+          # Modify this directory if your nodes are using a different one
+          # default is "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" 
+          # below is Azure default
+          path: "/etc/kubernetes/volumeplugins" 
+        name: volplugins
+      nodeSelector:
+        beta.kubernetes.io/os: linux