Add support for remote validation schemas (#731)

* Update atmos.yaml with new manifest URL * add URL handling functions * modify ExecuteValidateStacksCmd to handle get remote file * add context with timeout * add handling for invalid URLs and unsupported URL schemes. * Refactor GetFileNameFromURL to handle invalid file name * add doc Specify remote schemas in atmos.yaml * Update json-schema.mdx url * Update website/docs/core-concepts/validate/json-schema.mdx Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * check fileName is empty * Update internal/exec/validate_stacks.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * Refactor file download logic in validate_stacks.go * Refactor file_utils.go and json-schema.mdx * updates --------- Co-authored-by: Erik Osterman (CEO @ Cloud Posse) <[email protected]> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Andriy Knysh <[email protected]> Co-authored-by: aknysh <[email protected]>
cloudposse · Oct 30, 2024 · e8b4fb9 · e8b4fb9
1 parent 682d6e5
commit e8b4fb9
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 4 deletions.
diff --git a/examples/demo-context/atmos.yaml b/examples/demo-context/atmos.yaml
@@ -2,8 +2,7 @@ base_path: "./"
 
 schemas:
   atmos:
-    manifest: "schemas/atmos-manifest.json"
-
+    manifest: "https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
 # https://pkg.go.dev/text/template
 templates:
   settings:

diff --git a/internal/exec/validate_stacks.go b/internal/exec/validate_stacks.go
@@ -1,11 +1,16 @@
 package exec
 
 import (
+	"context"
 	"fmt"
+	"net/url"
+	"os"
 	"path"
 	"reflect"
 	"strings"
+	"time"
 
+	"github.com/hashicorp/go-getter"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
@@ -86,15 +91,20 @@ func ValidateStacks(cliConfig schema.CliConfiguration) error {
 			atmosManifestJsonSchemaFilePath = cliConfig.Schemas.Atmos.Manifest
 		} else if u.FileExists(atmosManifestJsonSchemaFileAbsPath) {
 			atmosManifestJsonSchemaFilePath = atmosManifestJsonSchemaFileAbsPath
+		} else if u.IsURL(cliConfig.Schemas.Atmos.Manifest) {
+			atmosManifestJsonSchemaFilePath, err = downloadSchemaFromURL(cliConfig.Schemas.Atmos.Manifest)
+			if err != nil {
+				return err
+			}
 		} else {
 			return fmt.Errorf("the Atmos JSON Schema file '%s' does not exist.\n"+
 				"It can be configured in the 'schemas.atmos.manifest' section in 'atmos.yaml', or provided using the 'ATMOS_SCHEMAS_ATMOS_MANIFEST' "+
 				"ENV variable or '--schemas-atmos-manifest' command line argument.\n"+
-				"The path to the schema file should be an absolute path or a path relative to the 'base_path' setting in 'atmos.yaml'.",
+				"The path to the schema file should be an absolute path or a path relative to the 'base_path' setting in 'atmos.yaml'. \n"+
+				"Alternatively, you can specify a schema file using a URL that will be downloaded automatically.",
 				cliConfig.Schemas.Atmos.Manifest)
 		}
 	}
-
 	// Include (process and validate) all YAML files in the `stacks` folder in all subfolders
 	includedPaths := []string{"**/*"}
 	// Don't exclude any YAML files for validation
@@ -340,3 +350,32 @@ func checkComponentStackMap(componentStackMap map[string]map[string][]string) ([
 
 	return res, nil
 }
+
+// downloadSchemaFromURL downloads the Atmos JSON Schema file from the provided URL
+func downloadSchemaFromURL(manifestURL string) (string, error) {
+	parsedURL, err := url.Parse(manifestURL)
+	if err != nil {
+		return "", fmt.Errorf("invalid URL '%s': %w", manifestURL, err)
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return "", fmt.Errorf("unsupported URL scheme '%s' for schema manifest", parsedURL.Scheme)
+	}
+	tempDir := os.TempDir()
+	fileName, err := u.GetFileNameFromURL(manifestURL)
+	if err != nil || fileName == "" {
+		return "", fmt.Errorf("failed to get the file name from the URL '%s': %w", manifestURL, err)
+	}
+	atmosManifestJsonSchemaFilePath := path.Join(tempDir, fileName)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	client := &getter.Client{
+		Ctx:  ctx,
+		Dst:  atmosManifestJsonSchemaFilePath,
+		Src:  manifestURL,
+		Mode: getter.ClientModeFile,
+	}
+	if err = client.Get(); err != nil {
+		return "", fmt.Errorf("failed to download the Atmos JSON Schema file '%s' from the URL '%s': %w", fileName, manifestURL, err)
+	}
+	return atmosManifestJsonSchemaFilePath, nil
+}
diff --git a/pkg/utils/file_utils.go b/pkg/utils/file_utils.go
@@ -1,7 +1,9 @@
 package utils
 
 import (
+	"fmt"
 	"io/fs"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -192,3 +194,40 @@ func SearchConfigFile(path string) (string, bool) {
 	}
 	return "", false
 }
+
+// IsURL checks if a string is a URL
+func IsURL(s string) bool {
+	url, err := url.Parse(s)
+	if err != nil {
+		return false
+	}
+	validSchemes := []string{"http", "https"}
+	schemeValid := false
+	for _, scheme := range validSchemes {
+		if url.Scheme == scheme {
+			schemeValid = true
+			break
+		}
+	}
+	return schemeValid
+
+}
+
+// GetFileNameFromURL extracts the file name from a URL
+func GetFileNameFromURL(rawURL string) (string, error) {
+	if rawURL == "" {
+		return "", fmt.Errorf("empty URL provided")
+	}
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", err
+	}
+	// Extract the path from the URL
+	urlPath := parsedURL.Path
+	fileName := path.Base(urlPath)
+	if fileName == "/" || fileName == "." {
+		return "", fmt.Errorf("unable to extract filename from URL: %s", rawURL)
+	}
+	// Get the base name of the path
+	return fileName, nil
+}
diff --git a/website/docs/core-concepts/validate/json-schema.mdx b/website/docs/core-concepts/validate/json-schema.mdx
@@ -43,6 +43,19 @@ schemas:
 In the component [manifest](https://github.com/cloudposse/atmos/blob/master/examples/quick-start-advanced/stacks/catalog/vpc/defaults.yaml), add
 the `settings.validation` section:
 
+### Use Remote Schemas
+
+You can specify remote schemas by setting the `manifest` field to a remote URL in your `atmos.yaml` configuration file.
+
+<File title="atmos.yaml">
+```yaml
+# Validation schemas (for validating atmos stacks and components)
+schemas:
+  atmos:
+    # You can specify a remote schema URL as well
+    manifest: "https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json"
+```
+</File>
 <EmbedFile filePath="examples/quick-start-advanced/stacks/catalog/vpc/defaults.yaml"/>
 
 Add the following JSON Schema in the