refactor(jsonschema): relocate schema generator (#1219)

.github/workflows/jsonschema.yml

@@ -0,0 +1,35 @@
+# name of the action
+name: jsonschema
+
+# trigger on release events
+on:
+  release:
+    types: [created]
+
+# pipeline to execute
+jobs:
+  schema:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: clone
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: install go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          # use version from go.mod file
+          go-version-file: "go.mod"
+          cache: true
+          check-latest: true
+
+      - name: build
+        run: |
+          make jsonschema
+
+      - name: upload
+        uses: skx/github-action-publish-binaries@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          args: "schema.json"

.github/workflows/test.yml

@@ -12,23 +12,29 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: clone
-      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-
-    - name: install go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
-      with:
-        # use version from go.mod file
-        go-version-file: 'go.mod'
-        cache: true
-        check-latest: true
-
-    - name: test
-      run: |
-        make test
-
-    - name: coverage
-      uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.out
+      - name: clone
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: install go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          # use version from go.mod file
+          go-version-file: "go.mod"
+          cache: true
+          check-latest: true
+
+      - name: test
+        run: |
+          make test
+
+      - name: test jsonschema
+        run: |
+          go install github.com/santhosh-tekuri/jsonschema/cmd/[email protected]
+          make test-jsonschema
+
+      - name: coverage
+        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: coverage.out
+

.gitignore

@@ -67,4 +67,6 @@ __debug_bin
 .history
 .ionide
 
-# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode
+
+schema.json

Makefile

@@ -326,6 +326,50 @@ spec-version-update:
 .PHONY: spec
 spec: spec-gen spec-version-update spec-validate
 
+# The `jsonschema` target is intended to create
+# a jsonschema for a Vela pipeline.
+#
+# Usage: `make jsonschema`
+.PHONY: jsonschema
+jsonschema:
+	@echo
+	@echo "### Generating JSON schema"
+	@go run cmd/jsonschema-gen/main.go > schema.json
+
+# The `test-jsonschema` target is intended to test
+# the created jsonschema against a set of failing
+# and passing example vela templates located in
+# schema/testdata/pipeline.
+#
+# The test relies on the `jv` command line tool,
+# which can be installed via:
+#
+# go install github.com/santhosh-tekuri/jsonschema/cmd/jv@latest
+#
+# Usage: `make test-jsonschema`
+.PHONY: test-jsonschema
+test-jsonschema: jsonschema
+	@echo
+	@echo "### Testing Pipelines against JSON Schema"
+	@echo
+	@echo "=== Expected Failing Tests"
+	@for file in schema/testdata/pipeline/fail/*.yml; do \
+		echo "› Test: $$file"; \
+		if jv schema.json $$file >/dev/null 2>&1; then \
+			echo "Unexpected success for $$file"; \
+			exit 1; \
+		fi; \
+	done
+	@echo
+	@echo "=== Expected Passing Tests"
+	@for file in schema/testdata/pipeline/pass/*.yml; do \
+		echo "› Test: $$file"; \
+		if ! jv schema.json $$file >/dev/null 2>&1; then \
+			echo "Unexpected failure for $$file"; \
+			exit 1; \
+		fi; \
+	done
+
 # The `lint` target is intended to lint the
 # Go source code with golangci-lint.
 #

cmd/jsonschema-gen/main.go

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build ignore
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/schema"
+)
+
+func main() {
+	js, err := schema.NewPipelineSchema()
+	if err != nil {
+		logrus.Fatal("schema generation failed:", err)
+	}
+
+	// output json
+	j, err := json.MarshalIndent(js, "", "  ")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", j)
+}

compiler/types/raw/map.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"errors"
 	"strings"
+
+	"github.com/invopop/jsonschema"
 )
 
 // StringSliceMap represents an array of strings or a map of strings.
@@ -138,3 +140,27 @@ func (s *StringSliceMap) UnmarshalYAML(unmarshal func(interface{}) error) error
 
 	return errors.New("unable to unmarshal into StringSliceMap")
 }
+
+// JSONSchema handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Without these changes it would only allow a map of string,
+// but we do some special handling to support array of strings.
+func (StringSliceMap) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		OneOf: []*jsonschema.Schema{
+			{
+				Type: "array",
+				Items: &jsonschema.Schema{
+					Type: "string",
+				},
+			},
+			{
+				Type: "object",
+				AdditionalProperties: &jsonschema.Schema{
+					Type: "string",
+				},
+			},
+		},
+	}
+}

compiler/types/raw/slice.go

@@ -5,6 +5,8 @@ package raw
 import (
 	"encoding/json"
 	"errors"
+
+	"github.com/invopop/jsonschema"
 )
 
 // StringSlice represents a string or an array of strings.
@@ -71,3 +73,24 @@ func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return errors.New("unable to unmarshal into StringSlice")
 }
+
+// JSONSchema handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Without these changes it would only allow an array of strings,
+// but we do some special handling to support plain string also.
+func (StringSlice) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		OneOf: []*jsonschema.Schema{
+			{
+				Type: "string",
+			},
+			{
+				Type: "array",
+				Items: &jsonschema.Schema{
+					Type: "string",
+				},
+			},
+		},
+	}
+}

compiler/types/yaml/ruleset.go

@@ -3,6 +3,8 @@
 package yaml
 
 import (
+	"github.com/invopop/jsonschema"
+
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
 	"github.com/go-vela/server/constants"
@@ -22,13 +24,15 @@ type (
 	// Rules is the yaml representation of the ruletypes
 	// from a ruleset block for a step in a pipeline.
 	Rules struct {
-		Branch   []string `yaml:"branch,omitempty,flow"   json:"branch,omitempty"   jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Comment  []string `yaml:"comment,omitempty,flow"  json:"comment,omitempty"  jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Event    []string `yaml:"event,omitempty,flow"    json:"event,omitempty"    jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Path     []string `yaml:"path,omitempty,flow"     json:"path,omitempty"     jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Repo     []string `yaml:"repo,omitempty,flow"     json:"repo,omitempty"     jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Sender   []string `yaml:"sender,omitempty,flow"   json:"sender,omitempty"   jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Status   []string `yaml:"status,omitempty,flow"   json:"status,omitempty"   jsonschema:"enum=[failure],enum=[success],description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Branch  []string `yaml:"branch,omitempty,flow"  json:"branch,omitempty"  jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Comment []string `yaml:"comment,omitempty,flow" json:"comment,omitempty" jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		// enums for 'event' jsonschema are set in JSONSchemaExtend() method below
+		Event  []string `yaml:"event,omitempty,flow"  json:"event,omitempty"  jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Path   []string `yaml:"path,omitempty,flow"   json:"path,omitempty"   jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Repo   []string `yaml:"repo,omitempty,flow"   json:"repo,omitempty"   jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Sender []string `yaml:"sender,omitempty,flow" json:"sender,omitempty" jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		// enums for 'status' jsonschema are set in JSONSchemaExtend() method below
+		Status   []string `yaml:"status,omitempty,flow"   json:"status,omitempty"   jsonschema:"description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
 		Tag      []string `yaml:"tag,omitempty,flow"      json:"tag,omitempty"      jsonschema:"description=Limits the execution of a step to matching build tag references.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
 		Target   []string `yaml:"target,omitempty,flow"   json:"target,omitempty"   jsonschema:"description=Limits the execution of a step to matching build deployment targets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
 		Label    []string `yaml:"label,omitempty,flow"    json:"label,omitempty"    jsonschema:"description=Limits step execution to match on pull requests labels.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
@@ -186,3 +190,89 @@ func (r *Rules) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return err
 }
+
+// JSONSchemaExtend handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Mainly it handles the fact that all Rules fields are raw.StringSlice
+// but also handles adding enums to select fields as they would be too
+// cumbersome to maintain in the jsonschema struct tag.
+func (Rules) JSONSchemaExtend(schema *jsonschema.Schema) {
+	for item := schema.Properties.Newest(); item != nil; item = item.Prev() {
+		currSchema := *item.Value
+
+		// store the current description so we can lift it to top level
+		currDescription := currSchema.Description
+		currSchema.Description = ""
+
+		// handle each field as needed
+		switch item.Key {
+		case "status":
+			// possible values for 'status'
+			enums := []string{
+				"success",
+				"failure",
+			}
+
+			for _, str := range enums {
+				currSchema.Items.Enum = append(currSchema.Items.Enum, str)
+			}
+
+			schema.Properties.Set(item.Key, &jsonschema.Schema{
+				OneOf: []*jsonschema.Schema{
+					&currSchema,
+					{
+						Type: "string",
+						Enum: currSchema.Items.Enum,
+					},
+				},
+				Description: currDescription,
+			})
+		case "event":
+			// possible values for 'event'
+			enums := []string{
+				"comment",
+				"comment:created",
+				"comment:edited",
+				"delete:branch",
+				"delete:tag",
+				"deployment",
+				"pull_request",
+				"pull_request*",
+				"pull_request:edited",
+				"pull_request:labeled",
+				"pull_request:opened",
+				"pull_request:reopened",
+				"pull_request:synchronize",
+				"pull_request:unlabeled",
+				"push",
+				"schedule",
+				"tag",
+			}
+
+			for _, str := range enums {
+				currSchema.Items.Enum = append(currSchema.Items.Enum, str)
+			}
+
+			schema.Properties.Set(item.Key, &jsonschema.Schema{
+				OneOf: []*jsonschema.Schema{
+					&currSchema,
+					{
+						Type: "string",
+						Enum: currSchema.Items.Enum,
+					},
+				},
+				Description: currDescription,
+			})
+		default:
+			// all other fields are raw.StringSlice
+			schema.Properties.Set(item.Key, &jsonschema.Schema{
+				OneOf: []*jsonschema.Schema{
+					&currSchema,
+					{Type: "string"},
+				},
+				Description: currDescription,
+			})
+		}
+	}
+}