Merge branch 'main' into fix/go-yaml

.github/workflows/jsonschema.yml

@@ -0,0 +1,35 @@
+# name of the action
+name: jsonschema
+
+# trigger on release events
+on:
+  release:
+    types: [created]
+
+# pipeline to execute
+jobs:
+  schema:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: clone
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: install go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          # use version from go.mod file
+          go-version-file: "go.mod"
+          cache: true
+          check-latest: true
+
+      - name: build
+        run: |
+          make jsonschema
+
+      - name: upload
+        uses: skx/github-action-publish-binaries@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          args: "schema.json"

.github/workflows/test.yml

@@ -12,23 +12,29 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: clone
-      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-
-    - name: install go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
-      with:
-        # use version from go.mod file
-        go-version-file: 'go.mod'
-        cache: true
-        check-latest: true
-
-    - name: test
-      run: |
-        make test
-
-    - name: coverage
-      uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.out
+      - name: clone
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: install go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          # use version from go.mod file
+          go-version-file: "go.mod"
+          cache: true
+          check-latest: true
+
+      - name: test
+        run: |
+          make test
+
+      - name: test jsonschema
+        run: |
+          go install github.com/santhosh-tekuri/jsonschema/cmd/[email protected]
+          make test-jsonschema
+
+      - name: coverage
+        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: coverage.out
+

.gitignore

@@ -67,4 +67,6 @@ __debug_bin
 .history
 .ionide
 
-# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode
+
+schema.json

Makefile

@@ -326,6 +326,50 @@ spec-version-update:
 .PHONY: spec
 spec: spec-gen spec-version-update spec-validate
 
+# The `jsonschema` target is intended to create
+# a jsonschema for a Vela pipeline.
+#
+# Usage: `make jsonschema`
+.PHONY: jsonschema
+jsonschema:
+	@echo
+	@echo "### Generating JSON schema"
+	@go run cmd/jsonschema-gen/main.go > schema.json
+
+# The `test-jsonschema` target is intended to test
+# the created jsonschema against a set of failing
+# and passing example vela templates located in
+# schema/testdata/pipeline.
+#
+# The test relies on the `jv` command line tool,
+# which can be installed via:
+#
+# go install github.com/santhosh-tekuri/jsonschema/cmd/jv@latest
+#
+# Usage: `make test-jsonschema`
+.PHONY: test-jsonschema
+test-jsonschema: jsonschema
+	@echo
+	@echo "### Testing Pipelines against JSON Schema"
+	@echo
+	@echo "=== Expected Failing Tests"
+	@for file in schema/testdata/pipeline/fail/*.yml; do \
+		echo "› Test: $$file"; \
+		if jv schema.json $$file >/dev/null 2>&1; then \
+			echo "Unexpected success for $$file"; \
+			exit 1; \
+		fi; \
+	done
+	@echo
+	@echo "=== Expected Passing Tests"
+	@for file in schema/testdata/pipeline/pass/*.yml; do \
+		echo "› Test: $$file"; \
+		if ! jv schema.json $$file >/dev/null 2>&1; then \
+			echo "Unexpected failure for $$file"; \
+			exit 1; \
+		fi; \
+	done
+
 # The `lint` target is intended to lint the
 # Go source code with golangci-lint.
 #

api/build/compile_publish.go

@@ -26,13 +26,14 @@ import (
 
 // CompileAndPublishConfig is a struct that contains information for the CompileAndPublish function.
 type CompileAndPublishConfig struct {
-	Build    *types.Build
-	Metadata *internal.Metadata
-	BaseErr  string
-	Source   string
-	Comment  string
-	Labels   []string
-	Retries  int
+	Build      *types.Build
+	Deployment *types.Deployment
+	Metadata   *internal.Metadata
+	BaseErr    string
+	Source     string
+	Comment    string
+	Labels     []string
+	Retries    int
 }
 
 // CompileAndPublish is a helper function to generate the queue items for a build. It takes a form
@@ -307,6 +308,15 @@ func CompileAndPublish(
 				errors.New(skip)
 		}
 
+		// validate deployment config
+		if (b.GetEvent() == constants.EventDeploy) && cfg.Deployment != nil {
+			if err := p.Deployment.Validate(cfg.Deployment.GetTarget(), cfg.Deployment.GetPayload()); err != nil {
+				retErr := fmt.Errorf("%s: failed to validate deployment for %s: %w", baseErr, repo.GetFullName(), err)
+
+				return nil, nil, http.StatusBadRequest, retErr
+			}
+		}
+
 		// check if the pipeline did not already exist in the database
 		if pipeline == nil {
 			pipeline = compiled

api/deployment/get.go

@@ -87,7 +87,7 @@ func GetDeployment(c *gin.Context) {
 	}
 
 	// send API call to database to capture the deployment
-	d, err := database.FromContext(c).GetDeployment(ctx, int64(number))
+	d, err := database.FromContext(c).GetDeploymentForRepo(ctx, r, int64(number))
 	if err != nil {
 		// send API call to SCM to capture the deployment
 		d, err = scm.FromContext(c).GetDeployment(ctx, u, r, int64(number))

api/deployment/get_config.go

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package deployment
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+
+	"github.com/go-vela/server/compiler"
+	"github.com/go-vela/server/database"
+	"github.com/go-vela/server/router/middleware/repo"
+	"github.com/go-vela/server/router/middleware/user"
+	"github.com/go-vela/server/scm"
+	"github.com/go-vela/server/util"
+)
+
+// swagger:operation GET /api/v1/deployments/{org}/{repo}/config deployments GetDeploymentConfig
+//
+// Get a deployment config
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+// - in: path
+//   name: org
+//   description: Name of the organization
+//   required: true
+//   type: string
+// - in: path
+//   name: repo
+//   description: Name of the repository
+//   required: true
+//   type: string
+// - in: query
+//   name: ref
+//   description: Ref to target for the deployment config
+//   type: string
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully retrieved the deployment config
+//     schema:
+//       "$ref": "#/definitions/Deployment"
+//   '400':
+//     description: Invalid request payload or path
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '401':
+//     description: Unauthorized
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '404':
+//     description: Not found
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '500':
+//     description: Unexpected server error
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// GetDeploymentConfig represents the API handler to get a deployment config at a given ref.
+func GetDeploymentConfig(c *gin.Context) {
+	// capture middleware values
+	l := c.MustGet("logger").(*logrus.Entry)
+	r := repo.Retrieve(c)
+	u := user.Retrieve(c)
+
+	ctx := c.Request.Context()
+
+	// capture ref from parameters - use default branch if not provided
+	ref := util.QueryParameter(c, "ref", r.GetBranch())
+
+	entry := fmt.Sprintf("%s@%s", r.GetFullName(), ref)
+
+	l.Debugf("reading deployment config %s", entry)
+
+	var config []byte
+
+	// check if the pipeline exists in the database
+	p, err := database.FromContext(c).GetPipelineForRepo(ctx, ref, r)
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			l.Debugf("pipeline %s not found in database, fetching from scm", entry)
+
+			config, err = scm.FromContext(c).ConfigBackoff(ctx, u, r, ref)
+			if err != nil {
+				retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", entry, err)
+
+				util.HandleError(c, http.StatusNotFound, retErr)
+
+				return
+			}
+		} else {
+			// some other error
+			retErr := fmt.Errorf("unable to get pipeline for %s: %w", entry, err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+	} else {
+		l.Debugf("pipeline %s found in database", entry)
+
+		config = p.GetData()
+	}
+
+	// set up compiler
+	compiler := compiler.FromContext(c).Duplicate().WithCommit(ref).WithRepo(r).WithUser(u)
+
+	// compile the pipeline
+	pipeline, _, err := compiler.CompileLite(ctx, config, nil, true)
+	if err != nil {
+		retErr := fmt.Errorf("unable to compile pipeline %s: %w", entry, err)
+
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	c.JSON(http.StatusOK, pipeline.Deployment)
+}

api/webhook/post.go

@@ -378,13 +378,14 @@ func PostWebhook(c *gin.Context) {
 
 	// construct CompileAndPublishConfig
 	config := build.CompileAndPublishConfig{
-		Build:    b,
-		Metadata: m,
-		BaseErr:  baseErr,
-		Source:   "webhook",
-		Comment:  prComment,
-		Labels:   prLabels,
-		Retries:  3,
+		Build:      b,
+		Deployment: webhook.Deployment,
+		Metadata:   m,
+		BaseErr:    baseErr,
+		Source:     "webhook",
+		Comment:    prComment,
+		Labels:     prLabels,
+		Retries:    3,
 	}
 
 	// generate the queue item

cmd/jsonschema-gen/main.go

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build ignore
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/schema"
+)
+
+func main() {
+	js, err := schema.NewPipelineSchema()
+	if err != nil {
+		logrus.Fatal("schema generation failed:", err)
+	}
+
+	// output json
+	j, err := json.MarshalIndent(js, "", "  ")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", j)
+}