Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(yaml): add go-yaml types package #1225

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

chore(yaml): add go-yaml types package #1225

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8ba7055
enhance(yaml): allow for users to parse pipelines using old library
ecrupper Nov 5, 2024
23205cd
testing file for internal yaml
ecrupper Nov 5, 2024
d408913
Merge branch 'main' into enhance/yaml-parser-flag
ecrupper Nov 21, 2024
43f6e4b
Merge branch 'main' into HEAD
ecrupper Dec 3, 2024
e241725
chore(compiler): convert unmarshaled buildkite to go-yaml
ecrupper Dec 3, 2024
978cade
remove tests used in later PRs
ecrupper Dec 3, 2024
5cee6b0
lintfix
ecrupper Dec 3, 2024
4590305
Merge branch 'main' into fix/go-yaml
ecrupper Dec 19, 2024
2c4ca55
fix schema
ecrupper Dec 19, 2024
de8b579
Merge branch 'main' into fix/go-yaml
ecrupper Dec 19, 2024
0f94592
gci
ecrupper Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/pipeline/template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler"
"github.com/go-vela/server/compiler/registry/github"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/internal"
"github.com/go-vela/server/router/middleware/org"
"github.com/go-vela/server/router/middleware/pipeline"
Expand Down
2 changes: 1 addition & 1 deletion api/types/string.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (
"strconv"
"strings"

"github.com/buildkite/yaml"
json "github.com/ghodss/yaml"
"gopkg.in/yaml.v3"
)

// ToString is a helper function to convert
Expand Down
2 changes: 1 addition & 1 deletion compiler/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/go-vela/server/api/types/settings"
"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/internal"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/clone.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
package native

import (
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/clone_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"github.com/urfave/cli/v2"

"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

const defaultCloneImage = "target/vela-git-slim:latest"
Expand Down
4 changes: 2 additions & 2 deletions compiler/native/compile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ import (
"strings"
"time"

yml "github.com/buildkite/yaml"
"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/go-retryablehttp"
yml "gopkg.in/yaml.v3"

api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down
8 changes: 4 additions & 4 deletions compiler/native/compile_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ import (
"testing"
"time"

yml "github.com/buildkite/yaml"
"github.com/gin-gonic/gin"
"github.com/google/go-cmp/cmp"
"github.com/google/go-github/v65/github"
"github.com/urfave/cli/v2"
yml "gopkg.in/yaml.v3"

api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
"github.com/go-vela/server/internal"
)
Expand Down Expand Up @@ -2056,7 +2056,7 @@ func Test_client_modifyConfig(t *testing.T) {
Name: "docker",
Pull: "always",
Parameters: map[string]interface{}{
"init_options": map[interface{}]interface{}{
"init_options": map[string]interface{}{
"get_plugins": "true",
},
},
Expand Down Expand Up @@ -2089,7 +2089,7 @@ func Test_client_modifyConfig(t *testing.T) {
Name: "docker",
Pull: "always",
Parameters: map[string]interface{}{
"init_options": map[interface{}]interface{}{
"init_options": map[string]interface{}{
"get_plugins": "true",
},
},
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
"github.com/go-vela/server/internal"
)
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/environment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (

api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/internal"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/expand.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/go-vela/server/compiler/template/starlark"
"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/expand_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

func TestNative_ExpandStages(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/initialize.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
package native

import (
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/initialize_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"github.com/urfave/cli/v2"

"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

func TestNative_InitStage(t *testing.T) {
Expand Down
25 changes: 13 additions & 12 deletions compiler/native/parse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@ import (
"io"
"os"

"github.com/buildkite/yaml"
bkYaml "github.com/buildkite/yaml"

"github.com/go-vela/server/compiler/template/native"
"github.com/go-vela/server/compiler/template/starlark"
typesRaw "github.com/go-vela/server/compiler/types/raw"
types "github.com/go-vela/server/compiler/types/yaml"
bkYamlTypes "github.com/go-vela/server/compiler/types/yaml/buildkite"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down Expand Up @@ -41,9 +42,9 @@ func (c *client) ParseRaw(v interface{}) (string, error) {
}

// Parse converts an object to a yaml configuration.
func (c *client) Parse(v interface{}, pipelineType string, template *types.Template) (*types.Build, []byte, error) {
func (c *client) Parse(v interface{}, pipelineType string, template *yaml.Template) (*yaml.Build, []byte, error) {
var (
p *types.Build
p *yaml.Build
raw []byte
)

Expand Down Expand Up @@ -112,11 +113,11 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ
}

// ParseBytes converts a byte slice to a yaml configuration.
func ParseBytes(data []byte) (*types.Build, []byte, error) {
config := new(types.Build)
func ParseBytes(data []byte) (*yaml.Build, []byte, error) {
config := new(bkYamlTypes.Build)

// unmarshal the bytes into the yaml configuration
err := yaml.Unmarshal(data, config)
err := bkYaml.Unmarshal(data, config)
if err != nil {
return nil, data, fmt.Errorf("unable to unmarshal yaml: %w", err)
}
Expand All @@ -128,11 +129,11 @@ func ParseBytes(data []byte) (*types.Build, []byte, error) {
config.Environment = typesRaw.StringSliceMap{}
}

return config, data, nil
return config.ToYAML(), data, nil
}

// ParseFile converts an os.File into a yaml configuration.
func ParseFile(f *os.File) (*types.Build, []byte, error) {
func ParseFile(f *os.File) (*yaml.Build, []byte, error) {
return ParseReader(f)
}

Expand All @@ -142,7 +143,7 @@ func ParseFileRaw(f *os.File) (string, error) {
}

// ParsePath converts a file path into a yaml configuration.
func ParsePath(p string) (*types.Build, []byte, error) {
func ParsePath(p string) (*yaml.Build, []byte, error) {
// open the file for reading
f, err := os.Open(p)
if err != nil {
Expand All @@ -168,7 +169,7 @@ func ParsePathRaw(p string) (string, error) {
}

// ParseReader converts an io.Reader into a yaml configuration.
func ParseReader(r io.Reader) (*types.Build, []byte, error) {
func ParseReader(r io.Reader) (*yaml.Build, []byte, error) {
// read all the bytes from the reader
data, err := io.ReadAll(r)
if err != nil {
Expand All @@ -190,6 +191,6 @@ func ParseReaderRaw(r io.Reader) (string, error) {
}

// ParseString converts a string into a yaml configuration.
func ParseString(s string) (*types.Build, []byte, error) {
func ParseString(s string) (*yaml.Build, []byte, error) {
return ParseBytes([]byte(s))
}
2 changes: 1 addition & 1 deletion compiler/native/parse_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (

api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/script.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"fmt"
"strings"

"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

// ScriptStages injects the script for each step in every stage in a yaml configuration.
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/script_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/urfave/cli/v2"

"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

func TestNative_ScriptStages(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions compiler/native/substitute.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import (
"fmt"
"strings"

"github.com/buildkite/yaml"
"github.com/drone/envsubst"
"gopkg.in/yaml.v3"

types "github.com/go-vela/server/compiler/types/yaml"
types "github.com/go-vela/server/compiler/types/yaml/yaml"
)

// SubstituteStages replaces every declared environment
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/substitute_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/urfave/cli/v2"

"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

func Test_client_SubstituteStages(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/transform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"fmt"

"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

const (
Expand Down
2 changes: 1 addition & 1 deletion compiler/native/transform_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/urfave/cli/v2"

"github.com/go-vela/server/compiler/types/pipeline"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/internal"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (

"github.com/hashicorp/go-multierror"

"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
"github.com/go-vela/server/constants"
)

Expand Down
2 changes: 1 addition & 1 deletion compiler/native/validate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/urfave/cli/v2"

"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

func TestNative_Validate_NoVersion(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion compiler/template/native/convert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package native
import (
"strings"

"github.com/buildkite/yaml"
"gopkg.in/yaml.v3"

"github.com/go-vela/server/compiler/types/raw"
)
Expand Down
11 changes: 6 additions & 5 deletions compiler/template/native/render.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ import (
"github.com/buildkite/yaml"

"github.com/go-vela/server/compiler/types/raw"
types "github.com/go-vela/server/compiler/types/yaml"
bkTypes "github.com/go-vela/server/compiler/types/yaml/buildkite"
types "github.com/go-vela/server/compiler/types/yaml/yaml"
)

// Render combines the template with the step in the yaml pipeline.
func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}) (*types.Build, error) {
buffer := new(bytes.Buffer)
config := new(types.Build)
config := new(bkTypes.Build)

velaFuncs := funcHandler{envs: convertPlatformVars(environment, name)}
templateFuncMap := map[string]interface{}{
Expand Down Expand Up @@ -57,13 +58,13 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM
config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name)
}

return &types.Build{Metadata: config.Metadata, Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment, Templates: config.Templates}, nil
return &types.Build{Metadata: *config.Metadata.ToYAML(), Steps: *config.Steps.ToYAML(), Secrets: *config.Secrets.ToYAML(), Services: *config.Services.ToYAML(), Environment: config.Environment, Templates: *config.Templates.ToYAML()}, nil
}

// RenderBuild renders the templated build.
func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}) (*types.Build, error) {
buffer := new(bytes.Buffer)
config := new(types.Build)
config := new(bkTypes.Build)

velaFuncs := funcHandler{envs: convertPlatformVars(envs, tmpl)}
templateFuncMap := map[string]interface{}{
Expand Down Expand Up @@ -98,5 +99,5 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st
return nil, fmt.Errorf("unable to unmarshal yaml: %w", err)
}

return config, nil
return config.ToYAML(), nil
}
4 changes: 2 additions & 2 deletions compiler/template/native/render_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ import (
"os"
"testing"

goyaml "github.com/buildkite/yaml"
"github.com/google/go-cmp/cmp"
goyaml "gopkg.in/yaml.v3"

"github.com/go-vela/server/compiler/types/raw"
"github.com/go-vela/server/compiler/types/yaml"
"github.com/go-vela/server/compiler/types/yaml/yaml"
)

func TestNative_Render(t *testing.T) {
Expand Down
Loading
Loading