Skip to content

Commit

Permalink
TEP-0075: Object variable replacement on Pipeline/PipelineRun level
Browse files Browse the repository at this point in the history 
Replace the following references with actual value
- the reference to the whole object param defined in PipelineSpec
- the reference to the individual keys of an object param
  • Loading branch information
@chuangw6
chuangw6 committed Jun 21, 2022
1 parent d48cfdd commit 4ceb2d8
Show file tree
Hide file tree
Showing 6 changed files with 572 additions and 37 deletions.
2 changes: 1 addition & 1 deletion pkg/apis/pipeline/v1alpha1/param_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements)
tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements, nil)
if d := cmp.Diff(tt.expectedOutput, tt.args.input); d != "" {
t.Errorf("ApplyReplacements() output did not match expected value %s", diff.PrintWantGot(d))
}
Expand Down
51 changes: 47 additions & 4 deletions pkg/apis/pipeline/v1beta1/param_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,15 +195,58 @@ func (arrayOrString ArrayOrString) MarshalJSON() ([]byte, error) {
}

// ApplyReplacements applyes replacements for ArrayOrString type
func (arrayOrString *ArrayOrString) ApplyReplacements(stringReplacements map[string]string, arrayReplacements map[string][]string) {
if arrayOrString.Type == ParamTypeString {
arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
} else {
func (arrayOrString *ArrayOrString) ApplyReplacements(stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) {
switch arrayOrString.Type {
case ParamTypeArray:
var newArrayVal []string
for _, v := range arrayOrString.ArrayVal {
newArrayVal = append(newArrayVal, substitution.ApplyArrayReplacements(v, stringReplacements, arrayReplacements)...)
}
arrayOrString.ArrayVal = newArrayVal
case ParamTypeObject:
newObjectVal := map[string]string{}
for k, v := range arrayOrString.ObjectVal {
newObjectVal[k] = substitution.ApplyReplacements(v, stringReplacements)
}
arrayOrString.ObjectVal = newObjectVal
default:
arrayOrString.applyOrCorrect(stringReplacements, arrayReplacements, objectReplacements)
}
}

// applyOrCorrect deals with string param whose value can be string literal or a reference to a string/array/object param/result.
// If the value of arrayOrString is a reference to array or object, the type will be corrected from string to array/object.
func (arrayOrString *ArrayOrString) applyOrCorrect(stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) {
stringVal := arrayOrString.StringVal

// if the stringVal is a string literal or a string that mixed with var references
// just do the normal string replacement
if !exactVariableSubstitutionRegex.MatchString(stringVal) {
arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
return
}

// trim the head "$(" and the tail ")" or "[*])"
// i.e. get "params.name" from "$(params.name)" or "$(params.name[*])"
trimedStringVal := strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(stringVal, "$("), ")"), "[*]")

// if the stringVal is a reference to a string param
if _, ok := stringReplacements[trimedStringVal]; ok {
arrayOrString.StringVal = substitution.ApplyReplacements(arrayOrString.StringVal, stringReplacements)
}

// if the stringVal is a reference to an array param, we need to change the type other than apply replacement
if _, ok := arrayReplacements[trimedStringVal]; ok {
arrayOrString.StringVal = ""
arrayOrString.ArrayVal = substitution.ApplyArrayReplacements(stringVal, stringReplacements, arrayReplacements)
arrayOrString.Type = ParamTypeArray
}

// if the stringVal is a reference an object param, we need to change the type other than apply replacement
if _, ok := objectReplacements[trimedStringVal]; ok {
arrayOrString.StringVal = ""
arrayOrString.ObjectVal = objectReplacements[trimedStringVal]
arrayOrString.Type = ParamTypeObject
}
}

Expand Down
73 changes: 71 additions & 2 deletions pkg/apis/pipeline/v1beta1/param_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
input *v1beta1.ArrayOrString
stringReplacements map[string]string
arrayReplacements map[string][]string
objectReplacements map[string]map[string]string
}
tests := []struct {
name string
Expand All @@ -176,7 +177,15 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
},
expectedOutput: v1beta1.NewArrayOrString("an", "array"),
}, {
name: "string replacements on string",
name: "single string replacement on string",
args: args{
input: v1beta1.NewArrayOrString("$(params.myString1)"),
stringReplacements: map[string]string{"params.myString1": "value1", "params.myString2": "value2"},
arrayReplacements: map[string][]string{"arraykey": {"array", "value"}, "sdfdf": {"asdf", "sdfsd"}},
},
expectedOutput: v1beta1.NewArrayOrString("value1"),
}, {
name: "multiple string replacements on string",
args: args{
input: v1beta1.NewArrayOrString("astring$(some) asdf $(anotherkey)"),
stringReplacements: map[string]string{"some": "value", "anotherkey": "value"},
Expand Down Expand Up @@ -207,10 +216,70 @@ func TestArrayOrString_ApplyReplacements(t *testing.T) {
arrayReplacements: map[string][]string{"arraykey": {}},
},
expectedOutput: v1beta1.NewArrayOrString("firstvalue", "lastvalue"),
}, {
name: "array replacement on string val",
args: args{
input: v1beta1.NewArrayOrString("$(params.myarray)"),
arrayReplacements: map[string][]string{"params.myarray": {"a", "b", "c"}},
},
expectedOutput: v1beta1.NewArrayOrString("a", "b", "c"),
}, {
name: "array star replacement on string val",
args: args{
input: v1beta1.NewArrayOrString("$(params.myarray[*])"),
arrayReplacements: map[string][]string{"params.myarray": {"a", "b", "c"}},
},
expectedOutput: v1beta1.NewArrayOrString("a", "b", "c"),
}, {
name: "object replacement on string val",
args: args{
input: v1beta1.NewArrayOrString("$(params.object)"),
objectReplacements: map[string]map[string]string{
"params.object": {
"url": "abc.com",
"commit": "af234",
},
},
},
expectedOutput: v1beta1.NewObject(map[string]string{
"url": "abc.com",
"commit": "af234",
}),
}, {
name: "object star replacement on string val",
args: args{
input: v1beta1.NewArrayOrString("$(params.object[*])"),
objectReplacements: map[string]map[string]string{
"params.object": {
"url": "abc.com",
"commit": "af234",
},
},
},
expectedOutput: v1beta1.NewObject(map[string]string{
"url": "abc.com",
"commit": "af234",
}),
}, {
name: "string replacement on object individual variables",
args: args{
input: v1beta1.NewObject(map[string]string{
"key1": "$(mystring)",
"key2": "$(anotherObject.key)",
}),
stringReplacements: map[string]string{
"mystring": "foo",
"anotherObject.key": "bar",
},
},
expectedOutput: v1beta1.NewObject(map[string]string{
"key1": "foo",
"key2": "bar",
}),
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements)
tt.args.input.ApplyReplacements(tt.args.stringReplacements, tt.args.arrayReplacements, tt.args.objectReplacements)
if d := cmp.Diff(tt.expectedOutput, tt.args.input); d != "" {
t.Errorf("ApplyReplacements() output did not match expected value %s", diff.PrintWantGot(d))
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/apis/pipeline/v1beta1/resultref.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,16 @@ const (
ResultResultPart = "results"
// TODO(#2462) use one regex across all substitutions
// variableSubstitutionFormat matches format like $result.resultname, $result.resultname[int] and $result.resultname[*]
variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9])*\*?\])?\)`
variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9]+|\*)\])?\)`
exactVariableSubstitutionFormat = `^\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9]+|\*)\])?\)$`
// arrayIndexing will match all `[int]` and `[*]` for parseExpression
arrayIndexing = `\[([0-9])*\*?\]`
// ResultNameFormat Constant used to define the the regex Result.Name should follow
ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
)

var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
var exactVariableSubstitutionRegex = regexp.MustCompile(exactVariableSubstitutionFormat)
var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
var arrayIndexingRegex = regexp.MustCompile(arrayIndexing)

Expand Down
86 changes: 59 additions & 27 deletions pkg/reconciler/pipelinerun/resources/apply.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package resources
import (
"context"
"fmt"
"log"
"strconv"
"strings"

Expand All @@ -34,45 +35,65 @@ import (
func ApplyParameters(ctx context.Context, p *v1beta1.PipelineSpec, pr *v1beta1.PipelineRun) *v1beta1.PipelineSpec {
// This assumes that the PipelineRun inputs have been validated against what the Pipeline requests.

// stringReplacements is used for standard single-string stringReplacements, while arrayReplacements contains arrays
// that need to be further processed.
// stringReplacements is used for standard single-string stringReplacements,
// while arrayReplacements/objectReplacements contains arrays/objects that need to be further processed.
stringReplacements := map[string]string{}
arrayReplacements := map[string][]string{}
objectReplacements := map[string]map[string]string{}

patterns := []string{
"params.%s",
"params[%q]",
"params['%s']",
}

// reference pattern for object individual keys params.<object_param_name>.<key_name>
objectIndividualVariablePattern := "params.%s.%s"

// Set all the default stringReplacements
for _, p := range p.Params {
if p.Default != nil {
if p.Default.Type == v1beta1.ParamTypeString {
switch p.Default.Type {
case v1beta1.ParamTypeArray:
for _, pattern := range patterns {
stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.StringVal
arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ArrayVal
}
} else {
case v1beta1.ParamTypeObject:
for _, pattern := range patterns {
arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ArrayVal
objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ObjectVal
}
for k, v := range p.Default.ObjectVal {
stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v
}
default:
for _, pattern := range patterns {
stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.StringVal
}
}
}
}
// Set and overwrite params with the ones from the PipelineRun
for _, p := range pr.Spec.Params {
if p.Value.Type == v1beta1.ParamTypeString {
switch p.Value.Type {
case v1beta1.ParamTypeArray:
for _, pattern := range patterns {
stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.StringVal
arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ArrayVal
}
} else {
case v1beta1.ParamTypeObject:
for _, pattern := range patterns {
arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ArrayVal
objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ObjectVal
}
for k, v := range p.Value.ObjectVal {
stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v
}
default:
for _, pattern := range patterns {
stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.StringVal
}
}
}

return ApplyReplacements(ctx, p, stringReplacements, arrayReplacements)
return ApplyReplacements(ctx, p, stringReplacements, arrayReplacements, objectReplacements)
}

// ApplyContexts applies the substitution from $(context.(pipelineRun|pipeline).*) with the specified values.
Expand All @@ -84,7 +105,7 @@ func ApplyContexts(ctx context.Context, spec *v1beta1.PipelineSpec, pipelineName
"context.pipelineRun.namespace": pr.Namespace,
"context.pipelineRun.uid": string(pr.ObjectMeta.UID),
}
return ApplyReplacements(ctx, spec, replacements, map[string][]string{})
return ApplyReplacements(ctx, spec, replacements, map[string][]string{}, map[string]map[string]string{})
}

// ApplyPipelineTaskContexts applies the substitution from $(context.pipelineTask.*) with the specified values.
Expand All @@ -94,8 +115,8 @@ func ApplyPipelineTaskContexts(pt *v1beta1.PipelineTask) *v1beta1.PipelineTask {
replacements := map[string]string{
"context.pipelineTask.retries": strconv.Itoa(pt.Retries),
}
pt.Params = replaceParamValues(pt.Params, replacements, map[string][]string{})
pt.Matrix = replaceParamValues(pt.Matrix, replacements, map[string][]string{})
pt.Params = replaceParamValues(pt.Params, replacements, map[string][]string{}, map[string]map[string]string{})
pt.Matrix = replaceParamValues(pt.Matrix, replacements, map[string][]string{}, map[string]map[string]string{})
return pt
}

Expand All @@ -105,7 +126,7 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul
for _, resolvedPipelineRunTask := range targets {
if resolvedPipelineRunTask.PipelineTask != nil {
pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil)
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil, nil)
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, nil)
resolvedPipelineRunTask.PipelineTask = pipelineTask
}
Expand All @@ -117,7 +138,7 @@ func ApplyPipelineTaskStateContext(state PipelineRunState, replacements map[stri
for _, resolvedPipelineRunTask := range state {
if resolvedPipelineRunTask.PipelineTask != nil {
pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
pipelineTask.Params = replaceParamValues(pipelineTask.Params, replacements, nil)
pipelineTask.Params = replaceParamValues(pipelineTask.Params, replacements, nil, nil)
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(replacements, nil)
resolvedPipelineRunTask.PipelineTask = pipelineTask
}
Expand All @@ -137,39 +158,44 @@ func ApplyWorkspaces(ctx context.Context, p *v1beta1.PipelineSpec, pr *v1beta1.P
key := fmt.Sprintf("workspaces.%s.bound", boundWorkspace.Name)
replacements[key] = "true"
}
return ApplyReplacements(ctx, p, replacements, map[string][]string{})
return ApplyReplacements(ctx, p, replacements, map[string][]string{}, map[string]map[string]string{})
}

// ApplyReplacements replaces placeholders for declared parameters with the specified replacements.
func ApplyReplacements(ctx context.Context, p *v1beta1.PipelineSpec, replacements map[string]string, arrayReplacements map[string][]string) *v1beta1.PipelineSpec {
func ApplyReplacements(ctx context.Context, p *v1beta1.PipelineSpec, replacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) *v1beta1.PipelineSpec {
p = p.DeepCopy()

for i := range p.Tasks {
p.Tasks[i].Params = replaceParamValues(p.Tasks[i].Params, replacements, arrayReplacements)
p.Tasks[i].Matrix = replaceParamValues(p.Tasks[i].Matrix, replacements, arrayReplacements)
p.Tasks[i].Params = replaceParamValues(p.Tasks[i].Params, replacements, arrayReplacements, objectReplacements)
p.Tasks[i].Matrix = replaceParamValues(p.Tasks[i].Matrix, replacements, arrayReplacements, objectReplacements)
for j := range p.Tasks[i].Workspaces {
p.Tasks[i].Workspaces[j].SubPath = substitution.ApplyReplacements(p.Tasks[i].Workspaces[j].SubPath, replacements)
}
log.Println("chuangw", arrayReplacements)
p.Tasks[i].WhenExpressions = p.Tasks[i].WhenExpressions.ReplaceWhenExpressionsVariables(replacements, arrayReplacements)
p.Tasks[i], replacements, arrayReplacements = propagateParams(ctx, p.Tasks[i], replacements, arrayReplacements)
p.Tasks[i], replacements, arrayReplacements, objectReplacements = propagateParams(ctx, p.Tasks[i], replacements, arrayReplacements, objectReplacements)
}

for i := range p.Finally {
p.Finally[i].Params = replaceParamValues(p.Finally[i].Params, replacements, arrayReplacements)
p.Finally[i].Matrix = replaceParamValues(p.Finally[i].Matrix, replacements, arrayReplacements)
p.Finally[i].Params = replaceParamValues(p.Finally[i].Params, replacements, arrayReplacements, objectReplacements)
p.Finally[i].Matrix = replaceParamValues(p.Finally[i].Matrix, replacements, arrayReplacements, objectReplacements)
p.Finally[i].WhenExpressions = p.Finally[i].WhenExpressions.ReplaceWhenExpressionsVariables(replacements, arrayReplacements)
}

return p
}

func propagateParams(ctx context.Context, t v1beta1.PipelineTask, replacements map[string]string, arrayReplacements map[string][]string) (v1beta1.PipelineTask, map[string]string, map[string][]string) {
func propagateParams(ctx context.Context, t v1beta1.PipelineTask, replacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) (v1beta1.PipelineTask, map[string]string, map[string][]string, map[string]map[string]string) {
if t.TaskSpec != nil && config.FromContextOrDefaults(ctx).FeatureFlags.EnableAPIFields == "alpha" {
patterns := []string{
"params.%s",
"params[%q]",
"params['%s']",
}

// reference pattern for object individual keys params.<object_param_name>.<key_name>
objectIndividualVariablePattern := "params.%s.%s"

// check if there are task parameters defined that match the params at pipeline level
if len(t.Params) > 0 {
for _, par := range t.Params {
Expand All @@ -182,17 +208,23 @@ func propagateParams(ctx context.Context, t v1beta1.PipelineTask, replacements m
if _, ok := arrayReplacements[checkName]; ok {
arrayReplacements[checkName] = par.Value.ArrayVal
}
if _, ok := objectReplacements[checkName]; ok {
objectReplacements[checkName] = par.Value.ObjectVal
for k, v := range par.Value.ObjectVal {
replacements[fmt.Sprintf(objectIndividualVariablePattern, par.Name, k)] = v
}
}
}
}
}
t.TaskSpec.TaskSpec = *resources.ApplyReplacements(&t.TaskSpec.TaskSpec, replacements, arrayReplacements)
}
return t, replacements, arrayReplacements
return t, replacements, arrayReplacements, objectReplacements
}

func replaceParamValues(params []v1beta1.Param, stringReplacements map[string]string, arrayReplacements map[string][]string) []v1beta1.Param {
func replaceParamValues(params []v1beta1.Param, stringReplacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) []v1beta1.Param {
for i := range params {
params[i].Value.ApplyReplacements(stringReplacements, arrayReplacements)
params[i].Value.ApplyReplacements(stringReplacements, arrayReplacements, objectReplacements)
}
return params
}
Expand Down
Loading

0 comments on commit 4ceb2d8

Please sign in to comment.