Skip to content

Commit

Permalink
enhance(compiler): cache templates per compilation (#1224)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ecrupper
ecrupper authored Dec 3, 2024
1 parent 21761c9 commit aa973fe
Show file tree
Hide file tree
Showing 5 changed files with 263 additions and 8 deletions.
14 changes: 11 additions & 3 deletions compiler/native/compile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,17 @@ func (c *client) compileInline(ctx context.Context, p *yaml.Build, depth int) (*
}

for _, template := range p.Templates {
bytes, err := c.getTemplate(ctx, template, template.Name)
if err != nil {
return nil, err
var (
bytes []byte
found bool
err error
)

if bytes, found = c.TemplateCache[template.Source]; !found {
bytes, err = c.getTemplate(ctx, template, template.Name)
if err != nil {
return nil, err
}
}

format := template.Format
Expand Down
15 changes: 12 additions & 3 deletions compiler/native/expand.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,16 @@ func (c *client) ExpandSteps(ctx context.Context, s *yaml.Build, tmpls map[strin
return s, err
}

bytes, err := c.getTemplate(ctx, tmpl, step.Template.Name)
if err != nil {
return s, err
var (
bytes []byte
found bool
)

if bytes, found = c.TemplateCache[tmpl.Source]; !found {
bytes, err = c.getTemplate(ctx, tmpl, step.Template.Name)
if err != nil {
return s, err
}
}

// initialize variable map if not parsed from config
Expand Down Expand Up @@ -341,6 +348,8 @@ func (c *client) getTemplate(ctx context.Context, tmpl *yaml.Template, name stri
return bytes, fmt.Errorf("unsupported template type: %v", tmpl.Type)
}

c.TemplateCache[tmpl.Source] = bytes

return bytes, nil
}

Expand Down
231 changes: 231 additions & 0 deletions compiler/native/expand_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -954,6 +954,237 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) {
}
}

func TestNative_ExpandStepsDuplicateCalls(t *testing.T) {
// setup context
gin.SetMode(gin.TestMode)

resp := httptest.NewRecorder()
_, engine := gin.CreateTestContext(resp)

testCallsMap := make(map[string]bool)

// setup mock server
engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) {
testCallKey := c.Param("path")

if refQuery, exists := c.GetQuery("ref"); exists {
testCallKey += refQuery
}

// this is the real test
if testCallsMap[testCallKey] {
t.Errorf("ExpandSteps() called the same template %s twice", c.Param("path"))
}

testCallsMap[c.Param("path")] = true
body, err := convertFileToGithubResponse(c.Param("path"))
if err != nil {
t.Error(err)
}
c.JSON(http.StatusOK, body)
})

s := httptest.NewServer(engine)
defer s.Close()

// setup types
set := flag.NewFlagSet("test", 0)
set.Bool("github-driver", true, "doc")
set.String("github-url", s.URL, "doc")
set.String("github-token", "", "doc")
set.Int("max-template-depth", 5, "doc")
set.String("clone-image", defaultCloneImage, "doc")
c := cli.NewContext(nil, set, nil)

testRepo := new(api.Repo)

testRepo.SetID(1)
testRepo.SetOrg("foo")
testRepo.SetName("bar")

tests := []struct {
name string
tmpls map[string]*yaml.Template
}{
{
name: "GitHub",
tmpls: map[string]*yaml.Template{
"gradle": {
Name: "gradle",
Source: "github.example.com/foo/bar/long_template.yml",
Type: "github",
},
},
},
}

steps := yaml.StepSlice{
&yaml.Step{
Name: "sample",
Template: yaml.StepTemplate{
Name: "gradle",
Variables: map[string]interface{}{
"image": "openjdk:latest",
"environment": "{ GRADLE_USER_HOME: .gradle, GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false }",
"pull_policy": "pull: true",
},
},
},
&yaml.Step{
Name: "sample-dup",
Template: yaml.StepTemplate{
Name: "gradle",
Variables: map[string]interface{}{
"image": "openjdk:latest",
"environment": "{ GRADLE_USER_HOME: .gradle, GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false }",
"pull_policy": "pull: true",
},
},
},
}

globalEnvironment := raw.StringSliceMap{
"foo": "test1",
"bar": "test2",
}

wantSteps := yaml.StepSlice{
&yaml.Step{
Commands: []string{"./gradlew downloadDependencies"},
Environment: raw.StringSliceMap{
"GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false",
"GRADLE_USER_HOME": ".gradle",
},
Image: "openjdk:latest",
Name: "sample_install",
Pull: "always",
},
&yaml.Step{
Commands: []string{"./gradlew check"},
Environment: raw.StringSliceMap{
"GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false",
"GRADLE_USER_HOME": ".gradle",
},
Image: "openjdk:latest",
Name: "sample_test",
Pull: "always",
},
&yaml.Step{
Commands: []string{"./gradlew build", "echo gradle"},
Environment: raw.StringSliceMap{
"GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false",
"GRADLE_USER_HOME": ".gradle",
},
Image: "openjdk:latest",
Name: "sample_build",
Pull: "always",
},
&yaml.Step{
Commands: []string{"./gradlew downloadDependencies"},
Environment: raw.StringSliceMap{
"GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false",
"GRADLE_USER_HOME": ".gradle",
},
Image: "openjdk:latest",
Name: "sample-dup_install",
Pull: "always",
},
&yaml.Step{
Commands: []string{"./gradlew check"},
Environment: raw.StringSliceMap{
"GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false",
"GRADLE_USER_HOME": ".gradle",
},
Image: "openjdk:latest",
Name: "sample-dup_test",
Pull: "always",
},
&yaml.Step{
Commands: []string{"./gradlew build", "echo gradle"},
Environment: raw.StringSliceMap{
"GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false",
"GRADLE_USER_HOME": ".gradle",
},
Image: "openjdk:latest",
Name: "sample-dup_build",
Pull: "always",
},
}

wantSecrets := yaml.SecretSlice{
&yaml.Secret{
Name: "docker_username",
Key: "org/repo/foo/bar",
Engine: "native",
Type: "repo",
Origin: yaml.Origin{},
Pull: "build_start",
},
&yaml.Secret{
Name: "foo_password",
Key: "org/repo/foo/password",
Engine: "vault",
Type: "repo",
Origin: yaml.Origin{},
Pull: "build_start",
},
}

wantServices := yaml.ServiceSlice{
&yaml.Service{
Image: "postgres:12",
Name: "postgres",
Pull: "not_present",
},
}

wantEnvironment := raw.StringSliceMap{
"foo": "test1",
"bar": "test2",
"star": "test3",
}

// run test
compiler, err := FromCLIContext(c)
if err != nil {
t.Errorf("Creating new compiler returned err: %v", err)
}

compiler.WithCommit("123abc456def").WithRepo(testRepo)

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
build, err := compiler.ExpandSteps(
context.Background(),
&yaml.Build{
Steps: steps,
Services: yaml.ServiceSlice{},
Environment: globalEnvironment,
},
test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth())
if err != nil {
t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err)
}

if diff := cmp.Diff(build.Steps, wantSteps); diff != "" {
t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff)
}

if diff := cmp.Diff(build.Secrets, wantSecrets); diff != "" {
t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff)
}

if diff := cmp.Diff(build.Services, wantServices); diff != "" {
t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff)
}

if diff := cmp.Diff(build.Environment, wantEnvironment); diff != "" {
t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff)
}
})
}
}

func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) {
// setup context
gin.SetMode(gin.TestMode)
Expand Down
4 changes: 4 additions & 0 deletions compiler/native/native.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type client struct {
PrivateGithub registry.Service
UsePrivateGithub bool
ModificationService ModificationConfig
TemplateCache map[string][]byte

settings.Compiler

Expand Down Expand Up @@ -102,6 +103,8 @@ func FromCLIContext(ctx *cli.Context) (*client, error) {
c.UsePrivateGithub = true
}

c.TemplateCache = make(map[string][]byte)

return c, nil
}

Expand Down Expand Up @@ -131,6 +134,7 @@ func (c *client) Duplicate() compiler.Engine {
cc.CloneImage = c.CloneImage
cc.TemplateDepth = c.TemplateDepth
cc.StarlarkExecLimit = c.StarlarkExecLimit
cc.TemplateCache = c.TemplateCache

return cc
}
Expand Down
7 changes: 5 additions & 2 deletions compiler/native/native_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ func TestNative_New(t *testing.T) {
c := cli.NewContext(nil, set, nil)
public, _ := github.New(context.Background(), "", "")
want := &client{
Github: public,
Compiler: settings.CompilerMockEmpty(),
Github: public,
Compiler: settings.CompilerMockEmpty(),
TemplateCache: make(map[string][]byte),
}
want.SetCloneImage(defaultCloneImage)

Expand Down Expand Up @@ -56,6 +57,7 @@ func TestNative_New_PrivateGithub(t *testing.T) {
Github: public,
PrivateGithub: private,
UsePrivateGithub: true,
TemplateCache: make(map[string][]byte),
Compiler: settings.CompilerMockEmpty(),
}
want.SetCloneImage(defaultCloneImage)
Expand Down Expand Up @@ -88,6 +90,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) {
Github: public,
PrivateGithub: private,
UsePrivateGithub: true,
TemplateCache: make(map[string][]byte),
Compiler: settings.CompilerMockEmpty(),
}
want.SetCloneImage(defaultCloneImage)
Expand Down

0 comments on commit aa973fe

Please sign in to comment.