Skip to content

Commit

Permalink
Implement policy-level tag regex filtering
Browse files Browse the repository at this point in the history
Tag regex filtering allows the user to filter tags based on a regular
expression pattern and enables tag version extraction through capture
group replacement reference.

Fixes #73

Signed-off-by: Aurel Canciu <[email protected]>
  • Loading branch information
relu committed Jan 11, 2021
1 parent d768bfe commit bc7f16e
Show file tree
Hide file tree
Showing 12 changed files with 302 additions and 25 deletions.
2 changes: 0 additions & 2 deletions api/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,6 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fluxcd/pkg/apis/meta v0.4.0 h1:JChqB9GGgorW9HWKxirTVV0rzrcLyzBaVjinmqZ0iHA=
github.com/fluxcd/pkg/apis/meta v0.4.0/go.mod h1:wOzQQx8CdtUQCGaLzqGu4QgnNxYkI6/wvdvlovxWhF0=
github.com/fluxcd/pkg/apis/meta v0.5.0 h1:FaU++mQY0g4sVVl+hG+vk0CXBLbb4EVfRuzs3IjLXvo=
github.com/fluxcd/pkg/apis/meta v0.5.0/go.mod h1:aEUuZIawboAAFLlYz/juVJ7KNmlWbBtJFYkOWWmGUR4=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
Expand Down
17 changes: 17 additions & 0 deletions api/v1alpha1/imagepolicy_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ type ImagePolicySpec struct {
// selecting the most recent image
// +required
Policy ImagePolicyChoice `json:"policy"`
// FilterTags enables filtering for only a subset of tags based on a set of
// rules. If no rules are provided, all the tags from the repository will be
// ordered and compared.
// +optional
FilterTags *TagFilter `json:"filterTags,omitempty"`
}

// ImagePolicyChoice is a union of all the types of policy that can be
Expand Down Expand Up @@ -69,6 +74,18 @@ type AlphabeticalPolicy struct {
Order string `json:"order,omitempty"`
}

// TagFilter enables filtering tags based on a set of defined rules
type TagFilter struct {
// Pattern specifies a regular expression pattern used to filter for image
// tags.
// +optional
Pattern string `json:"pattern"`
// Extract allows a capture group to be extracted from the specified regular
// expression pattern, useful before tag evaluation.
// +optional
Extract string `json:"extract"`
}

// ImagePolicyStatus defines the observed state of ImagePolicy
type ImagePolicyStatus struct {
// LatestImage gives the first in the list of images scanned by
Expand Down
20 changes: 20 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,21 @@ spec:
description: ImagePolicySpec defines the parameters for calculating the
ImagePolicy
properties:
filterTags:
description: FilterTags enables filtering for only a subset of tags
based on a set of rules. If no rules are provided, all the tags
from the repository will be ordered and compared.
properties:
extract:
description: Extract allows a capture group to be extracted from
the specified regular expression pattern, useful before tag
evaluation.
type: string
pattern:
description: Pattern specifies a regular expression pattern used
to filter for image tags.
type: string
type: object
imageRepositoryRef:
description: ImageRepositoryRef points at the object specifying the
image being scanned
Expand Down
10 changes: 9 additions & 1 deletion controllers/imagepolicy_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,15 @@ func (r *ImagePolicyReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error)
if policer != nil {
tags, err := r.Database.Tags(repo.Status.CanonicalImageName)
if err == nil {
latest, err = policer.Latest(tags)
var filter *policy.RegexFilter
if pol.Spec.FilterTags != nil {
filter, err = policy.NewRegexFilter(pol.Spec.FilterTags.Pattern, pol.Spec.FilterTags.Extract)
if err != nil {
return ctrl.Result{}, err
}
filter.Apply(tags)
}
latest, err = policer.Latest(tags, filter)
}
}
if err != nil {
Expand Down
15 changes: 12 additions & 3 deletions internal/policy/alphabetical.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,17 +50,26 @@ func NewAlphabetical(order string) (*Alphabetical, error) {
}

// Latest returns latest version from a provided list of strings
func (p *Alphabetical) Latest(versions []string) (string, error) {
func (p *Alphabetical) Latest(versions []string, filter *RegexFilter) (string, error) {
if len(versions) == 0 {
return "", fmt.Errorf("version list argument cannot be empty")
}

sorted := sort.StringSlice(versions)
var sorted sort.StringSlice = versions
if filter != nil {
sorted = filter.Items()
}

if p.Order == AlphabeticalOrderDesc {
sort.Sort(sorted)
} else {
sort.Sort(sort.Reverse(sorted))
}

return sorted[0], nil
latest := sorted[0]
if filter != nil {
return filter.GetOriginalTag(latest), nil
}

return latest, nil
}
51 changes: 41 additions & 10 deletions internal/policy/alphabetical_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,48 +63,77 @@ func TestAlphabetical_Latest(t *testing.T) {
label string
order string
versions []string
filter *RegexFilter
expectedVersion string
expectErr bool
}{
{
label: "Ubuntu CalVer",
label: "With Ubuntu CalVer",
versions: []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"},
expectedVersion: "20.10",
},

{
label: "Ubuntu CalVer descending",
label: "With Ubuntu CalVer prefix include",
versions: []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"},
filter: newRegexFilter("16", ""),
expectedVersion: "16.10",
},
{
label: "With Ubuntu CalVer descending",
versions: []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"},
order: AlphabeticalOrderDesc,
expectedVersion: "16.04",
},
{
label: "Ubuntu code names",
label: "With Ubuntu code names",
versions: []string{"xenial", "yakkety", "zesty", "artful", "bionic"},
expectedVersion: "zesty",
},
{
label: "Ubuntu code names descending",
label: "With Ubuntu code names descending",
versions: []string{"xenial", "yakkety", "zesty", "artful", "bionic"},
order: AlphabeticalOrderDesc,
expectedVersion: "artful",
},
{
label: "Timestamps",
label: "With Timestamps",
versions: []string{"1606234201", "1606364286", "1606334092", "1606334284", "1606334201"},
expectedVersion: "1606364286",
},
{
label: "Timestamps desc",
label: "With Unix Timestamps desc",
versions: []string{"1606234201", "1606364286", "1606334092", "1606334284", "1606334201"},
order: AlphabeticalOrderDesc,
expectedVersion: "1606234201",
},
{
label: "Timestamps with prefix",
label: "With Unix Timestamps prefix",
versions: []string{"rel-1606234201", "rel-1606364286", "rel-1606334092", "rel-1606334284", "rel-1606334201"},
expectedVersion: "rel-1606364286",
},
{
label: "With RFC3339",
versions: []string{"2021-01-08T21-18-21Z", "2020-05-08T21-18-21Z", "2021-01-08T19-20-00Z", "1990-01-08T00-20-00Z", "2023-05-08T00-20-00Z"},
expectedVersion: "2023-05-08T00-20-00Z",
},
{
label: "With RFC3339 desc",
versions: []string{"2021-01-08T21-18-21Z", "2020-05-08T21-18-21Z", "2021-01-08T19-20-00Z", "1990-01-08T00-20-00Z", "2023-05-08T00-20-00Z"},
order: AlphabeticalOrderDesc,
expectedVersion: "1990-01-08T00-20-00Z",
},
{
label: "With prefix filter",
versions: []string{"rel-1", "rel-2", "rel-3", "rel-4", "dev-5", "dev-6"},
filter: newRegexFilter("rel-", ""),
expectedVersion: "rel-4",
},
{
label: "With prefix include replace capture group",
versions: []string{"rel-11", "rel-12", "rel-13", "rel-15", "dev-5", "dev-6", "ver-12", "ver-10", "gen-50"},
filter: newRegexFilter("(rel|ver)-(?P<tag>.*)", "$tag"),
expectedVersion: "rel-15",
},
{
label: "Empty version list",
versions: []string{},
Expand All @@ -118,8 +147,10 @@ func TestAlphabetical_Latest(t *testing.T) {
if err != nil {
t.Fatalf("returned unexpected error: %s", err)
}

latest, err := policy.Latest(tt.versions)
if tt.filter != nil {
tt.filter.Apply(tt.versions)
}
latest, err := policy.Latest(tt.versions, tt.filter)
if tt.expectErr && err == nil {
t.Fatalf("expecting error, got nil")
}
Expand Down
70 changes: 70 additions & 0 deletions internal/policy/filter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
"fmt"
"regexp"
)

// RegexFilter represents a regular expression filter
type RegexFilter struct {
filtered map[string]string

Regexp *regexp.Regexp
Replace string
}

// NewRegexFilter constructs new RegexFilter object
func NewRegexFilter(pattern string, replace string) (*RegexFilter, error) {
m, err := regexp.Compile(pattern)
if err != nil {
return nil, fmt.Errorf("invalid regular expression pattern '%s': %s", pattern, err.Error())
}
return &RegexFilter{
Regexp: m,
Replace: replace,
}, nil
}

// Apply will construct the filtered list of tags based on the provided list of tags
func (f *RegexFilter) Apply(list []string) {
f.filtered = map[string]string{}
for _, item := range list {
if f.Regexp.MatchString(item) {
tag := item
if f.Replace != "" {
tag = f.Regexp.ReplaceAllString(item, f.Replace)
}
f.filtered[tag] = item
}
}
}

// Items returns the list of filtered tags
func (f *RegexFilter) Items() []string {
var filtered []string
for k := range f.filtered {
filtered = append(filtered, k)
}
return filtered
}

// GetOriginalTag returns the original tag before replace extraction
func (f *RegexFilter) GetOriginalTag(tag string) string {
return f.filtered[tag]
}
67 changes: 67 additions & 0 deletions internal/policy/filter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
"reflect"
"sort"
"testing"
)

func TestRegexFilter(t *testing.T) {
cases := []struct {
label string
tags []string
pattern string
extract string
expected []string
}{
{
label: "none",
tags: []string{"a"},
expected: []string{"a"},
},
{
label: "valid pattern",
tags: []string{"ver1", "ver2", "ver3", "rel1"},
pattern: "^ver",
expected: []string{"ver1", "ver2", "ver3"},
},
{
label: "valid pattern with capture group",
tags: []string{"ver1", "ver2", "ver3", "rel1"},
pattern: `ver(\d+)`,
extract: `$1`,
expected: []string{"1", "2", "3"},
},
}
for _, tt := range cases {
t.Run(tt.label, func(t *testing.T) {
filter := newRegexFilter(tt.pattern, tt.extract)
filter.Apply(tt.tags)
r := sort.StringSlice(filter.Items())
if reflect.DeepEqual(r, tt.expected) {
t.Errorf("incorrect value returned, got '%s', expected '%s'", r, tt.expected)
}
})
}
}

func newRegexFilter(pattern string, extract string) *RegexFilter {
f, _ := NewRegexFilter(pattern, extract)
return f
}
2 changes: 1 addition & 1 deletion internal/policy/policer.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,5 @@ package policy

// Policer is an interface representing a policy implementation type
type Policer interface {
Latest([]string) (string, error)
Latest([]string, *RegexFilter) (string, error)
}
Loading

0 comments on commit bc7f16e

Please sign in to comment.