Implement policy-level tag regex filtering

api/v1alpha1/imagepolicy_types.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux authors
+Copyright 2020, 2021 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -36,6 +36,11 @@ type ImagePolicySpec struct {
 	// selecting the most recent image
 	// +required
 	Policy ImagePolicyChoice `json:"policy"`
+	// FilterTags enables filtering for only a subset of tags based on a set of
+	// rules. If no rules are provided, all the tags from the repository will be
+	// ordered and compared.
+	// +optional
+	FilterTags *TagFilter `json:"filterTags,omitempty"`
 }
 
 // ImagePolicyChoice is a union of all the types of policy that can be
@@ -69,6 +74,18 @@ type AlphabeticalPolicy struct {
 	Order string `json:"order,omitempty"`
 }
 
+// TagFilter enables filtering tags based on a set of defined rules
+type TagFilter struct {
+	// Pattern specifies a regular expression pattern used to filter for image
+	// tags.
+	// +optional
+	Pattern string `json:"pattern"`
+	// Extract allows a capture group to be extracted from the specified regular
+	// expression pattern, useful before tag evaluation.
+	// +optional
+	Extract string `json:"extract"`
+}
+
 // ImagePolicyStatus defines the observed state of ImagePolicy
 type ImagePolicyStatus struct {
 	// LatestImage gives the first in the list of images scanned by

config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml

@@ -41,6 +41,21 @@ spec:
             description: ImagePolicySpec defines the parameters for calculating the
               ImagePolicy
             properties:
+              filterTags:
+                description: FilterTags enables filtering for only a subset of tags
+                  based on a set of rules. If no rules are provided, all the tags
+                  from the repository will be ordered and compared.
+                properties:
+                  extract:
+                    description: Extract allows a capture group to be extracted from
+                      the specified regular expression pattern, useful before tag
+                      evaluation.
+                    type: string
+                  pattern:
+                    description: Pattern specifies a regular expression pattern used
+                      to filter for image tags.
+                    type: string
+                type: object
               imageRepositoryRef:
                 description: ImageRepositoryRef points at the object specifying the
                   image being scanned

controllers/imagepolicy_controller.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux authors
+Copyright 2020, 2021 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -124,7 +124,21 @@ func (r *ImagePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if policer != nil {
 		tags, err := r.Database.Tags(repo.Status.CanonicalImageName)
 		if err == nil {
-			latest, err = policer.Latest(tags)
+			var filter *policy.RegexFilter
+			if pol.Spec.FilterTags != nil {
+				filter, err = policy.NewRegexFilter(pol.Spec.FilterTags.Pattern, pol.Spec.FilterTags.Extract)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+				filter.Apply(tags)
+				tags = filter.Items()
+				latest, err = policer.Latest(tags)
+				if err == nil {
+					latest = filter.GetOriginalTag(latest)
+				}
+			} else {
+				latest, err = policer.Latest(tags)
+			}
 		}
 	}
 	if err != nil {

internal/policy/alphabetical.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux authors
+Copyright 2020, 2021 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -55,12 +55,11 @@ func (p *Alphabetical) Latest(versions []string) (string, error) {
 		return "", fmt.Errorf("version list argument cannot be empty")
 	}
 
-	sorted := sort.StringSlice(versions)
+	var sorted sort.StringSlice = versions
 	if p.Order == AlphabeticalOrderDesc {
 		sort.Sort(sorted)
 	} else {
 		sort.Sort(sort.Reverse(sorted))
 	}
-
 	return sorted[0], nil
 }

internal/policy/alphabetical_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux authors
+Copyright 2020, 2021 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -67,44 +67,54 @@ func TestAlphabetical_Latest(t *testing.T) {
 		expectErr       bool
 	}{
 		{
-			label:           "Ubuntu CalVer",
+			label:           "With Ubuntu CalVer",
 			versions:        []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"},
 			expectedVersion: "20.10",
 		},
-
 		{
-			label:           "Ubuntu CalVer descending",
+			label:           "With Ubuntu CalVer descending",
 			versions:        []string{"16.04", "16.04.1", "16.10", "20.04", "20.10"},
 			order:           AlphabeticalOrderDesc,
 			expectedVersion: "16.04",
 		},
 		{
-			label:           "Ubuntu code names",
+			label:           "With Ubuntu code names",
 			versions:        []string{"xenial", "yakkety", "zesty", "artful", "bionic"},
 			expectedVersion: "zesty",
 		},
 		{
-			label:           "Ubuntu code names descending",
+			label:           "With Ubuntu code names descending",
 			versions:        []string{"xenial", "yakkety", "zesty", "artful", "bionic"},
 			order:           AlphabeticalOrderDesc,
 			expectedVersion: "artful",
 		},
 		{
-			label:           "Timestamps",
+			label:           "With Timestamps",
 			versions:        []string{"1606234201", "1606364286", "1606334092", "1606334284", "1606334201"},
 			expectedVersion: "1606364286",
 		},
 		{
-			label:           "Timestamps desc",
+			label:           "With Unix Timestamps desc",
 			versions:        []string{"1606234201", "1606364286", "1606334092", "1606334284", "1606334201"},
 			order:           AlphabeticalOrderDesc,
 			expectedVersion: "1606234201",
 		},
 		{
-			label:           "Timestamps with prefix",
+			label:           "With Unix Timestamps prefix",
 			versions:        []string{"rel-1606234201", "rel-1606364286", "rel-1606334092", "rel-1606334284", "rel-1606334201"},
 			expectedVersion: "rel-1606364286",
 		},
+		{
+			label:           "With RFC3339",
+			versions:        []string{"2021-01-08T21-18-21Z", "2020-05-08T21-18-21Z", "2021-01-08T19-20-00Z", "1990-01-08T00-20-00Z", "2023-05-08T00-20-00Z"},
+			expectedVersion: "2023-05-08T00-20-00Z",
+		},
+		{
+			label:           "With RFC3339 desc",
+			versions:        []string{"2021-01-08T21-18-21Z", "2020-05-08T21-18-21Z", "2021-01-08T19-20-00Z", "1990-01-08T00-20-00Z", "2023-05-08T00-20-00Z"},
+			order:           AlphabeticalOrderDesc,
+			expectedVersion: "1990-01-08T00-20-00Z",
+		},
 		{
 			label:     "Empty version list",
 			versions:  []string{},
@@ -118,7 +128,6 @@ func TestAlphabetical_Latest(t *testing.T) {
 			if err != nil {
 				t.Fatalf("returned unexpected error: %s", err)
 			}
-
 			latest, err := policy.Latest(tt.versions)
 			if tt.expectErr && err == nil {
 				t.Fatalf("expecting error, got nil")

internal/policy/filter.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+import (
+	"fmt"
+	"regexp"
+)
+
+// RegexFilter represents a regular expression filter
+type RegexFilter struct {
+	filtered map[string]string
+
+	Regexp  *regexp.Regexp
+	Replace string
+}
+
+// NewRegexFilter constructs new RegexFilter object
+func NewRegexFilter(pattern string, replace string) (*RegexFilter, error) {
+	m, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, fmt.Errorf("invalid regular expression pattern '%s': %w", pattern, err)
+	}
+	return &RegexFilter{
+		Regexp:  m,
+		Replace: replace,
+	}, nil
+}
+
+// Apply will construct the filtered list of tags based on the provided list of tags
+func (f *RegexFilter) Apply(list []string) {
+	f.filtered = map[string]string{}
+	for _, item := range list {
+		if submatches := f.Regexp.FindStringSubmatchIndex(item); len(submatches) > 0 {
+			tag := item
+			if f.Replace != "" {
+				result := []byte{}
+				result = f.Regexp.ExpandString(result, f.Replace, item, submatches)
+				tag = string(result)
+			}
+			f.filtered[tag] = item
+		}
+	}
+}
+
+// Items returns the list of filtered tags
+func (f *RegexFilter) Items() []string {
+	var filtered []string
+	for k := range f.filtered {
+		filtered = append(filtered, k)
+	}
+	return filtered
+}
+
+// GetOriginalTag returns the original tag before replace extraction
+func (f *RegexFilter) GetOriginalTag(tag string) string {
+	return f.filtered[tag]
+}

internal/policy/filter_test.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package policy
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestRegexFilter(t *testing.T) {
+	cases := []struct {
+		label    string
+		tags     []string
+		pattern  string
+		extract  string
+		expected []string
+	}{
+		{
+			label:    "none",
+			tags:     []string{"a"},
+			expected: []string{"a"},
+		},
+		{
+			label:    "valid pattern",
+			tags:     []string{"ver1", "ver2", "ver3", "rel1"},
+			pattern:  "^ver",
+			expected: []string{"ver1", "ver2", "ver3"},
+		},
+		{
+			label:    "valid pattern with capture group",
+			tags:     []string{"ver1", "ver2", "ver3", "rel1"},
+			pattern:  `ver(\d+)`,
+			extract:  `$1`,
+			expected: []string{"1", "2", "3"},
+		},
+	}
+	for _, tt := range cases {
+		t.Run(tt.label, func(t *testing.T) {
+			filter := newRegexFilter(tt.pattern, tt.extract)
+			filter.Apply(tt.tags)
+			r := sort.StringSlice(filter.Items())
+			if reflect.DeepEqual(r, tt.expected) {
+				t.Errorf("incorrect value returned, got '%s', expected '%s'", r, tt.expected)
+			}
+		})
+	}
+}
+
+func newRegexFilter(pattern string, extract string) *RegexFilter {
+	f, _ := NewRegexFilter(pattern, extract)
+	return f
+}

internal/policy/policer.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux authors
+Copyright 2020, 2021 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

internal/policy/semver.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux authors
+Copyright 2020, 2021 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -50,13 +50,14 @@ func (p *SemVer) Latest(versions []string) (string, error) {
 	}
 
 	var latestVersion *semver.Version
-	for _, ver := range versions {
-		if v, err := version.ParseVersion(ver); err == nil {
+	for _, tag := range versions {
+		if v, err := version.ParseVersion(tag); err == nil {
 			if p.constraint.Check(v) && (latestVersion == nil || v.GreaterThan(latestVersion)) {
 				latestVersion = v
 			}
 		}
 	}
+
 	if latestVersion != nil {
 		return latestVersion.Original(), nil
 	}