protocol: LabelSelectors support prefix matches

In uses of Gazette, labels often encode a hierarchical namespace
such as "foo/bar/baz", and for such labels it's desireable to be able to
create a LabelSelector that includes or excludes an entire sub-hierarchy
of potential label values, such as including or excluding any label
that begins with "foo/bar/".

Extend Label to have a Prefix field which may only be set in the context
of a LabelSelector (and is a Validate() error otherwise).

In a LabelSelector context, Prefix instructs selector matching to match
any label value which is prefixed by the given selector label.

Introduce a convention of "my-label:prefix" as a special suffix which
indicates that prefix matching is desired, and update LabelSelector
parsing to round-trip such ":prefix" suffixes.

Update the implementation of the special meta-label "prefix" to be in
terms of a "name:prefix" selector label, and back out the bespoke
implementation that has until-now been used for journal name prefix
matching.

broker/list_apply_api.go

@@ -39,6 +39,18 @@ func (svc *Service) List(ctx context.Context, req *pb.ListRequest) (resp *pb.Lis
 	}
 	var metaLabels, allLabels pb.LabelSet
 
+	// Gazette has historically offered a special "prefix" label which matches
+	// slash-terminated prefixes of a journal name. Today, it's implemented in
+	// terms of a LabelSelector prefix match.
+	for _, set := range []*pb.LabelSet{&req.Selector.Include, &req.Selector.Exclude} {
+		if prefix := set.ValuesOf("prefix"); len(prefix) != 0 {
+			for _, val := range prefix {
+				set.AddValue("name:prefix", val)
+			}
+			set.Remove("prefix")
+		}
+	}
+
 	defer s.KS.Mu.RUnlock()
 	s.KS.Mu.RLock()
 

broker/list_apply_api_test.go

@@ -110,7 +110,14 @@ func TestListCases(t *testing.T) {
 	require.NoError(t, err)
 	verify(resp, specB)
 
-	// Case: Meta-label "prefix" selects journals by name prefix.
+	// Case: Meta-label "name:prefix" selects journals by name prefix.
+	resp, err = broker.client().List(ctx, &pb.ListRequest{
+		Selector: pb.LabelSelector{Include: pb.MustLabelSet("name:prefix", "journal/1")},
+	})
+	require.NoError(t, err)
+	verify(resp, specA, specC)
+
+	// Case: legacy meta-label "prefix" also selects journals by name prefix.
 	resp, err = broker.client().List(ctx, &pb.ListRequest{
 		Selector: pb.LabelSelector{Include: pb.MustLabelSet("prefix", "journal/1/")},
 	})

broker/protocol/journal_spec_extensions.go

@@ -359,24 +359,18 @@ func SubtractJournalSpecs(a, b JournalSpec) JournalSpec {
 }
 
 // ExtractJournalSpecMetaLabels adds to the LabelSet a singular label "name",
-// with value of the JournalSpec Name, and multi-label "prefix", having a value
-// for each path component prefix of Name.
+// with value of the JournalSpec Name.
 func ExtractJournalSpecMetaLabels(spec *JournalSpec, out LabelSet) LabelSet {
 	var name = spec.Name.String()
 	out.Labels = append(out.Labels[:0], Label{Name: "name", Value: name})
-
-	for i, j := 0, strings.IndexByte(name, '/'); j != -1; j = strings.IndexByte(name[i:], '/') {
-		i += j + 1
-		out.Labels = append(out.Labels, Label{Name: "prefix", Value: name[:i]})
-	}
 	return out
 }
 
 // validateJournalLabelConstraints asserts expected invariants of MessageType,
 // MessageSubType, and ContentType labels:
-//  * ContentType must parse as a RFC 1521 MIME / media-type.
-//  * If MessageType is present, so is ContentType.
-//  * If MessageSubType is present, so is MessageType.
+//   - ContentType must parse as a RFC 1521 MIME / media-type.
+//   - If MessageType is present, so is ContentType.
+//   - If MessageSubType is present, so is MessageType.
 func validateJournalLabelConstraints(ls LabelSet) error {
 	if err := ValidateSingleValueLabels(ls); err != nil {
 		return err

broker/protocol/journal_spec_extensions_test.go

@@ -179,12 +179,7 @@ func (s *JournalSuite) TestSpecValidationCases(c *gc.C) {
 
 func (s *JournalSuite) TestMetaLabelExtraction(c *gc.C) {
 	c.Check(ExtractJournalSpecMetaLabels(&JournalSpec{Name: "path/to/my/journal"}, MustLabelSet("label", "buffer")),
-		gc.DeepEquals, MustLabelSet(
-			"name", "path/to/my/journal",
-			"prefix", "path/",
-			"prefix", "path/to/",
-			"prefix", "path/to/my/",
-		))
+		gc.DeepEquals, MustLabelSet("name", "path/to/my/journal"))
 }
 
 func (s *JournalSuite) TestFlagYAMLRoundTrip(c *gc.C) {

broker/protocol/label_extensions.go

@@ -4,16 +4,22 @@ import (
 	"bytes"
 	"regexp"
 	"sort"
+	"strings"
 
 	"go.gazette.dev/core/labels"
 )
 
 // Validate returns an error if the Label is not well-formed.
 func (m Label) Validate() error {
+	return m.validate(false)
+}
+func (m Label) validate(allowPrefix bool) error {
 	if err := ValidateToken(m.Name, TokenSymbols, minLabelLen, maxLabelLen); err != nil {
 		return ExtendContext(err, "Name")
 	} else if err = ValidateToken(m.Value, pathSymbols, 0, maxLabelValueLen); err != nil {
 		return ExtendContext(err, "Value")
+	} else if m.Prefix && !allowPrefix {
+		return NewValidationError("Prefix may not be set outside of a LabelSelector")
 	}
 	return nil
 }
@@ -27,16 +33,19 @@ func MustLabelSet(nv ...string) (set LabelSet) {
 	for i := 0; i != len(nv); i += 2 {
 		set.AddValue(nv[i], nv[i+1])
 	}
-	if err := set.Validate(); err != nil {
+	if err := set.validate(true); err != nil {
 		panic(err.Error())
 	}
 	return
 }
 
 // Validate returns an error if the LabelSet is not well-formed.
 func (m LabelSet) Validate() error {
+	return m.validate(false)
+}
+func (m LabelSet) validate(allowPrefix bool) error {
 	for i := range m.Labels {
-		if err := m.Labels[i].Validate(); err != nil {
+		if err := m.Labels[i].validate(allowPrefix); err != nil {
 			return ExtendContext(err, "Labels[%d]", i)
 		} else if i == 0 {
 			continue
@@ -108,15 +117,20 @@ func (m *LabelSet) SetValue(name, value string) {
 }
 
 // AddValue adds Label |name| with |value|, retaining any existing Labels |name|.
+// If |name| has the special suffix ":prefix", the Label is marked as a prefix
+// match. It's only valid to use ":prefix" within the context of a LabelSelector.
 func (m *LabelSet) AddValue(name, value string) {
+	var prefix = strings.HasSuffix(name, ":prefix")
+	name = strings.TrimSuffix(name, ":prefix")
+
 	var ind = sort.Search(len(m.Labels), func(i int) bool {
 		if m.Labels[i].Name != name {
 			return m.Labels[i].Name > name
 		} else {
 			return m.Labels[i].Value >= value
 		}
 	})
-	var label = Label{Name: name, Value: value}
+	var label = Label{Name: name, Value: value, Prefix: prefix}
 
 	if ind != len(m.Labels) && m.Labels[ind] == label {
 		// No-op.
@@ -215,9 +229,9 @@ func SubtractLabelSet(lhs, rhs, out LabelSet) LabelSet {
 
 // Validate returns an error if the LabelSelector is not well-formed.
 func (m LabelSelector) Validate() error {
-	if err := m.Include.Validate(); err != nil {
+	if err := m.Include.validate(true); err != nil {
 		return ExtendContext(err, "Include")
-	} else if err := m.Exclude.Validate(); err != nil {
+	} else if err := m.Exclude.validate(true); err != nil {
 		return ExtendContext(err, "Exclude")
 	}
 	return nil
@@ -237,7 +251,20 @@ func (m LabelSelector) Matches(s LabelSet) bool {
 func (s LabelSelector) String() string {
 	var w = bytes.NewBuffer(nil)
 
-	var f = func(l []Label, exc bool) {
+	var f = func(input []Label, exc bool) {
+		var l []Label
+
+		// Attach literal ":prefix" suffixes to names so that prefixes are
+		// distinguished during the subsequent self-join over `l`.
+		for _, ll := range input {
+			if ll.Prefix {
+				ll.Name = ll.Name + ":prefix"
+			}
+			l = append(l, ll)
+		}
+		// We may have changed Name ordering. Re-index.
+		sort.Slice(l, func(i, j int) bool { return l[i].Name < l[j].Name })
+
 		var it = labelJoin{setL: l, setR: l, lenL: len(l), lenR: len(l)}
 		for cur, ok := it.next(); ok; cur, ok = it.next() {
 			if cur.lBeg+1 == cur.lEnd && l[cur.lBeg].Value == "" {
@@ -298,7 +325,9 @@ func matchSelector(sel, set []Label, reqAll bool) bool {
 		var matched bool
 
 		for a, b := sel[cur.lBeg:cur.lEnd], set[cur.rBeg:cur.rEnd]; !matched && len(a) != 0 && len(b) != 0; {
-			if a[0].Value == "" || a[0].Value == b[0].Value {
+			if a[0].Value == "" ||
+				(!a[0].Prefix && a[0].Value == b[0].Value) ||
+				(a[0].Prefix && strings.HasPrefix(b[0].Value, a[0].Value)) {
 				matched = true // Selector value "" implicitly matches any value of the label.
 			} else if a[0].Value < b[0].Value {
 				a = a[1:]
@@ -378,16 +407,22 @@ func labelValuesEqual(it labelJoin, cur labelJoinCursor) bool {
 // expression types are equality, in-equality, set membership, set exclusion,
 // existence, and non-existence. Eg:
 //
-//   * "foo = bar" requires that label "foo" be present with value "bar"
-//   * "foo != bar" requires that label "foo" not be present with value "bar"
-//   * "foo" requires that label "foo" be present (with any value).
-//   * "!foo" requires that label "foo" not be present.
-//   * "foo in (bar,baz)" requires that "foo" be present with either "bar" or "baz".
-//   * "foo notin (bar,baz)" requires that "foo", if present, not have value "bar" or "baz".
+//   - "foo = bar" requires that label "foo" be present with value "bar"
+//   - "foo != bar" requires that label "foo" not be present with value "bar"
+//   - "foo" requires that label "foo" be present (with any value).
+//   - "!foo" requires that label "foo" not be present.
+//   - "foo in (bar,baz)" requires that "foo" be present with either "bar" or "baz".
+//   - "foo notin (bar,baz)" requires that "foo", if present, not have value "bar" or "baz".
+//
+// A label name within a selector can have a "my-label:prefix" suffix which
+// tests whether the selector value is a prefix of the named label value,
+// instead of using exact equality.
+// For example, "foo:prefix in (one/two/, three/)" would match "one/two/three"
+// and "three/four/five" but not "one/one/" or "four/five/six".
 //
 // Additional examples of composite expressions:
-//   * "topic in (topic/one, topic/two), prefix=/my/journal/prefix"
-//   * "env in (production, qa), tier not in (frontend,backend), partition"
+//   - "topic in (topic/one, topic/two), name:prefix=/my/journal/prefix"
+//   - "env in (production, qa), tier not in (frontend,backend), partition"
 //
 // ParseLabelSelector is invariant to _reasonable_ spacing: eg, "not in" and
 // "notin" may be used interchangeably, as may "==" and "=", with or without
@@ -398,60 +433,46 @@ func ParseLabelSelector(s string) (LabelSelector, error) {
 	for len(s) != 0 {
 		var m []string
 		if m = reSelectorEqual.FindStringSubmatch(s); m != nil {
-			out.Include.Labels = append(out.Include.Labels, Label{Name: m[1], Value: m[2]})
+			out.Include.AddValue(m[1], m[2])
 		} else if m = reSelectorNotEqual.FindStringSubmatch(s); m != nil {
-			out.Exclude.Labels = append(out.Exclude.Labels, Label{Name: m[1], Value: m[2]})
+			out.Exclude.AddValue(m[1], m[2])
 		} else if m = reSelectorSetIn.FindStringSubmatch(s); m != nil {
-			if parts, err := parseSetParts(m[1], m[2]); err != nil {
+			if err := parseSetParts(&out.Include, m[1], m[2]); err != nil {
 				return LabelSelector{}, ExtendContext(err, "parsing %q", s)
-			} else {
-				out.Include.Labels = append(out.Include.Labels, parts...)
 			}
 		} else if m = reSelectorSetNotIn.FindStringSubmatch(s); m != nil {
-			if parts, err := parseSetParts(m[1], m[2]); err != nil {
+			if err := parseSetParts(&out.Exclude, m[1], m[2]); err != nil {
 				return LabelSelector{}, ExtendContext(err, "parsing %q", s)
-			} else {
-				out.Exclude.Labels = append(out.Exclude.Labels, parts...)
 			}
 		} else if m = reSelectorSetExists.FindStringSubmatch(s); m != nil {
-			out.Include.Labels = append(out.Include.Labels, Label{Name: m[1]})
+			out.Include.AddValue(m[1], "")
 		} else if m = reSelectorSetNotExists.FindStringSubmatch(s); m != nil {
-			out.Exclude.Labels = append(out.Exclude.Labels, Label{Name: m[1]})
+			out.Exclude.AddValue(m[1], "")
 		} else {
 			return LabelSelector{}, NewValidationError("could not match %q to a label selector expression", s)
 		}
 		s = s[len(m[0]):]
 	}
 
-	for _, l := range [][]Label{out.Include.Labels, out.Exclude.Labels} {
-		sort.Slice(l, func(i, j int) bool {
-			if l[i].Name != l[j].Name {
-				return l[i].Name < l[j].Name
-			}
-			return l[i].Value < l[j].Value
-		})
-	}
 	return out, out.Validate()
 }
 
-func parseSetParts(name, s string) ([]Label, error) {
-	var out []Label
-
+func parseSetParts(set *LabelSet, name, s string) error {
 	for len(s) != 0 {
 		var m []string
 
 		if m = reSelectorSetValue.FindStringSubmatch(s); m != nil {
-			out = append(out, Label{Name: name, Value: m[1]})
+			set.AddValue(name, m[1])
 		} else {
-			return nil, NewValidationError("could not match %q to a label selector set expression", s)
+			return NewValidationError("could not match %q to a label selector set expression", s)
 		}
 		s = s[len(m[0]):]
 	}
-	return out, nil
+	return nil
 }
 
 var (
-	reToken         = ` ?([\pL\pN\` + regexp.QuoteMeta(TokenSymbols) + `]{2,})`
+	reToken         = ` ?([\pL\pN\` + regexp.QuoteMeta(TokenSymbols) + `]{2,}(?:\:prefix)?)`
 	rePath          = ` ?([\pL\pN\` + regexp.QuoteMeta(pathSymbols) + `]{0,})`
 	reCommaOrEnd    = ` ?(?:,|$)`
 	reParenthetical = ` ?\(([^)]+)\)`