feat: Regex filters for P2P preheat (phase 3)

The preheat policy filters bypassed the selector registry: buildFilter reached
straight for the doublestar selector, so a repository or tag pattern could only
be a glob, while the retention, immutability and replication filters can already
be regular expressions.

A preheat filter now carries an optional "kind", the pattern engine of its
value, with the same literals the other filters use. The field is additive and
omitted when it is the doublestar default, so a policy stored before this keeps
decoding and behaving exactly as it did.

Only the repository and tag filters take a kind. The label filter matches label
names exactly, and signature and vulnerability carry a bool and an int, so a
kind there is rejected rather than silently ignored.

The policy schema validates the filters at write time, which is the create and
update path of the controller, and returns a 400 for an unknown kind, a kind on
a filter type that has no pattern, or an expression that does not compile. The
filter builder keeps its own check as a backstop for a policy that was stored by
an older version or written past the API.

The engine is picked per pattern input in the preheat policy dialog, next to the
repositories and the tags, and the payload carries the kind only when it is not
the default. A regex is stored verbatim: the portal wraps a comma separated
doublestar list in braces, which a regex uses for quantifiers.

Signed-off-by: Vadim Bauer <vb@container-registry.com>

Author: Vad1mo

src/controller/p2p/preheat/controller_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/goharbor/harbor/src/lib/config"
+	harborerrors "github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy"
@@ -253,6 +254,73 @@ func (s *preheatSuite) TestCreatePolicy() {
 	s.False(policy.UpdatedTime.IsZero())
 }
 
+func (s *preheatSuite) TestPolicyFilterKindValidation() {
+	cases := []struct {
+		name       string
+		filtersStr string
+		wantErr    bool
+	}{
+		{
+			name:       "no kind",
+			filtersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`,
+		},
+		{
+			name:       "regex kind",
+			filtersStr: `[{"type":"repository","value":"harbor.*","kind":"regex"},{"type":"tag","value":"2.*","kind":"regex"}]`,
+		},
+		{
+			name:       "unknown kind",
+			filtersStr: `[{"type":"repository","value":"harbor*","kind":"glob"}]`,
+			wantErr:    true,
+		},
+		{
+			name:       "invalid regex",
+			filtersStr: `[{"type":"repository","value":"foo)|(?:bar","kind":"regex"}]`,
+			wantErr:    true,
+		},
+		{
+			name:       "kind on a label filter",
+			filtersStr: `[{"type":"label","value":"prod","kind":"regex"}]`,
+			wantErr:    true,
+		},
+	}
+
+	stored := &policy.Schema{ID: 99, Name: "test-filter-kind", Trigger: &policy.Trigger{Type: policy.TriggerTypeManual}}
+	s.fakePolicyMgr.On("Get", s.ctx, int64(99)).Return(stored, nil)
+
+	for _, c := range cases {
+		s.Run(c.name, func() {
+			p := &policy.Schema{
+				ID:         99,
+				Name:       "test-filter-kind",
+				FiltersStr: c.filtersStr,
+				TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{}}`, policy.TriggerTypeManual),
+			}
+			s.fakePolicyMgr.On("Create", s.ctx, p).Return(int64(99), nil).Maybe()
+			s.fakePolicyMgr.On("Update", s.ctx, p, mock.Anything).Return(nil).Maybe()
+
+			_, createErr := s.controller.CreatePolicy(s.ctx, p)
+			updateErr := s.controller.UpdatePolicy(s.ctx, p, "")
+			if !c.wantErr {
+				s.NoError(createErr)
+				s.NoError(updateErr)
+				// the policy of this case is a distinct pointer, so the recorded calls
+				// of the other tests in the suite never satisfy these
+				s.fakePolicyMgr.AssertCalled(s.T(), "Create", s.ctx, p)
+				s.fakePolicyMgr.AssertCalled(s.T(), "Update", s.ctx, p, mock.Anything)
+				return
+			}
+			s.Error(createErr)
+			s.True(harborerrors.IsErr(createErr, harborerrors.BadRequestCode), "create error is a bad request: %v", createErr)
+			s.Error(updateErr)
+			s.True(harborerrors.IsErr(updateErr, harborerrors.BadRequestCode), "update error is a bad request: %v", updateErr)
+			// validation runs before persistence, so a rejected policy is never stored
+			s.fakePolicyMgr.AssertNotCalled(s.T(), "Create", s.ctx, p)
+			s.fakePolicyMgr.AssertNotCalled(s.T(), "Update", s.ctx, p, mock.Anything)
+		})
+	}
+}
+
 func (s *preheatSuite) TestGetPolicy() {
 	s.fakePolicyMgr.On("Get", s.ctx, int64(1)).Return(&policy.Schema{Name: "test"}, nil)
 	p, err := s.controller.GetPolicy(s.ctx, 1)

src/pkg/p2p/preheat/models/policy/policy.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/pattern"
 	"github.com/goharbor/harbor/src/lib/q"
 )
 
@@ -48,6 +49,12 @@ const (
 	// FilterTypeLabel represents the label filter type
 	FilterTypeLabel FilterType = "label"
 
+	// FilterKindDoublestar interprets the filter value as a doublestar pattern, the default
+	FilterKindDoublestar = pattern.KindDoublestar
+	// FilterKindRegex interprets the filter value as a regular expression matching the whole
+	// value, the same engine the retention, immutability and replication filters use
+	FilterKindRegex = pattern.KindRegex
+
 	// TriggerTypeManual represents the manual trigger type
 	TriggerTypeManual TriggerType = "manual"
 	// TriggerTypeScheduled represents the scheduled trigger type
@@ -103,6 +110,50 @@ type FilterType = string
 type Filter struct {
 	Type  FilterType `json:"type"`
 	Value any        `json:"value"`
+	// Kind selects the pattern engine used for Value, empty means FilterKindDoublestar.
+	// Only the repository and tag filters carry a pattern, the others are exact or numeric.
+	Kind string `json:"kind,omitempty"`
+}
+
+// supportsKind reports whether the filter type evaluates its value as a pattern
+func (f *Filter) supportsKind() bool {
+	return f.Type == FilterTypeRepository || f.Type == FilterTypeTag
+}
+
+// Validate checks the pattern engine selection of the filter. The value type checks stay
+// in the filter builder, which is also reached by the policies stored before this existed.
+func (f *Filter) Validate() error {
+	if f.Kind == "" {
+		return nil
+	}
+
+	if f.Kind != FilterKindDoublestar && f.Kind != FilterKindRegex {
+		return errors.New(nil).WithCode(errors.BadRequestCode).
+			WithMessagef("invalid filter kind: %s", f.Kind)
+	}
+
+	if !f.supportsKind() {
+		return errors.New(nil).WithCode(errors.BadRequestCode).
+			WithMessagef("only the %s and %s filters support kind, got: %s",
+				FilterTypeRepository, FilterTypeTag, f.Type)
+	}
+
+	if f.Kind != FilterKindRegex {
+		return nil
+	}
+
+	value, ok := f.Value.(string)
+	if !ok {
+		return errors.New(nil).WithCode(errors.BadRequestCode).
+			WithMessagef("the value of the %s filter isn't a string", f.Type)
+	}
+
+	if err := pattern.ValidateRegex(value); err != nil {
+		return errors.New(nil).WithCode(errors.BadRequestCode).
+			WithMessagef("invalid regex filter value %q: %v", value, err)
+	}
+
+	return nil
 }
 
 // TriggerType represents the type of trigger.
@@ -120,6 +171,15 @@ type Trigger struct {
 
 // ValidatePreheatPolicy validate preheat policy
 func (s *Schema) ValidatePreheatPolicy() error {
+	for _, filter := range s.Filters {
+		if filter == nil {
+			continue
+		}
+		if err := filter.Validate(); err != nil {
+			return err
+		}
+	}
+
 	// currently only validate cron string of preheat policy
 	if s.Trigger != nil && s.Trigger.Type == TriggerTypeScheduled && len(s.Trigger.Settings.Cron) > 0 {
 		if err := utils.ValidateCronString(s.Trigger.Settings.Cron); err != nil {

src/pkg/p2p/preheat/models/policy/policy_test.go

@@ -15,9 +15,12 @@
 package policy
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+
+	"github.com/goharbor/harbor/src/lib/errors"
 )
 
 // PolicyTestSuite is a test suite for policy schema.
@@ -66,6 +69,124 @@ func (p *PolicyTestSuite) TestValidatePreheatPolicy() {
 	p.NoError(p.schema.ValidatePreheatPolicy())
 }
 
+// TestValidateFilterKind tests the pattern engine validation of the policy filters
+func (p *PolicyTestSuite) TestValidateFilterKind() {
+	cases := []struct {
+		name    string
+		filter  *Filter
+		wantErr bool
+	}{
+		{
+			name:   "absent kind",
+			filter: &Filter{Type: FilterTypeRepository, Value: "**"},
+		},
+		{
+			name:   "explicit doublestar kind",
+			filter: &Filter{Type: FilterTypeTag, Value: "prod*", Kind: FilterKindDoublestar},
+		},
+		{
+			name:   "regex repository",
+			filter: &Filter{Type: FilterTypeRepository, Value: "library/.*", Kind: FilterKindRegex},
+		},
+		{
+			name:   "regex tag",
+			filter: &Filter{Type: FilterTypeTag, Value: `v\d+\.\d+`, Kind: FilterKindRegex},
+		},
+		{
+			name:   "empty regex pattern",
+			filter: &Filter{Type: FilterTypeTag, Value: "", Kind: FilterKindRegex},
+		},
+		{
+			name:    "unknown kind",
+			filter:  &Filter{Type: FilterTypeTag, Value: "**", Kind: "glob"},
+			wantErr: true,
+		},
+		{
+			name:    "kind on a label filter",
+			filter:  &Filter{Type: FilterTypeLabel, Value: "prod", Kind: FilterKindRegex},
+			wantErr: true,
+		},
+		{
+			name:    "kind on a signature filter",
+			filter:  &Filter{Type: FilterTypeSignature, Value: true, Kind: FilterKindDoublestar},
+			wantErr: true,
+		},
+		{
+			name:    "kind on a vulnerability filter",
+			filter:  &Filter{Type: FilterTypeVulnerability, Value: 3, Kind: FilterKindRegex},
+			wantErr: true,
+		},
+		{
+			name:    "invalid regex",
+			filter:  &Filter{Type: FilterTypeTag, Value: "[", Kind: FilterKindRegex},
+			wantErr: true,
+		},
+		{
+			name:    "regex escaping the anchoring",
+			filter:  &Filter{Type: FilterTypeTag, Value: "foo)|(?:bar", Kind: FilterKindRegex},
+			wantErr: true,
+		},
+		{
+			name:    "regex longer than the pattern limit",
+			filter:  &Filter{Type: FilterTypeTag, Value: strings.Repeat("a", 513), Kind: FilterKindRegex},
+			wantErr: true,
+		},
+		{
+			name:   "regex at the pattern limit",
+			filter: &Filter{Type: FilterTypeTag, Value: strings.Repeat("a", 512), Kind: FilterKindRegex},
+		},
+		{
+			name:    "regex value that isn't a string",
+			filter:  &Filter{Type: FilterTypeTag, Value: 100, Kind: FilterKindRegex},
+			wantErr: true,
+		},
+	}
+
+	for _, c := range cases {
+		p.Run(c.name, func() {
+			s := &Schema{Filters: []*Filter{c.filter}, Trigger: &Trigger{Type: TriggerTypeManual}}
+			err := s.ValidatePreheatPolicy()
+			if !c.wantErr {
+				p.NoError(err)
+				return
+			}
+			p.Error(err)
+			p.True(errors.IsErr(err, errors.BadRequestCode), "error is a bad request: %v", err)
+		})
+	}
+}
+
+// TestFilterKindRoundTrip tests that the kind survives an encode/decode cycle and that a
+// policy stored without a kind keeps decoding
+func (p *PolicyTestSuite) TestFilterKindRoundTrip() {
+	s := &Schema{
+		Filters: []*Filter{
+			{Type: FilterTypeRepository, Value: "library/.*", Kind: FilterKindRegex},
+			{Type: FilterTypeTag, Value: "**"},
+		},
+		Trigger: &Trigger{Type: TriggerTypeManual},
+	}
+	p.NoError(s.Encode())
+	p.Equal(`[{"type":"repository","value":"library/.*","kind":"regex"},{"type":"tag","value":"**"}]`, s.FiltersStr)
+
+	// a policy stored before the kind existed decodes into the doublestar default
+	stored := &Schema{
+		FiltersStr: `[{"type":"repository","value":"**"},{"type":"tag","value":"**"},{"type":"label","value":"test"}]`,
+		TriggerStr: `{"type":"manual","trigger_setting":{"cron":""}}`,
+	}
+	p.NoError(stored.Decode())
+	p.Len(stored.Filters, 3)
+	for _, f := range stored.Filters {
+		p.Empty(f.Kind)
+	}
+	p.NoError(stored.ValidatePreheatPolicy())
+
+	decoded := &Schema{FiltersStr: s.FiltersStr, TriggerStr: `{"type":"manual","trigger_setting":{"cron":""}}`}
+	p.NoError(decoded.Decode())
+	p.Equal(FilterKindRegex, decoded.Filters[0].Kind)
+	p.Empty(decoded.Filters[1].Kind)
+}
+
 // TestDecode tests decode.
 func (p *PolicyTestSuite) TestDecode() {
 	s := &Schema{

src/pkg/p2p/preheat/policy/filter.go

@@ -23,6 +23,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/selector"
 	"github.com/goharbor/harbor/src/lib/selector/selectors/doublestar"
 	"github.com/goharbor/harbor/src/lib/selector/selectors/label"
+	regexpselector "github.com/goharbor/harbor/src/lib/selector/selectors/regexp"
 	"github.com/goharbor/harbor/src/lib/selector/selectors/severity"
 	"github.com/goharbor/harbor/src/lib/selector/selectors/signature"
 	"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy"
@@ -152,6 +153,12 @@ func buildFilter(f *policy.Filter) (selector.Selector, error) {
 		return nil, errors.Errorf("pattern value is missing for filter: %s", f.Type)
 	}
 
+	// Backstop for the write time validation: a policy stored by an older version, or
+	// written past the API, still has to fail here instead of building a broken selector.
+	if err := f.Validate(); err != nil {
+		return nil, err
+	}
+
 	// Current value type
 	cvt := reflect.TypeOf(f.Value).Name()
 
@@ -176,8 +183,14 @@ func buildFilter(f *policy.Filter) (selector.Selector, error) {
 	// Build selectors
 	switch f.Type {
 	case policy.FilterTypeRepository:
+		if f.Kind == policy.FilterKindRegex {
+			return regexpselector.New(regexpselector.RepoMatches, f.Value, ""), nil
+		}
 		return doublestar.New(doublestar.RepoMatches, f.Value, ""), nil
 	case policy.FilterTypeTag:
+		if f.Kind == policy.FilterKindRegex {
+			return regexpselector.New(regexpselector.Matches, f.Value, ""), nil
+		}
 		return doublestar.New(doublestar.Matches, f.Value, ""), nil
 	case policy.FilterTypeLabel:
 		return label.New(label.With, f.Value, ""), nil