Add Default Lookup Services (#1367)

cmd/common.go

@@ -119,6 +119,17 @@ func evaluateTargetRuntimeGroupOrControlPlaneName(targetContent *file.Content) e
 	return nil
 }
 
+func RemoveConsumerPlugins(targetContentPlugins []file.FPlugin) []file.FPlugin {
+	filteredPlugins := []file.FPlugin{}
+
+	for _, plugin := range targetContentPlugins {
+		if plugin.Consumer == nil && plugin.ConsumerGroup == nil {
+			filteredPlugins = append(filteredPlugins, plugin)
+		}
+	}
+	return filteredPlugins
+}
+
 func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	delay int, workspace string, enableJSONOutput bool,
 ) error {
@@ -139,6 +150,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	if dumpConfig.SkipConsumers {
 		targetContent.Consumers = []file.FConsumer{}
 		targetContent.ConsumerGroups = []file.FConsumerGroupObject{}
+		targetContent.Plugins = RemoveConsumerPlugins(targetContent.Plugins)
 	}
 	if dumpConfig.SkipCACerts {
 		targetContent.CACertificates = []file.FCACertificate{}
@@ -236,6 +248,25 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return err
 	}
 
+	dumpConfig.LookUpSelectorTagsConsumerGroups, err = determineLookUpSelectorTagsConsumerGroups(*targetContent)
+	if err != nil {
+		return fmt.Errorf("error determining lookup selector tags for consumer groups: %w", err)
+	}
+
+	if dumpConfig.LookUpSelectorTagsConsumerGroups != nil {
+		consumerGroupsGlobal, err := dump.GetAllConsumerGroups(ctx, kongClient, dumpConfig.LookUpSelectorTagsConsumerGroups)
+		if err != nil {
+			return fmt.Errorf("error retrieving global consumer groups via lookup selector tags: %w", err)
+		}
+		for _, c := range consumerGroupsGlobal {
+			targetContent.ConsumerGroups = append(targetContent.ConsumerGroups,
+				file.FConsumerGroupObject{ConsumerGroup: *c.ConsumerGroup})
+			if err != nil {
+				return fmt.Errorf("error adding global consumer group %v: %w", *c.ConsumerGroup.Name, err)
+			}
+		}
+	}
+
 	dumpConfig.LookUpSelectorTagsConsumers, err = determineLookUpSelectorTagsConsumers(*targetContent)
 	if err != nil {
 		return fmt.Errorf("error determining lookup selector tags for consumers: %w", err)
@@ -272,6 +303,24 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		}
 	}
 
+	dumpConfig.LookUpSelectorTagsServices, err = determineLookUpSelectorTagsServices(*targetContent)
+	if err != nil {
+		return fmt.Errorf("error determining lookup selector tags for services: %w", err)
+	}
+
+	if dumpConfig.LookUpSelectorTagsServices != nil {
+		servicesGlobal, err := dump.GetAllServices(ctx, kongClient, dumpConfig.LookUpSelectorTagsServices)
+		if err != nil {
+			return fmt.Errorf("error retrieving global services via lookup selector tags: %w", err)
+		}
+		for _, r := range servicesGlobal {
+			targetContent.Services = append(targetContent.Services, file.FService{Service: *r})
+			if err != nil {
+				return fmt.Errorf("error adding global service %v: %w", r.FriendlyName(), err)
+			}
+		}
+	}
+
 	if reconcilerUtils.Kong340Version.LTE(parsedKongVersion) {
 		dumpConfig.IsConsumerGroupScopedPluginSupported = true
 	}
@@ -355,6 +404,21 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	return nil
 }
 
+func determineLookUpSelectorTagsConsumerGroups(targetContent file.Content) ([]string, error) {
+	if targetContent.Info != nil &&
+		targetContent.Info.LookUpSelectorTags != nil &&
+		targetContent.Info.LookUpSelectorTags.ConsumerGroups != nil {
+		if len(targetContent.Info.LookUpSelectorTags.ConsumerGroups) == 0 {
+			return nil, fmt.Errorf("global consumer groups specified but no global tags")
+		}
+		reconcilerUtils.RemoveDuplicates(&targetContent.Info.LookUpSelectorTags.ConsumerGroups)
+		sort.Strings(targetContent.Info.LookUpSelectorTags.ConsumerGroups)
+		return targetContent.Info.LookUpSelectorTags.ConsumerGroups, nil
+
+	}
+	return nil, nil
+}
+
 func determineLookUpSelectorTagsConsumers(targetContent file.Content) ([]string, error) {
 	if targetContent.Info != nil &&
 		targetContent.Info.LookUpSelectorTags != nil &&
@@ -385,6 +449,21 @@ func determineLookUpSelectorTagsRoutes(targetContent file.Content) ([]string, er
 	return nil, nil
 }
 
+func determineLookUpSelectorTagsServices(targetContent file.Content) ([]string, error) {
+	if targetContent.Info != nil &&
+		targetContent.Info.LookUpSelectorTags != nil &&
+		targetContent.Info.LookUpSelectorTags.Services != nil {
+		if len(targetContent.Info.LookUpSelectorTags.Services) == 0 {
+			return nil, fmt.Errorf("global services specified but no global tags")
+		}
+		reconcilerUtils.RemoveDuplicates(&targetContent.Info.LookUpSelectorTags.Services)
+		sort.Strings(targetContent.Info.LookUpSelectorTags.Services)
+		return targetContent.Info.LookUpSelectorTags.Services, nil
+
+	}
+	return nil, nil
+}
+
 func determineSelectorTag(targetContent file.Content, config dump.Config) ([]string, error) {
 	if targetContent.Info != nil {
 		if len(targetContent.Info.SelectorTags) > 0 {

cmd/gateway_validate.go

@@ -50,6 +50,26 @@ func executeValidate(cmd *cobra.Command, _ []string) error {
 			return err
 		}
 
+		// if this is an online validation, we need to look up upstream consumers if required.
+		lookUpSelectorTagsConsumerGroups, err := determineLookUpSelectorTagsConsumerGroups(*targetContent)
+		if err != nil {
+			return fmt.Errorf("error determining lookup selector tags for consumer grous: %w", err)
+		}
+
+		if lookUpSelectorTagsConsumerGroups != nil {
+			consumerGroupsGlobal, err := dump.GetAllConsumerGroups(ctx, kongClient, lookUpSelectorTagsConsumerGroups)
+			if err != nil {
+				return fmt.Errorf("error retrieving global consumer groups via lookup selector tags: %w", err)
+			}
+			for _, c := range consumerGroupsGlobal {
+				targetContent.ConsumerGroups = append(targetContent.ConsumerGroups,
+					file.FConsumerGroupObject{ConsumerGroup: *c.ConsumerGroup})
+				if err != nil {
+					return fmt.Errorf("error adding global consumer group %v: %w", *c.ConsumerGroup.Name, err)
+				}
+			}
+		}
+
 		// if this is an online validation, we need to look up upstream consumers if required.
 		lookUpSelectorTagsConsumers, err := determineLookUpSelectorTagsConsumers(*targetContent)
 		if err != nil {
@@ -87,6 +107,25 @@ func executeValidate(cmd *cobra.Command, _ []string) error {
 				}
 			}
 		}
+
+		// if this is an online validation, we need to look up upstream services if required.
+		lookUpSelectorTagsServices, err := determineLookUpSelectorTagsServices(*targetContent)
+		if err != nil {
+			return fmt.Errorf("error determining lookup selector tags for services: %w", err)
+		}
+
+		if lookUpSelectorTagsServices != nil {
+			servicesGlobal, err := dump.GetAllServices(ctx, kongClient, lookUpSelectorTagsServices)
+			if err != nil {
+				return fmt.Errorf("error retrieving global services via lookup selector tags: %w", err)
+			}
+			for _, r := range servicesGlobal {
+				targetContent.Services = append(targetContent.Services, file.FService{Service: *r})
+				if err != nil {
+					return fmt.Errorf("error adding global service %v: %w", r.FriendlyName(), err)
+				}
+			}
+		}
 	}
 
 	rawState, err := file.Get(ctx, targetContent, file.RenderConfig{

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/fatih/color v1.17.0
 	github.com/google/go-cmp v0.6.0
 	github.com/kong/go-apiops v0.1.37
-	github.com/kong/go-database-reconciler v1.14.7
+	github.com/kong/go-database-reconciler v1.15.0
 	github.com/kong/go-kong v0.59.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.8.1

go.sum

@@ -181,8 +181,8 @@ github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/q
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kong/go-apiops v0.1.37 h1:AwIslsbnx9JAFdPy8UI6NF/I+X2ypi7TJmdDNwKe/bY=
 github.com/kong/go-apiops v0.1.37/go.mod h1:B0WFsqonn+xnHgHg0x063fADFC21mhNOsOXsbKdZTpM=
-github.com/kong/go-database-reconciler v1.14.7 h1:U8VnKYFz8oBnAgDxipBHNR9KCOxexffQlmCyfWhafV4=
-github.com/kong/go-database-reconciler v1.14.7/go.mod h1:T5BkBw13PZWub3y2jKAoM7fYD+UmXp2iNqj1YqD0L90=
+github.com/kong/go-database-reconciler v1.15.0 h1:5F5Zzp2H14aiDmqWUCaU4+LGR/lGnvhwBTmtr3N6RZQ=
+github.com/kong/go-database-reconciler v1.15.0/go.mod h1:T5BkBw13PZWub3y2jKAoM7fYD+UmXp2iNqj1YqD0L90=
 github.com/kong/go-kong v0.59.1 h1:AJZtyCD+Zyqe/mF/m+x3/qN/GPVxAH7jq9zGJTHRfjc=
 github.com/kong/go-kong v0.59.1/go.mod h1:8Vt6HmtgLNgL/7bSwAlz3DIWqBtzG7qEt9+OnMiQOa0=
 github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY=

tests/integration/sync_test.go

@@ -3765,6 +3765,55 @@ func Test_Sync_SkipConsumers_34x(t *testing.T) {
 						Enabled:   kong.Bool(true),
 						Protocols: []*string{kong.String("grpc"), kong.String("grpcs"), kong.String("http"), kong.String("https")},
 					},
+					{
+						Name: kong.String("rate-limiting-advanced"),
+						Consumer: &kong.Consumer{
+							ID: kong.String("416b038a-fd00-45fd-a5a2-a74bf70017fa"),
+						},
+						Config: kong.Configuration{
+							"consumer_groups":         nil,
+							"dictionary_name":         string("kong_rate_limiting_counters"),
+							"disable_penalty":         bool(false),
+							"enforce_consumer_groups": bool(false),
+							"error_code":              float64(429),
+							"error_message":           string("API rate limit exceeded"),
+							"header_name":             nil,
+							"hide_client_headers":     bool(false),
+							"identifier":              string("consumer"),
+							"limit":                   []any{float64(10)},
+							"namespace":               string("foo"),
+							"path":                    nil,
+							"redis": map[string]any{
+								"cluster_addresses":   nil,
+								"connect_timeout":     nil,
+								"database":            float64(0),
+								"host":                nil,
+								"keepalive_backlog":   nil,
+								"keepalive_pool_size": float64(30),
+								"password":            nil,
+								"port":                nil,
+								"read_timeout":        nil,
+								"send_timeout":        nil,
+								"sentinel_addresses":  nil,
+								"sentinel_master":     nil,
+								"sentinel_password":   nil,
+								"sentinel_role":       nil,
+								"sentinel_username":   nil,
+								"server_name":         nil,
+								"ssl":                 false,
+								"ssl_verify":          false,
+								"timeout":             float64(2000),
+								"username":            nil,
+							},
+							"retry_after_jitter_max": float64(1),
+							"strategy":               string("local"),
+							"sync_rate":              float64(-1),
+							"window_size":            []any{float64(60)},
+							"window_type":            string("sliding"),
+						},
+						Enabled:   kong.Bool(true),
+						Protocols: []*string{kong.String("grpc"), kong.String("grpcs"), kong.String("http"), kong.String("https")},
+					},
 				},
 			},
 			skipConsumers: false,
@@ -4959,21 +5008,21 @@ func Test_Sync_LookupConsumerTags(t *testing.T) {
 	setup(t)
 
 	// test that reference to non-existing consumer fails.
-	pluginsNoLookupStateFile := "testdata/sync/029-lookup-tags/plugins_no_lookup.yaml"
+	pluginsNoLookupStateFile := "testdata/sync/029-lookup-tags-consumers/plugins_no_lookup.yaml"
 	err := sync(pluginsNoLookupStateFile)
 	require.Error(t, err)
 	require.EqualError(t, err, "building state: consumer foo for plugin rate-limiting-advanced: entity not found")
 
 	// test that reference to existing local consumer succeeds.
-	pluginsAndConsumersStateFile := "testdata/sync/029-lookup-tags/plugins_and_consumers.yaml"
+	pluginsAndConsumersStateFile := "testdata/sync/029-lookup-tags-consumers/plugins_and_consumers.yaml"
 	require.NoError(t, sync(pluginsAndConsumersStateFile))
 	reset(t)
 
 	// test that reference to existing global consumer succeeds via lookup tags.
-	globalConsumersStateFile := "testdata/sync/029-lookup-tags/global_consumers.yaml"
+	globalConsumersStateFile := "testdata/sync/029-lookup-tags-consumers/global_consumers.yaml"
 	require.NoError(t, sync(globalConsumersStateFile))
 	// sync plugins with lookup reference to global consumers.
-	pluginsLookupStateFile := "testdata/sync/029-lookup-tags/plugins_lookup.yaml"
+	pluginsLookupStateFile := "testdata/sync/029-lookup-tags-consumers/plugins_lookup.yaml"
 	require.NoError(t, sync(pluginsLookupStateFile))
 	reset(t)
 
@@ -5053,6 +5102,45 @@ func Test_Sync_ConsumerGroupConsumersWithCustomID(t *testing.T) {
 	testKongState(t, client, false, expectedState, nil)
 }
 
+// Test_Sync_LookupServicesTags tests that existing behavior when referencing
+// services from plugins is preserved:
+// - if a referenced service is not present in the state file, the sync fails
+// - if a referenced service is present in the state file, the sync succeeds
+//
+// This test also tests that the new behavior is implemented correctly:
+//   - if a referenced service is not present in the state file, but is present
+//     in Kong when using the new lookup selector tags, the sync succeeds
+//   - if a referenced service is not present in the state file and neither in
+//     Kong when using the new lookup selector tags, the sync fails
+func Test_Sync_LookupServicesTags(t *testing.T) {
+	runWhen(t, "enterprise", ">=3.0.0")
+	setup(t)
+
+	// test that reference to non-existing service fails.
+	pluginsNoLookupServiceStateFile := "testdata/sync/035-lookup-tags-services/plugins_no_lookup.yaml"
+	err := sync(pluginsNoLookupServiceStateFile)
+	require.Error(t, err)
+	require.EqualError(t, err, "building state: service foo for plugin rate-limiting-advanced: entity not found")
+
+	// test that reference to existing local service succeeds.
+	pluginsAndServicesStateFile := "testdata/sync/035-lookup-tags-services/plugins_and_services.yaml"
+	require.NoError(t, sync(pluginsAndServicesStateFile))
+	reset(t)
+
+	// test that reference to existing global service succeeds via lookup tags.
+	globalServicesStateFile := "testdata/sync/035-lookup-tags-services/global_services.yaml"
+	require.NoError(t, sync(globalServicesStateFile))
+
+	// sync plugins with lookup reference to global services.
+	pluginsLookupServiceStateFile := "testdata/sync/035-lookup-tags-services/plugins_lookup.yaml"
+	require.NoError(t, sync(pluginsLookupServiceStateFile))
+	reset(t)
+
+	// test that reference to non-existing global service fails via lookup tags.
+	require.Error(t, sync(pluginsLookupServiceStateFile))
+	require.EqualError(t, err, "building state: service foo for plugin rate-limiting-advanced: entity not found")
+}
+
 // Test_Sync_LookupRoutesTags tests that existing behavior when referencing
 // routes from plugins is preserved:
 // - if a referenced route is not present in the state file, the sync fails
@@ -5091,6 +5179,77 @@ func Test_Sync_LookupRoutesTags(t *testing.T) {
 	require.EqualError(t, err, "building state: route foo for plugin rate-limiting-advanced: entity not found")
 }
 
+// Test_Sync_LookupConsumerGroupsTags tests that existing behavior when referencing
+// consumer groups from plugins is preserved:
+// - if a referenced service/route and consumer group are not present in the state file, the sync fails
+// - if a referenced service/route and consumer group are present in the state file, the sync succeeds
+//
+// This test also tests that the new behavior is implemented correctly:
+//   - if a referenced service/route/consumer or consumer group is not present in the state file, but is present
+//     in Kong when using the new lookup selector tags, the sync succeeds
+//   - if a referenced service/route or consumer group is not present in the state file and neither in
+//     Kong when using the new lookup selector tags, the sync fails
+func Test_Sync_LookupConsumerGroupsTags(t *testing.T) {
+	runWhen(t, "enterprise", ">=3.0.0")
+	setup(t)
+
+	// test that reference to non-existing service fails.
+	pluginsNoLookupServiceStateFile := "testdata/sync/034-lookup-tags-consumerGroups/plugins_no_lookup_service.yaml"
+	errNoService := sync(pluginsNoLookupServiceStateFile)
+	require.Error(t, errNoService)
+	require.EqualError(t, errNoService, "building state: service foo for plugin rate-limiting-advanced: entity not found")
+
+	// test that reference to non-existing route fails.
+	pluginsNoLookupRouteStateFile := "testdata/sync/034-lookup-tags-consumerGroups/plugins_no_lookup_route.yaml"
+	errNoRoute := sync(pluginsNoLookupRouteStateFile)
+	require.Error(t, errNoRoute)
+	require.EqualError(t, errNoRoute, "building state: route bar for plugin rate-limiting-advanced: entity not found")
+
+	// test that reference to non-existing consumer group fails.
+	pluginsNoLookupConsumergroupStateFile := "testdata/sync/034-lookup-tags-consumerGroups/plugins_no_lookup_consumerGroup.yaml" //nolint:lll
+	errNoConsumerGroup := sync(pluginsNoLookupConsumergroupStateFile)
+	require.Error(t, errNoConsumerGroup)
+	require.EqualError(
+		t,
+		errNoConsumerGroup,
+		"building state: consumer-group foo2 for plugin rate-limiting-advanced: entity not found",
+	)
+
+	// test that reference to existing local service and consumer group succeeds.
+	pluginsAndEntitiesConsumersGroupsStateFile := "testdata/sync/034-lookup-tags-consumerGroups/plugins_and_entities_consumerGroups.yaml" //nolint:lll
+	require.NoError(t, sync(pluginsAndEntitiesConsumersGroupsStateFile))
+	reset(t)
+
+	// test that reference to existing global service and consumer group succeeds via lookup tags.
+	globalEntitiesConsumerGroupsStateFile := "testdata/sync/034-lookup-tags-consumerGroups/global_entities_consumerGroups.yaml" //nolint:lll
+	require.NoError(t, sync(globalEntitiesConsumerGroupsStateFile))
+	// sync plugins with lookup reference to global service.
+	pluginsServiceLookupStateFile := "testdata/sync/034-lookup-tags-consumerGroups/plugins_service_lookup.yaml"
+	require.NoError(t, sync(pluginsServiceLookupStateFile))
+	// sync plugins with lookup reference to global route.
+	pluginsRouteLookupStateFile := "testdata/sync/034-lookup-tags-consumerGroups/plugins_route_lookup.yaml"
+	require.NoError(t, sync(pluginsRouteLookupStateFile))
+	reset(t)
+
+	// test that reference to non-existing global service or consumer groups fails via lookup tags.
+	errServiceNoReference := sync(pluginsServiceLookupStateFile)
+	require.Error(t, errServiceNoReference)
+	require.EqualError(
+		t,
+		errServiceNoReference,
+		"building state: service foo for plugin rate-limiting-advanced: entity not found",
+	)
+
+	// test that reference to non-existing global route or consumer groups fails via lookup tags.
+	errRouteNoReference := sync(pluginsRouteLookupStateFile)
+	require.Error(t, errRouteNoReference)
+	require.EqualError(
+		t,
+		errRouteNoReference,
+		"building state: route bar for plugin rate-limiting-advanced: entity not found",
+	)
+}
+
 // test scope:
 //   - 3.5.0+
 //   - konnect