diff --git a/cmd/network-crawler/network-crawler.go b/cmd/network-crawler/network-crawler.go index c635064..1ad272e 100644 --- a/cmd/network-crawler/network-crawler.go +++ b/cmd/network-crawler/network-crawler.go @@ -62,16 +62,24 @@ func run() error { flagBucketName = flag.String("bucket-name", "", "GCS bucket name to upload external networks to") flagDryRun = flag.Bool("dry-run", false, "Skip uploading external networks to GCS") flagSkippedProviders skippedProviderFlag + flagVerbose bool + flagVerboseUsage = "Prints extra debug message" ) skippedProvidersUsage := fmt.Sprintf("Comma separated list of providers. Currently acceptable providers are: %v", common.AllProviders()) flag.Var(&flagSkippedProviders, "skipped-providers", skippedProvidersUsage) + flag.BoolVar(&flagVerbose, "verbose", flagVerbose, flagVerboseUsage) + flag.BoolVar(&flagVerbose, "v", flagVerbose, flagVerboseUsage+" (shorthand)") flag.Parse() if flagBucketName == nil || *flagBucketName == "" { return common.NoBucketNameSpecified() } + if flagVerbose { + common.SetVerbose() + } + if *flagDryRun { log.Print("Dry run specified. Instead of uploading the content to bucket will just print to stdout.") } diff --git a/pkg/common/envvars.go b/pkg/common/envvars.go new file mode 100644 index 0000000..b995d26 --- /dev/null +++ b/pkg/common/envvars.go @@ -0,0 +1,15 @@ +package common + +var ( + verbose = false +) + +// Verbose returns if verbose options is set +func Verbose() bool { + return verbose +} + +// SetVerbose enables verbose mode +func SetVerbose() { + verbose = true +} diff --git a/pkg/common/errors.go b/pkg/common/errors.go index 9fe6a87..f5a0119 100644 --- a/pkg/common/errors.go +++ b/pkg/common/errors.go @@ -74,3 +74,13 @@ func ErroneousPrefixOrderingError(bucketName string, prefixes []string) error { func NoBucketNameSpecified() error { return errors.New("bucket name not specified") } + +// RegionNetworksNotFound is returned when a region networks spec is not found +func RegionNetworksNotFound(region string) error { + return fmt.Errorf("region networks for region %s not found", region) +} + +// ServiceNetworksNotFound is returned when a service networks spec is not found +func ServiceNetworksNotFound(service string) error { + return fmt.Errorf("service networks for service %s not found", service) +} diff --git a/pkg/common/utils/test_utils.go b/pkg/common/testutils/test_utils.go similarity index 99% rename from pkg/common/utils/test_utils.go rename to pkg/common/testutils/test_utils.go index a28c13c..6e66899 100644 --- a/pkg/common/utils/test_utils.go +++ b/pkg/common/testutils/test_utils.go @@ -1,4 +1,4 @@ -package utils +package testutils import ( "testing" diff --git a/pkg/common/types.go b/pkg/common/types.go index 0d34e7c..7935c5e 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -1,9 +1,13 @@ package common import ( + "fmt" + "log" "net" + "strings" "github.com/pkg/errors" + "github.com/stackrox/external-network-pusher/pkg/common/utils" ) // NetworkCrawler defines an interface for the implementation @@ -37,6 +41,11 @@ type RegionNetworkDetail struct { type ProviderNetworkRanges struct { ProviderName string `json:"providerName"` RegionNetworks []*RegionNetworkDetail `json:"regionNetworks"` + + // prefixToRegionServiceNames is used to remove "redundant" network + // Redundancy is determined by user's predicate while adding a new IP prefix. + // More about the predicate function below. + prefixToRegionServiceNames map[string][]*RegionServicePair } // ExternalNetworkSources contains all the external networks for all providers @@ -44,18 +53,110 @@ type ExternalNetworkSources struct { ProviderNetworks []*ProviderNetworkRanges `json:"providerNetworks"` } +// RegionServicePair is a tuple of region and service names +type RegionServicePair struct { + Region string + Service string +} + +// Equals checks if two RegionServicePairs are equal +func (r *RegionServicePair) Equals(p *RegionServicePair) bool { + return r.Region == p.Region && r.Service == p.Service +} + +// String returns the string representation of a RegionServicePair +func (r *RegionServicePair) String() string { + return fmt.Sprintf("{%s, %s}", r.Region, r.Service) +} + +// IsRedundantRegionServicePairFn is a predicate function to determine +// if a new region service pair should be added to output or not. +// For example, if an IP address belongs to multiple region/service pairs, +// user needs to provide a predicate function which looks at the pairs that are +// already recorded in ProviderNetworkRanges and the new pair that is about +// to be added, then decide if the new pair should be added as well or not. +// The existing pairs are given one by one to the user. +// +// The return value indicates which pair to remove. There could be three +// different return outcomes. Remove the new pair, remove the existing pair, or keep +// both. The returned pair is first checked with the new pair before checking with +// the existing pair. In case of keeping both pairs, a nil value should be returned +type IsRedundantRegionServicePairFn func( + newPair *RegionServicePair, + existingPair *RegionServicePair, +) (*RegionServicePair, error) + +// GetDefaultRegionServicePairRedundancyCheck returns the default check +// Default check checks if region and service names are the same +func GetDefaultRegionServicePairRedundancyCheck() IsRedundantRegionServicePairFn { + return func( + newPair *RegionServicePair, + existingPair *RegionServicePair, + ) (*RegionServicePair, error) { + if newPair.Equals(existingPair) { + return newPair, nil + } + + return nil, nil + } +} + +// NewProviderNetworkRanges returns a new instance of ProviderNetworkRanges +func NewProviderNetworkRanges(providerName string) *ProviderNetworkRanges { + return &ProviderNetworkRanges{ + ProviderName: providerName, + RegionNetworks: make([]*RegionNetworkDetail, 0), + prefixToRegionServiceNames: make(map[string][]*RegionServicePair), + } +} + // AddIPPrefix adds the specified IP prefix to the region and service name pair // returns error if the IP given is not a valid IP prefix -func (p *ProviderNetworkRanges) AddIPPrefix(region, service, ipPrefix string) error { +func (p *ProviderNetworkRanges) AddIPPrefix(region, service, ipPrefix string, fn IsRedundantRegionServicePairFn) error { ip, prefix, err := net.ParseCIDR(ipPrefix) if err != nil || ip == nil || prefix == nil { return errors.Wrapf(err, "failed to parse address: %s", ip) } - if ip.To4() != nil { - p.addIPPrefix(region, service, ipPrefix, true) - } else { - p.addIPPrefix(region, service, ipPrefix, false) + isIPv4 := ip.To4() != nil + + // Check redundancy + existingPairs, ok := p.prefixToRegionServiceNames[ipPrefix] + if ok { + newPair := RegionServicePair{Region: region, Service: service} + for _, pair := range existingPairs { + redundantPair, err := fn(&newPair, pair) + if err != nil { + return err + } + if redundantPair != nil { + if redundantPair == &newPair { + // The new pair is redundant. Not adding it + return nil + } + // Check did not match with the new pair. Removing an old pair and continue pruning + err := p.removeIPPrefix(redundantPair.Region, redundantPair.Service, ipPrefix, isIPv4) + if err != nil { + return err + } + } + } } + + if existingPairs := p.prefixToRegionServiceNames[ipPrefix]; Verbose() && len(existingPairs) > 0 { + strs := make([]string, 0, len(existingPairs)) + for _, pair := range existingPairs { + strs = append(strs, pair.String()) + } + log.Printf( + "Multple usages found for CIDR: %s. About to add region: %s and service: %s."+ + " Existing region and service pairs are: %s", + ipPrefix, + region, + service, + strings.Join(strs, ", ")) + } + + p.addIPPrefix(region, service, ipPrefix, isIPv4) return nil } @@ -91,4 +192,96 @@ func (p *ProviderNetworkRanges) addIPPrefix(region, service, ip string, isIPv4 b } else { serviceIPRanges.IPv6Prefixes = append(serviceIPRanges.IPv6Prefixes, ip) } + + // Update cache + p.prefixToRegionServiceNames[ip] = + append(p.prefixToRegionServiceNames[ip], &RegionServicePair{Region: region, Service: service}) +} + +func (p *ProviderNetworkRanges) removeIPPrefix(region, service, ip string, isIPv4 bool) error { + var regionNetwork *RegionNetworkDetail + regionIndex := -1 + for i, network := range p.RegionNetworks { + if network.RegionName == region { + regionIndex = i + regionNetwork = network + break + } + } + if regionNetwork == nil { + return RegionNetworksNotFound(region) + } + + var serviceIPRanges *ServiceIPRanges + serviceIndex := -1 + for i, ips := range regionNetwork.ServiceNetworks { + if ips.ServiceName == service { + serviceIPRanges = ips + serviceIndex = i + break + } + } + if serviceIPRanges == nil { + return ServiceNetworksNotFound(service) + } + + serviceIPRanges.removeIPPrefix(ip, isIPv4) + if serviceIPRanges.isEmpty() { + // Remove this service networks spec + regionNetwork.ServiceNetworks = SvcIPRangesSliceRemove(regionNetwork.ServiceNetworks, serviceIndex) + } + if regionNetwork.isEmpty() { + // Remove this region + p.RegionNetworks = RgnNetDetSliceRemove(p.RegionNetworks, regionIndex) + } + + // Delete from cache as well + deletingIndex := -1 + existingPairs := p.prefixToRegionServiceNames[ip] + for i, pair := range existingPairs { + if pair.Region == region && pair.Service == service { + deletingIndex = i + break + } + } + if deletingIndex == -1 { + // Not found in cache. No-op + return nil + } + + p.prefixToRegionServiceNames[ip] = RgnSvcPairSliceRemove(p.prefixToRegionServiceNames[ip], deletingIndex) + return nil +} + +func (s *ServiceIPRanges) removeIPPrefix(deletingIP string, isIPv4 bool) { + deletingIndex := -1 + var deletingSlice *[]string + if isIPv4 { + deletingSlice = &s.IPv4Prefixes + } else { + deletingSlice = &s.IPv6Prefixes + } + + for i, ip := range *deletingSlice { + if ip == deletingIP { + deletingIndex = i + break + } + } + + if deletingIndex == -1 { + // Deleting an element that does not exist. No-op. + return + } + + // Move the last element to the deleting position and truncate + *deletingSlice = utils.StrSliceRemove(*deletingSlice, deletingIndex) +} + +func (s *ServiceIPRanges) isEmpty() bool { + return len(s.IPv4Prefixes)+len(s.IPv6Prefixes) == 0 +} + +func (r *RegionNetworkDetail) isEmpty() bool { + return len(r.ServiceNetworks) == 0 } diff --git a/pkg/common/types_util.go b/pkg/common/types_util.go new file mode 100644 index 0000000..828d1a0 --- /dev/null +++ b/pkg/common/types_util.go @@ -0,0 +1,36 @@ +package common + +import ( + "log" +) + +// (sigh, only if using reflection could be deemed "idigomatic"...) +// Also don't put these into pkg/common/utils/util.go since utils package should not +// depend on application specifics (for example, this common package). + +// RgnSvcPairSliceRemove removes an element from a RegionServicePair slice at the specified index +func RgnSvcPairSliceRemove(in []*RegionServicePair, i int) []*RegionServicePair { + if i < 0 || i >= len(in) { + log.Panicf("Index out of bound: %d", i) + } + in[i] = in[len(in)-1] + return in[:len(in)-1] +} + +// SvcIPRangesSliceRemove removes an element from a ServiceIPRanges slice at the specified index +func SvcIPRangesSliceRemove(in []*ServiceIPRanges, i int) []*ServiceIPRanges { + if i < 0 || i >= len(in) { + log.Panicf("Index out of bound: %d", i) + } + in[i] = in[len(in)-1] + return in[:len(in)-1] +} + +// RgnNetDetSliceRemove removes an element from a RegionNetworkDetail slice at the specified index +func RgnNetDetSliceRemove(in []*RegionNetworkDetail, i int) []*RegionNetworkDetail { + if i < 0 || i >= len(in) { + log.Panicf("Index out of bound: %d", i) + } + in[i] = in[len(in)-1] + return in[:len(in)-1] +} diff --git a/pkg/common/utils/util.go b/pkg/common/utils/util.go index ef4afb1..b6bd7ba 100644 --- a/pkg/common/utils/util.go +++ b/pkg/common/utils/util.go @@ -1,6 +1,7 @@ package utils import ( + "log" "strings" ) @@ -8,7 +9,10 @@ import ( // It ignores any empty tag (i.e.: empty string) // If the final element only contains one string, then that string // is returned as the compound name -func ToCompoundName(tags ...string) string { +func ToCompoundName(delim string, tags ...string) string { + if delim == "" { + delim = "/" + } filtered := make([]string, 0, len(tags)) for _, tag := range tags { if tag != "" { @@ -21,5 +25,22 @@ func ToCompoundName(tags ...string) string { if len(filtered) == 1 { return filtered[0] } - return strings.Join(filtered, "-") + return strings.Join(filtered, delim) +} + +// ToTags splits the compound name to a list of individual tags (names). +func ToTags(delim, compoundName string) []string { + if delim == "" { + delim = "/" + } + return strings.Split(compoundName, delim) +} + +// StrSliceRemove removes an element from a string slice at the specified index +func StrSliceRemove(in []string, i int) []string { + if i < 0 || i >= len(in) { + log.Panicf("Index out of bound: %d", i) + } + in[i] = in[len(in)-1] + return in[:len(in)-1] } diff --git a/pkg/crawlers/aws/aws.go b/pkg/crawlers/aws/aws.go index 582b453..492a637 100644 --- a/pkg/crawlers/aws/aws.go +++ b/pkg/crawlers/aws/aws.go @@ -81,14 +81,19 @@ func (c *awsNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkR return nil, errors.Wrap(err, "failed to unmarshal Amazon's network data") } - providerNetworks := common.ProviderNetworkRanges{ProviderName: c.GetProviderKey().String()} + providerNetworks := common.NewProviderNetworkRanges(c.GetProviderKey().String()) for _, ipv4Spec := range awsNetworkSpec.Prefixes { if ipv4Spec.IPPrefix == "" { // Empty IPv4. Something might be wrong here. Logging for warning log.Printf("Received an empty IPv4 definition: %v", ipv4Spec) continue } - err := providerNetworks.AddIPPrefix(ipv4Spec.Region, ipv4Spec.Service, ipv4Spec.IPPrefix) + err := + providerNetworks.AddIPPrefix( + ipv4Spec.Region, + ipv4Spec.Service, + ipv4Spec.IPPrefix, + c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf(err, "failed to add Amazon IPv4 prefix: %s", ipv4Spec.IPPrefix) } @@ -99,11 +104,20 @@ func (c *awsNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkR log.Printf("Received an empty IPv6 definition: %v", ipv6Spec) continue } - err := providerNetworks.AddIPPrefix(ipv6Spec.Region, ipv6Spec.Service, ipv6Spec.IPv6Prefix) + err := + providerNetworks.AddIPPrefix( + ipv6Spec.Region, + ipv6Spec.Service, + ipv6Spec.IPv6Prefix, + c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf(err, "failed to add Amazon IPv6 prefix: %s", ipv6Spec.IPv6Prefix) } } - return &providerNetworks, nil + return providerNetworks, nil +} + +func (c *awsNetworkCrawler) getComputeRedundancyFn() common.IsRedundantRegionServicePairFn { + return common.GetDefaultRegionServicePairRedundancyCheck() } diff --git a/pkg/crawlers/aws/aws_test.go b/pkg/crawlers/aws/aws_test.go index 1f1b808..8aa5b1c 100644 --- a/pkg/crawlers/aws/aws_test.go +++ b/pkg/crawlers/aws/aws_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/stackrox/external-network-pusher/pkg/common/utils" + "github.com/stackrox/external-network-pusher/pkg/common/testutils" "github.com/stretchr/testify/require" ) @@ -15,25 +15,25 @@ func TestAWSParseNetworks(t *testing.T) { service1, service2, service3 := "service1", "service2", "service3" testData := awsNetworkSpec{ - SyncToken: utils.UnusedString, - CreateDate: utils.UnusedString, + SyncToken: testutils.UnusedString, + CreateDate: testutils.UnusedString, Prefixes: []awsIPv4Spec{ { IPPrefix: ipv41, Region: region1, - NetworkBorderGroup: utils.UnusedString, + NetworkBorderGroup: testutils.UnusedString, Service: service1, }, { IPPrefix: ipv42, Region: region1, - NetworkBorderGroup: utils.UnusedString, + NetworkBorderGroup: testutils.UnusedString, Service: service2, }, { IPPrefix: ipv43, Region: region2, - NetworkBorderGroup: utils.UnusedString, + NetworkBorderGroup: testutils.UnusedString, Service: service1, }, }, @@ -41,19 +41,19 @@ func TestAWSParseNetworks(t *testing.T) { { IPv6Prefix: ipv61, Region: region1, - NetworkBorderGroup: utils.UnusedString, + NetworkBorderGroup: testutils.UnusedString, Service: service1, }, { IPv6Prefix: ipv62, Region: region2, - NetworkBorderGroup: utils.UnusedString, + NetworkBorderGroup: testutils.UnusedString, Service: service2, }, { IPv6Prefix: ipv63, Region: region3, - NetworkBorderGroup: utils.UnusedString, + NetworkBorderGroup: testutils.UnusedString, Service: service3, }, }, @@ -69,7 +69,7 @@ func TestAWSParseNetworks(t *testing.T) { // Three (region1, 2, 3) regions in total require.Equal(t, 3, len(parsedResult.RegionNetworks)) - regionNameToDetail := utils.GetRegionNameToDetails(parsedResult) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) // region1 { @@ -78,9 +78,9 @@ func TestAWSParseNetworks(t *testing.T) { // Two services in total for region1 require.Equal(t, 2, len(regionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) // service1 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service1, @@ -88,7 +88,7 @@ func TestAWSParseNetworks(t *testing.T) { []string{ipv61}) // service2 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service2, @@ -102,9 +102,9 @@ func TestAWSParseNetworks(t *testing.T) { // Two services in total for region2 require.Equal(t, 2, len(regionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) // service1 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service1, @@ -112,7 +112,7 @@ func TestAWSParseNetworks(t *testing.T) { []string{}) // service2 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service2, @@ -126,9 +126,9 @@ func TestAWSParseNetworks(t *testing.T) { // Only one service in region3 require.Equal(t, 1, len(regionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) // service3 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service3, @@ -136,3 +136,73 @@ func TestAWSParseNetworks(t *testing.T) { []string{ipv63}) } } + +func TestAWSRegionServiceRedundancyCheck(t *testing.T) { + addr := "3.5.140.0/22" + regionName := "testRegion" + serviceName1, serviceName2 := "testService1", "testService2" + + testData := awsNetworkSpec{ + SyncToken: testutils.UnusedString, + CreateDate: testutils.UnusedString, + Prefixes: []awsIPv4Spec{ + { + IPPrefix: addr, + Region: regionName, + NetworkBorderGroup: testutils.UnusedString, + Service: serviceName1, + }, + { + IPPrefix: addr, + Region: regionName, + NetworkBorderGroup: testutils.UnusedString, + Service: serviceName1, + }, + { + IPPrefix: addr, + Region: regionName, + NetworkBorderGroup: testutils.UnusedString, + Service: serviceName2, + }, + }, + IPv6Prefixes: []awsIPv6Spec{}, + } + + networks, err := json.Marshal(testData) + require.Nil(t, err) + + crawler := awsNetworkCrawler{} + parsedResult, err := crawler.parseNetworks(networks) + require.Nil(t, err) + require.Equal(t, parsedResult.ProviderName, crawler.GetProviderKey().String()) + + // One region in total + require.Equal(t, 1, len(parsedResult.RegionNetworks)) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) + + // testRegion + { + // Although in test data we have three entries, only two should be left + regionNetworks, ok := regionNameToDetail[regionName] + require.True(t, ok) + // Two services in total for region1 + require.Equal(t, 2, len(regionNetworks.ServiceNetworks)) + + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + // service1 + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + serviceName1, + []string{addr}, + []string{}) + + // service2 + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + serviceName2, + []string{addr}, + []string{}) + } +} diff --git a/pkg/crawlers/azure/azure.go b/pkg/crawlers/azure/azure.go index a7b82b6..5eb3289 100644 --- a/pkg/crawlers/azure/azure.go +++ b/pkg/crawlers/azure/azure.go @@ -26,6 +26,8 @@ import ( // If azureCloudEntityProperties.SystemService is empty, we just use the Platform // as the final service name. +const azureCompoundNameDelim = "/" + type azureNetworkCrawler struct { urls []string } @@ -68,8 +70,8 @@ func (c *azureNetworkCrawler) GetHumanReadableProviderName() string { } func (c *azureNetworkCrawler) GetNumRequiredIPPrefixes() int { - // Observed from past .json. In past we had 42440 - return 42000 + // Observed from past runs after dedupe. In the past we had 24387 + return 24000 } func (c *azureNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkRanges, error) { @@ -88,7 +90,7 @@ func (c *azureNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetwor } func (c *azureNetworkCrawler) parseAzureNetworks(cloudInfos [][]byte) (*common.ProviderNetworkRanges, error) { - providerNetworks := common.ProviderNetworkRanges{ProviderName: c.GetProviderKey().String()} + providerNetworks := common.NewProviderNetworkRanges(c.GetProviderKey().String()) for _, data := range cloudInfos { var cloud azureCloud err := json.Unmarshal(data, &cloud) @@ -104,7 +106,7 @@ func (c *azureNetworkCrawler) parseAzureNetworks(cloudInfos [][]byte) (*common.P serviceName := toServiceName(entity.Properties.Platform, entity.Properties.SystemService) for _, ipStr := range entity.Properties.AddressPrefixes { - err := providerNetworks.AddIPPrefix(regionName, serviceName, ipStr) + err := providerNetworks.AddIPPrefix(regionName, serviceName, ipStr, c.getComputeRedundancyFn()) if err != nil { // Stop here if we have detected an invalid IP string. This // means we probably are doing something very wrong (using expired @@ -115,15 +117,159 @@ func (c *azureNetworkCrawler) parseAzureNetworks(cloudInfos [][]byte) (*common.P } } - return &providerNetworks, nil + return providerNetworks, nil +} + +func (c *azureNetworkCrawler) getComputeRedundancyFn() common.IsRedundantRegionServicePairFn { + return func( + newPair *common.RegionServicePair, + existingPair *common.RegionServicePair, + ) (*common.RegionServicePair, error) { + // Since for an Azure IP prefix, we are doing some tricks with its region and service name + // We determine B redundant when for two prefixes A and B: + // A.region.IsSubsetOfOrEqualTo(B.region) && A.service.IsSubsetOfOrEqualTo(B.service) + // Example: + // A: Region: Azure/useast1, Service: APIGateway; B: Region: Azure, Service: APIGateway + redundantPair, err := c.isSubsetOfOrEqualTo(newPair, existingPair) + if err != nil { + return nil, err + } + + // If not redundant, this should be (nil, nil) + return redundantPair, nil + } +} + +// Returns the pair that is the superset (redundant) of the other. +// If it is the same then a random one is returned +func (c *azureNetworkCrawler) isSubsetOfOrEqualTo( + pair1, pair2 *common.RegionServicePair, +) (*common.RegionServicePair, error) { + if pair1.Equals(pair2) { + return pair1, nil + } + + redundantPairBasedOnRegion, err := c.checkSubsetBasedOnRegionName(pair1, pair2) + if err != nil { + return nil, err + } + redundantPairBasedOnService, err := c.checkSubsetBasedOnServiceName(pair1, pair2) + if err != nil { + return nil, err + } + if redundantPairBasedOnRegion == nil && redundantPairBasedOnService == nil { + return nil, nil + } + + if (redundantPairBasedOnRegion == nil || redundantPairBasedOnRegion.Equals(pair1)) && + (redundantPairBasedOnService == nil || redundantPairBasedOnService.Equals(pair1)) { + // It is impossible that both region and service results are nil (covered above). + // Thus either based on region or based on service, or based on both, pair1 deemed redundant. + return pair1, nil + } + if (redundantPairBasedOnRegion == nil || redundantPairBasedOnRegion.Equals(pair2)) && + (redundantPairBasedOnService == nil || redundantPairBasedOnService.Equals(pair2)) { + // It is impossible that both region and service results are nil (covered above). + // Thus either based on region or based on service, or based on both, pair2 deemed redundant. + return pair2, nil + } + + // No inclusive relationship. + return nil, nil +} + +// Returns the pair that is the superset (redundant) of the other +// Assuming the region names aren't equal. +func (c *azureNetworkCrawler) checkSubsetBasedOnRegionName( + pair1, pair2 *common.RegionServicePair, +) (*common.RegionServicePair, error) { + pair1CloudName, pair1RegionName, err := regionNameToCloudAndRegionNames(pair1.Region) + if err != nil { + return nil, err + } + pair2CloudName, pair2RegionName, err := regionNameToCloudAndRegionNames(pair2.Region) + if err != nil { + return nil, err + } + if pair1CloudName == pair2CloudName { + if pair1RegionName == "" { + // pair2 specific region name not empty. pair1 >> pair2 + return pair1, nil + } + if pair2RegionName == "" { + // pair1 specific region name not empty. pair2 >> pair1 + return pair2, nil + } + // Reaching here means although both cloud names are the same, specific region names + // are not empty and different. + } + + // No inclusive relationship. + return nil, nil +} + +// Returns the pair that is the superset (redundant) of the other +// Assuming the service names aren't equal. +func (c *azureNetworkCrawler) checkSubsetBasedOnServiceName( + pair1, pair2 *common.RegionServicePair, +) (*common.RegionServicePair, error) { + pair1PlatformName, pair1ServiceName, err := serviceNameToPlatformAndServiceNames(pair1.Service) + if err != nil { + return nil, err + } + pair2PlatformName, pair2ServiceName, err := serviceNameToPlatformAndServiceNames(pair2.Service) + if err != nil { + return nil, err + } + if pair1PlatformName == pair2PlatformName { + if pair1ServiceName == "" { + // pair2 specific service name not empty. pair1 >> pair2 + return pair1, nil + } + if pair2ServiceName == "" { + // pair1 specific service name not empty. pair2 >> pair1 + return pair2, nil + } + // Reaching here means although both platform names are the same, specific service names + // are not empty and different. + } + + // No inclusive relationship. + return nil, nil +} + +func regionNameToCloudAndRegionNames(regionName string) (string, string, error) { + splitted := utils.ToTags(azureCompoundNameDelim, regionName) + switch len(splitted) { + case 1: + // No specific region name provided + return regionName, "", nil + case 2: + return splitted[0], splitted[1], nil + default: + return "", "", InvalidAzureCompoundRegionName(regionName) + } +} + +func serviceNameToPlatformAndServiceNames(serviceName string) (string, string, error) { + splitted := utils.ToTags(azureCompoundNameDelim, serviceName) + switch len(splitted) { + case 1: + // No specific service name provided + return serviceName, "", nil + case 2: + return splitted[0], splitted[1], nil + default: + return "", "", InvalidAzureCompoundServiceName(serviceName) + } } func toRegionName(cloudName, regionName string) string { - return utils.ToCompoundName(cloudName, regionName) + return utils.ToCompoundName(azureCompoundNameDelim, cloudName, regionName) } func toServiceName(platformName, serviceName string) string { - return utils.ToCompoundName(platformName, serviceName) + return utils.ToCompoundName(azureCompoundNameDelim, platformName, serviceName) } func (c *azureNetworkCrawler) fetchAll() ([][]byte, error) { diff --git a/pkg/crawlers/azure/azure_test.go b/pkg/crawlers/azure/azure_test.go index c646fea..3a33801 100644 --- a/pkg/crawlers/azure/azure_test.go +++ b/pkg/crawlers/azure/azure_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/stackrox/external-network-pusher/pkg/common/utils" + "github.com/stackrox/external-network-pusher/pkg/common/testutils" "github.com/stretchr/testify/require" ) @@ -38,78 +38,78 @@ func TestAzureParseNetwork(t *testing.T) { ) testCloud1 := azureCloud{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Cloud: cloud1, Values: []azureCloudEntity{ { Name: service1, ID: service1, Properties: azureCloudEntityProperties{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Region: region1, - RegionID: utils.UnusedInt, + RegionID: testutils.UnusedInt, Platform: platform, SystemService: service1, AddressPrefixes: []string{c1r1s1IPv41, c1r1s1IPv42}, - NetworkFeatures: utils.UnusedStrSlice, + NetworkFeatures: testutils.UnusedStrSlice, }, }, { Name: service2, ID: service2, Properties: azureCloudEntityProperties{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Region: region1, - RegionID: utils.UnusedInt, + RegionID: testutils.UnusedInt, Platform: platform, SystemService: service2, AddressPrefixes: []string{c1r1s2IPv6}, - NetworkFeatures: utils.UnusedStrSlice, + NetworkFeatures: testutils.UnusedStrSlice, }, }, { Name: "AzureCloud." + region1, ID: "AzureCloud." + region1, Properties: azureCloudEntityProperties{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Region: region1, - RegionID: utils.UnusedInt, + RegionID: testutils.UnusedInt, Platform: platform, SystemService: emptyService, AddressPrefixes: []string{c1r1NoServiceIPv4, c1r1NoServiceIPv6}, - NetworkFeatures: utils.UnusedStrSlice, + NetworkFeatures: testutils.UnusedStrSlice, }, }, }, } testCloud2 := azureCloud{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Cloud: cloud2, Values: []azureCloudEntity{ { Name: service2, ID: service2, Properties: azureCloudEntityProperties{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Region: region2, - RegionID: utils.UnusedInt, + RegionID: testutils.UnusedInt, Platform: platform, SystemService: service2, AddressPrefixes: []string{c2r2s2IPv4}, - NetworkFeatures: utils.UnusedStrSlice, + NetworkFeatures: testutils.UnusedStrSlice, }, }, { Name: "Azure", ID: "Azure", Properties: azureCloudEntityProperties{ - ChangeNumber: utils.UnusedInt, + ChangeNumber: testutils.UnusedInt, Region: emptyRegion, - RegionID: utils.UnusedInt, + RegionID: testutils.UnusedInt, Platform: platform, SystemService: emptyService, AddressPrefixes: []string{c2NoRegionNoServiceIPv4, c2NoRegionNoServiceIPv6}, - NetworkFeatures: utils.UnusedStrSlice, + NetworkFeatures: testutils.UnusedStrSlice, }, }, }, @@ -127,7 +127,7 @@ func TestAzureParseNetwork(t *testing.T) { // There should be 3 regions in total (c1r1, c2r2, c2) require.Equal(t, 3, len(parsedResult.RegionNetworks)) - regionNameToDetail := utils.GetRegionNameToDetails(parsedResult) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) // Region1 (c1r1) { @@ -137,8 +137,8 @@ func TestAzureParseNetwork(t *testing.T) { require.Equal(t, 3, len(regionNetworks.ServiceNetworks)) service := toServiceName(platform, service1) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) - utils.CheckServiceIPsInRegion( + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -146,7 +146,7 @@ func TestAzureParseNetwork(t *testing.T) { []string{}) service = toServiceName(platform, service2) - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -154,7 +154,7 @@ func TestAzureParseNetwork(t *testing.T) { []string{c1r1s2IPv6}) service = toServiceName(platform, emptyService) - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -169,8 +169,8 @@ func TestAzureParseNetwork(t *testing.T) { require.Equal(t, 1, len(regionNetworks.ServiceNetworks)) service := toServiceName(platform, service2) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) - utils.CheckServiceIPsInRegion( + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -185,8 +185,8 @@ func TestAzureParseNetwork(t *testing.T) { require.Equal(t, 1, len(regionNetworks.ServiceNetworks)) service := toServiceName(platform, emptyService) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) - utils.CheckServiceIPsInRegion( + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -194,3 +194,105 @@ func TestAzureParseNetwork(t *testing.T) { []string{c2NoRegionNoServiceIPv6}) } } + +func TestAzureRegionServiceRedundancyCheck(t *testing.T) { + cloudName := "test-cloud" + regionName, emptyRegion := "test-region", "" + platformName := "test-platform" + serviceName, emptyService := "test-service", "" + addr := "20.140.48.160/27" + + // Given four entities. + // First one has: (cloud, platform/service) + // Second one has: (cloud, platform) + // Third one has: (cloud/region, platform) + // Fourth one has: (cloud/region, platform/service) + // After parsing, there should only be one entry left (everything but the fourth one should be cleared). + testCloud := azureCloud{ + ChangeNumber: testutils.UnusedInt, + Cloud: cloudName, + Values: []azureCloudEntity{ + { + Name: testutils.UnusedString, + ID: testutils.UnusedString, + Properties: azureCloudEntityProperties{ + ChangeNumber: testutils.UnusedInt, + Region: emptyRegion, + RegionID: testutils.UnusedInt, + Platform: platformName, + SystemService: serviceName, + AddressPrefixes: []string{addr}, + NetworkFeatures: testutils.UnusedStrSlice, + }, + }, + { + Name: testutils.UnusedString, + ID: testutils.UnusedString, + Properties: azureCloudEntityProperties{ + ChangeNumber: testutils.UnusedInt, + Region: emptyRegion, + RegionID: testutils.UnusedInt, + Platform: platformName, + SystemService: emptyService, + AddressPrefixes: []string{addr}, + NetworkFeatures: testutils.UnusedStrSlice, + }, + }, + { + Name: testutils.UnusedString, + ID: testutils.UnusedString, + Properties: azureCloudEntityProperties{ + ChangeNumber: testutils.UnusedInt, + Region: regionName, + RegionID: testutils.UnusedInt, + Platform: platformName, + SystemService: emptyService, + AddressPrefixes: []string{addr}, + NetworkFeatures: testutils.UnusedStrSlice, + }, + }, + { + Name: testutils.UnusedString, + ID: testutils.UnusedString, + Properties: azureCloudEntityProperties{ + ChangeNumber: testutils.UnusedInt, + Region: regionName, + RegionID: testutils.UnusedInt, + Platform: platformName, + SystemService: serviceName, + AddressPrefixes: []string{addr}, + NetworkFeatures: testutils.UnusedStrSlice, + }, + }, + }, + } + + cloudNetworks, err := json.Marshal(testCloud) + require.Nil(t, err) + + crawler := azureNetworkCrawler{} + parsedResult, err := crawler.parseAzureNetworks([][]byte{cloudNetworks}) + require.Nil(t, err) + require.Equal(t, parsedResult.ProviderName, crawler.GetProviderKey().String()) + + // One region + require.Equal(t, 1, len(parsedResult.RegionNetworks)) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) + + // Check region content + { + region := toRegionName(cloudName, regionName) + regionNetworks, ok := regionNameToDetail[region] + require.True(t, ok) + require.Equal(t, 1, len(regionNetworks.ServiceNetworks)) + + service := toServiceName(platformName, serviceName) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + service, + []string{addr}, + []string{}) + } +} diff --git a/pkg/crawlers/azure/errors.go b/pkg/crawlers/azure/errors.go new file mode 100644 index 0000000..0c93653 --- /dev/null +++ b/pkg/crawlers/azure/errors.go @@ -0,0 +1,15 @@ +package azure + +import ( + "fmt" +) + +// InvalidAzureCompoundRegionName is returned when an invalid Azure compound region name is found +func InvalidAzureCompoundRegionName(regionName string) error { + return fmt.Errorf("invalid compound region name found: %s", regionName) +} + +// InvalidAzureCompoundServiceName is returned when an invalid Azure compound service name is found +func InvalidAzureCompoundServiceName(serviceName string) error { + return fmt.Errorf("invalid compound service name found: %s", serviceName) +} diff --git a/pkg/crawlers/cloudflare/cloudflare.go b/pkg/crawlers/cloudflare/cloudflare.go index e281565..f2e219b 100644 --- a/pkg/crawlers/cloudflare/cloudflare.go +++ b/pkg/crawlers/cloudflare/cloudflare.go @@ -74,25 +74,39 @@ func (c *cloudflareNetworkCrawler) parseNetworks(networks []byte) (*common.Provi return nil, errors.Wrap(err, "failed to unmarshal Cloudflare's network data") } - providerNetworks := common.ProviderNetworkRanges{ProviderName: c.GetProviderKey().String()} + providerNetworks := common.NewProviderNetworkRanges(c.GetProviderKey().String()) for _, ipv4Str := range cloudflareNetworkSpec.Result.IPv4CIDRs { ipv4Str = unescapeIPPrefix(ipv4Str) - err := providerNetworks.AddIPPrefix(common.DefaultRegion, common.DefaultService, ipv4Str) + err := + providerNetworks.AddIPPrefix( + common.DefaultRegion, + common.DefaultService, + ipv4Str, + c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf(err, "failed to add IPv4 prefix: %s to the Cloudflare's result", ipv4Str) } } for _, ipv6Str := range cloudflareNetworkSpec.Result.IPv6CIDRs { ipv6Str = unescapeIPPrefix(ipv6Str) - err := providerNetworks.AddIPPrefix(common.DefaultRegion, common.DefaultService, ipv6Str) + err := + providerNetworks.AddIPPrefix( + common.DefaultRegion, + common.DefaultService, + ipv6Str, + c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf(err, "failed to add IPv6 prefix: %s to the Cloudflare's result", ipv6Str) } } - return &providerNetworks, nil + return providerNetworks, nil } func unescapeIPPrefix(prefix string) string { return strings.ReplaceAll(prefix, `\/`, "/") } + +func (c *cloudflareNetworkCrawler) getComputeRedundancyFn() common.IsRedundantRegionServicePairFn { + return common.GetDefaultRegionServicePairRedundancyCheck() +} diff --git a/pkg/crawlers/cloudflare/cloudflare_test.go b/pkg/crawlers/cloudflare/cloudflare_test.go index 7a17786..e3c8f65 100644 --- a/pkg/crawlers/cloudflare/cloudflare_test.go +++ b/pkg/crawlers/cloudflare/cloudflare_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stackrox/external-network-pusher/pkg/common" - "github.com/stackrox/external-network-pusher/pkg/common/utils" + "github.com/stackrox/external-network-pusher/pkg/common/testutils" "github.com/stretchr/testify/require" ) @@ -19,11 +19,11 @@ func TestCloudflareParseNetwork(t *testing.T) { Result: cloudflareNetworkResult{ IPv4CIDRs: []string{ipv41, ipv42, ipv43}, IPv6CIDRs: []string{ipv61, ipv62, ipv63}, - ETag: utils.UnusedString, + ETag: testutils.UnusedString, }, - Success: utils.UnusedBool, - Errors: utils.UnusedStrSlice, - Messages: utils.UnusedStrSlice, + Success: testutils.UnusedBool, + Errors: testutils.UnusedStrSlice, + Messages: testutils.UnusedStrSlice, } networks, err := json.Marshal(testData) require.Nil(t, err) @@ -42,13 +42,56 @@ func TestCloudflareParseNetwork(t *testing.T) { require.Equal(t, 1, len(regionNetworks.ServiceNetworks)) // Check content of service - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) escapedIPv4s := []string{unescapeIPPrefix(ipv41), unescapeIPPrefix(ipv42), unescapeIPPrefix(ipv43)} escapedIPv6s := []string{unescapeIPPrefix(ipv61), unescapeIPPrefix(ipv62), unescapeIPPrefix(ipv63)} - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, common.DefaultService, escapedIPv4s, escapedIPv6s) } + +func TestCloudflareRegionServiceRedundancyCheck(t *testing.T) { + // Cloudflare provides their IPs at URL: https://api.cloudflare.com/client/v4/ips + // with slashes escaped. Mimic that + addr := `173.245.48.0\/20` + + testData := cloudflareNetworkSpec{ + Result: cloudflareNetworkResult{ + // Repeat the addresses couple times and make sure we dedupe + IPv4CIDRs: []string{addr, addr, addr}, + IPv6CIDRs: []string{}, + ETag: testutils.UnusedString, + }, + Success: testutils.UnusedBool, + Errors: testutils.UnusedStrSlice, + Messages: testutils.UnusedStrSlice, + } + networks, err := json.Marshal(testData) + require.Nil(t, err) + + crawler := cloudflareNetworkCrawler{} + parsedResult, err := crawler.parseNetworks(networks) + require.Nil(t, err) + require.Equal(t, parsedResult.ProviderName, crawler.GetProviderKey().String()) + + // Just one region in total. common.DefaultRegion + require.Equal(t, 1, len(parsedResult.RegionNetworks)) + + // Check content of the region + regionNetworks := parsedResult.RegionNetworks[0] + // Just one service in total. common.DefaultService + require.Equal(t, 1, len(regionNetworks.ServiceNetworks)) + + // Check content of service + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + escapedIPv4s := []string{unescapeIPPrefix(addr)} + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + common.DefaultService, + escapedIPv4s, + []string{}) +} diff --git a/pkg/crawlers/gcp/gcp.go b/pkg/crawlers/gcp/gcp.go index e4d6910..652e097 100644 --- a/pkg/crawlers/gcp/gcp.go +++ b/pkg/crawlers/gcp/gcp.go @@ -72,24 +72,38 @@ func (c *gcpNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkR return nil, errors.Wrap(err, "failed to unmarshal Google's network data") } - providerNetworks := common.ProviderNetworkRanges{ProviderName: c.GetProviderKey().String()} + providerNetworks := common.NewProviderNetworkRanges(c.GetProviderKey().String()) for _, gcpIPSpec := range gcpNetworkSpec.Prefixes { if gcpIPSpec.Ipv4Prefix == "" && gcpIPSpec.Ipv6Prefix == "" { continue } if gcpIPSpec.Ipv4Prefix != "" { - err := providerNetworks.AddIPPrefix(gcpIPSpec.Scope, gcpIPSpec.Service, gcpIPSpec.Ipv4Prefix) + err := + providerNetworks.AddIPPrefix( + gcpIPSpec.Scope, + gcpIPSpec.Service, + gcpIPSpec.Ipv4Prefix, + c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf(err, "failed to add Google IPv4 Prefix: %s", gcpIPSpec.Ipv4Prefix) } } if gcpIPSpec.Ipv6Prefix != "" { - err := providerNetworks.AddIPPrefix(gcpIPSpec.Scope, gcpIPSpec.Service, gcpIPSpec.Ipv6Prefix) + err := + providerNetworks.AddIPPrefix( + gcpIPSpec.Scope, + gcpIPSpec.Service, + gcpIPSpec.Ipv6Prefix, + c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf(err, "failed to add Google IPv6 Prefix: %s", gcpIPSpec.Ipv6Prefix) } } } - return &providerNetworks, nil + return providerNetworks, nil +} + +func (c *gcpNetworkCrawler) getComputeRedundancyFn() common.IsRedundantRegionServicePairFn { + return common.GetDefaultRegionServicePairRedundancyCheck() } diff --git a/pkg/crawlers/gcp/gcp_test.go b/pkg/crawlers/gcp/gcp_test.go index 100efae..cb3f15a 100644 --- a/pkg/crawlers/gcp/gcp_test.go +++ b/pkg/crawlers/gcp/gcp_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/stackrox/external-network-pusher/pkg/common/utils" + "github.com/stackrox/external-network-pusher/pkg/common/testutils" "github.com/stretchr/testify/require" ) @@ -15,8 +15,8 @@ func TestGcpParseNetwork(t *testing.T) { region1, region2 := "asia-east1", "europe-west4" testData := gcpNetworkSpec{ - SyncToken: utils.UnusedString, - CreationTime: utils.UnusedString, + SyncToken: testutils.UnusedString, + CreationTime: testutils.UnusedString, Prefixes: []gcpIPSpec{ { Ipv4Prefix: ipv41, @@ -51,7 +51,7 @@ func TestGcpParseNetwork(t *testing.T) { // Two regions in total require.Equal(t, 2, len(parsedResult.RegionNetworks)) - regionNameToDetail := utils.GetRegionNameToDetails(parsedResult) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) // Check content of the first region { @@ -60,10 +60,10 @@ func TestGcpParseNetwork(t *testing.T) { // Two services in total for region1 require.Equal(t, 2, len(firstRegionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(firstRegionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(firstRegionNetworks) // service1 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service1, @@ -71,7 +71,7 @@ func TestGcpParseNetwork(t *testing.T) { []string{ipv61}) // service2 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service2, @@ -85,10 +85,10 @@ func TestGcpParseNetwork(t *testing.T) { // Only one service in region2 require.Equal(t, 1, len(secondRegionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(secondRegionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(secondRegionNetworks) // service1 - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service1, @@ -96,3 +96,68 @@ func TestGcpParseNetwork(t *testing.T) { []string{}) } } + +func TestGCPRegionServiceRedundancyCheck(t *testing.T) { + addr := "34.80.0.0/15" + service1, service2 := "Google Cloud", "Google Ads" + regionName := "test-region" + + testData := gcpNetworkSpec{ + SyncToken: testutils.UnusedString, + CreationTime: testutils.UnusedString, + Prefixes: []gcpIPSpec{ + { + Ipv4Prefix: addr, + Service: service1, + Scope: regionName, + }, + { + Ipv4Prefix: addr, + Service: service1, + Scope: regionName, + }, + { + Ipv4Prefix: addr, + Service: service2, + Scope: regionName, + }, + }, + } + networks, err := json.Marshal(testData) + require.Nil(t, err) + + crawler := gcpNetworkCrawler{} + parsedResult, err := crawler.parseNetworks(networks) + require.Nil(t, err) + require.Equal(t, parsedResult.ProviderName, crawler.GetProviderKey().String()) + + // One regions in total + require.Equal(t, 1, len(parsedResult.RegionNetworks)) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) + + // Check content of the region + { + firstRegionNetworks, ok := regionNameToDetail[regionName] + require.True(t, ok) + // Two services in total for region1 + require.Equal(t, 2, len(firstRegionNetworks.ServiceNetworks)) + + serviceToIPs := testutils.GetServiceNameToIPs(firstRegionNetworks) + + // service1 + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + service1, + []string{addr}, + []string{}) + + // service2 + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + service2, + []string{addr}, + []string{}) + } +} diff --git a/pkg/crawlers/oracle/oracle.go b/pkg/crawlers/oracle/oracle.go index e04553a..bf75c1a 100644 --- a/pkg/crawlers/oracle/oracle.go +++ b/pkg/crawlers/oracle/oracle.go @@ -75,12 +75,13 @@ func (c *ociNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkR return nil, errors.Wrap(err, "failed to unmarshal Oracle's network data") } - providerNetworks := common.ProviderNetworkRanges{ProviderName: c.GetProviderKey().String()} + providerNetworks := common.NewProviderNetworkRanges(c.GetProviderKey().String()) for _, regionNetworks := range ociNetworkSpec.Regions { for _, cidrDef := range regionNetworks.CIDRs { // sort the tags before creating service name to make service name consistent service := toServiceName(cidrDef.Tags) - err := providerNetworks.AddIPPrefix(regionNetworks.Region, service, cidrDef.CIDR) + err := + providerNetworks.AddIPPrefix(regionNetworks.Region, service, cidrDef.CIDR, c.getComputeRedundancyFn()) if err != nil { return nil, errors.Wrapf( err, @@ -90,10 +91,18 @@ func (c *ociNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkR } } - return &providerNetworks, nil + return providerNetworks, nil } func toServiceName(tags []string) string { sort.Strings(tags) - return utils.ToCompoundName(tags...) + // Using "|" as deliminator since these are tags and tag1|tag2 as a service name + // seems the best way to represent them, instead of "/" since this could potentially + // be understood in a way that the tags have some sort of hierarchical relationships + // between them. + return utils.ToCompoundName("|", tags...) +} + +func (c *ociNetworkCrawler) getComputeRedundancyFn() common.IsRedundantRegionServicePairFn { + return common.GetDefaultRegionServicePairRedundancyCheck() } diff --git a/pkg/crawlers/oracle/oracle_test.go b/pkg/crawlers/oracle/oracle_test.go index 89021e1..f253b44 100644 --- a/pkg/crawlers/oracle/oracle_test.go +++ b/pkg/crawlers/oracle/oracle_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/stackrox/external-network-pusher/pkg/common/utils" + "github.com/stackrox/external-network-pusher/pkg/common/testutils" "github.com/stretchr/testify/require" ) @@ -14,7 +14,7 @@ func TestOCIParseNetwork(t *testing.T) { ipv41, ipv42, ipv43, ipv44, ipv45 := "129.146.0.0/21", "129.146.64.0/18", "158.101.0.0/18", "193.123.0.0/19", "207.135.0.0/22" testData := ociNetworkSpec{ - LastUpdatedTimestamp: utils.UnusedString, + LastUpdatedTimestamp: testutils.UnusedString, Regions: []ociRegionNetworkDetails{ { Region: region1, @@ -59,7 +59,7 @@ func TestOCIParseNetwork(t *testing.T) { // Two regions in total require.Equal(t, 2, len(parsedResult.RegionNetworks)) - regionNameToDetail := utils.GetRegionNameToDetails(parsedResult) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) // region1 { @@ -68,10 +68,10 @@ func TestOCIParseNetwork(t *testing.T) { // Two services in total for region1 (tag1, tag2-tag3) require.Equal(t, 2, len(regionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) // service1 service := toServiceName([]string{tag1}) - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -80,7 +80,7 @@ func TestOCIParseNetwork(t *testing.T) { // service2 service = toServiceName([]string{tag2, tag3}) - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -94,10 +94,10 @@ func TestOCIParseNetwork(t *testing.T) { // Two services in total for region2 (tag1, tag2) require.Equal(t, 2, len(regionNetworks.ServiceNetworks)) - serviceToIPs := utils.GetServiceNameToIPs(regionNetworks) + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) // service1 service := toServiceName([]string{tag1}) - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -106,7 +106,7 @@ func TestOCIParseNetwork(t *testing.T) { // service2 service = toServiceName([]string{tag2}) - utils.CheckServiceIPsInRegion( + testutils.CheckServiceIPsInRegion( t, serviceToIPs, service, @@ -114,3 +114,71 @@ func TestOCIParseNetwork(t *testing.T) { []string{}) } } + +func TestOCIRegionServiceRedundancyCheck(t *testing.T) { + regionName := "us-phoenix-1" + tag1, tag2, tag3 := "OCI", "OSN", "OBJECT_STORAGE" + addr := "129.146.0.0/21" + + testData := ociNetworkSpec{ + LastUpdatedTimestamp: testutils.UnusedString, + Regions: []ociRegionNetworkDetails{ + { + Region: regionName, + CIDRs: []ociCIDRDefinition{ + { + CIDR: addr, + Tags: []string{tag1}, + }, + { + CIDR: addr, + Tags: []string{tag1}, + }, + { + CIDR: addr, + Tags: []string{tag2, tag3}, + }, + }, + }, + }, + } + + networks, err := json.Marshal(testData) + require.Nil(t, err) + + crawler := ociNetworkCrawler{} + parsedResult, err := crawler.parseNetworks(networks) + require.Nil(t, err) + require.Equal(t, parsedResult.ProviderName, crawler.GetProviderKey().String()) + + // One regions in total + require.Equal(t, 1, len(parsedResult.RegionNetworks)) + regionNameToDetail := testutils.GetRegionNameToDetails(parsedResult) + + // Check region content + { + regionNetworks, ok := regionNameToDetail[regionName] + require.True(t, ok) + // Two services in total for region1 (tag1, tag2-tag3) + require.Equal(t, 2, len(regionNetworks.ServiceNetworks)) + + serviceToIPs := testutils.GetServiceNameToIPs(regionNetworks) + // service1 + service := toServiceName([]string{tag1}) + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + service, + []string{addr}, + []string{}) + + // service2 + service = toServiceName([]string{tag2, tag3}) + testutils.CheckServiceIPsInRegion( + t, + serviceToIPs, + service, + []string{addr}, + []string{}) + } +}