Implement related.entity on CSPM (#2397)

* Add entity.id to findings events * Change to `related.entities` * Add test verification that related field is present * Change to `related.entity`
elastic · Aug 8, 2024 · 21ed3ca · 21ed3ca
1 parent 229a005
commit 21ed3ca
Show file tree

Hide file tree

Showing 27 changed files with 203 additions and 13 deletions.
diff --git a/internal/evaluator/opa_test.go b/internal/evaluator/opa_test.go
@@ -36,15 +36,20 @@ import (
 	"github.com/elastic/cloudbeat/internal/resources/utils/testhelper"
 )
 
-type DummyResource struct {
-}
+type DummyResource struct{}
 
 func (d *DummyResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return fetching.ResourceMetadata{}, nil
 }
+
 func (d *DummyResource) GetData() any {
 	return d
 }
+
+func (d *DummyResource) GetIds() []string {
+	return nil
+}
+
 func (d *DummyResource) GetElasticCommonData() (map[string]any, error) {
 	return nil, nil
 }

diff --git a/internal/resources/fetching/fetcher.go b/internal/resources/fetching/fetcher.go
@@ -137,6 +137,7 @@ type Resource interface {
 	GetMetadata() (ResourceMetadata, error)
 	GetData() any
 	GetElasticCommonData() (map[string]any, error)
+	GetIds() []string
 }
 
 type ResourceFields struct {

diff --git a/internal/resources/fetching/fetchers/aws/ecr_fetcher.go b/internal/resources/fetching/fetchers/aws/ecr_fetcher.go
@@ -31,6 +31,7 @@ import (
 	"github.com/elastic/cloudbeat/internal/resources/fetching"
 	"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
 	"github.com/elastic/cloudbeat/internal/resources/providers/awslib/ecr"
+	"github.com/elastic/cloudbeat/internal/resources/utils/pointers"
 )
 
 const (
@@ -126,6 +127,10 @@ func (res EcrResource) GetData() any {
 	return res
 }
 
+func (res EcrResource) GetIds() []string {
+	return []string{pointers.Deref(res.RepositoryArn), pointers.Deref(res.RegistryId)}
+}
+
 func (res EcrResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	if res.RepositoryArn == nil || res.RepositoryName == nil {
 		return fetching.ResourceMetadata{}, errors.New("received nil pointer")

diff --git a/internal/resources/fetching/fetchers/aws/elb_fetcher.go b/internal/resources/fetching/fetchers/aws/elb_fetcher.go
@@ -111,16 +111,24 @@ func (r ElbResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	if r.lb.LoadBalancerName == nil {
 		return fetching.ResourceMetadata{}, errors.New("received nil pointer")
 	}
-
 	return fetching.ResourceMetadata{
-		// A compromise because aws-sdk do not return an arn for an Elb
-		ID:      fmt.Sprintf("%s-%s", r.identity.Account, *r.lb.LoadBalancerName),
+		ID:      r.buildId(),
 		Type:    fetching.CloudLoadBalancer,
 		SubType: fetching.ElbType,
 		Name:    *r.lb.LoadBalancerName,
 	}, nil
 }
 
+// buildId A compromise because aws-sdk do not return an arn for an Elb
+func (r ElbResource) buildId() string {
+	id := fmt.Sprintf("%s-%s", r.identity.Account, *r.lb.LoadBalancerName)
+	return id
+}
+
+func (r ElbResource) GetIds() []string {
+	return []string{r.buildId()}
+}
+
 func (ElbResource) GetElasticCommonData() (map[string]any, error) {
 	return map[string]any{
 		"cloud.service.name": "ELB",

diff --git a/internal/resources/fetching/fetchers/aws/iam_fetcher.go b/internal/resources/fetching/fetchers/aws/iam_fetcher.go
@@ -115,6 +115,10 @@ func (r IAMResource) GetData() any {
 	return r.AwsResource
 }
 
+func (r IAMResource) GetIds() []string {
+	return []string{r.GetResourceArn(), r.identity.Account}
+}
+
 func (r IAMResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	identifier := r.GetResourceArn()
 	if identifier == "" {

diff --git a/internal/resources/fetching/fetchers/aws/kms_fetcher.go b/internal/resources/fetching/fetchers/aws/kms_fetcher.go
@@ -84,6 +84,10 @@ func (r KmsResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
+func (r KmsResource) GetIds() []string {
+	return []string{r.key.GetResourceArn()}
+}
+
 func (r KmsResource) GetElasticCommonData() (map[string]any, error) {
 	m := map[string]any{
 		"cloud.service.name": "KMS",

diff --git a/internal/resources/fetching/fetchers/aws/logging_fetcher.go b/internal/resources/fetching/fetchers/aws/logging_fetcher.go
@@ -100,6 +100,13 @@ func (r LoggingResource) GetData() any {
 	return r.AwsResource
 }
 
+func (r LoggingResource) GetIds() []string {
+	return []string{
+		r.GetResourceArn(),
+		r.GetResourceName(), // trail names are unique and at times the unique identifier present in logs
+	}
+}
+
 func (r LoggingResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return fetching.ResourceMetadata{
 		ID:      r.GetResourceArn(),
@@ -109,6 +116,7 @@ func (r LoggingResource) GetMetadata() (fetching.ResourceMetadata, error) {
 		Region:  r.GetRegion(),
 	}, nil
 }
+
 func (r LoggingResource) GetElasticCommonData() (map[string]any, error) {
 	return map[string]any{
 		"cloud.service.name": "CloudTrail",
@@ -129,4 +137,8 @@ func (c ConfigResource) GetData() any {
 	return c.configs
 }
 
+func (c ConfigResource) GetIds() []string {
+	return []string{}
+}
+
 func (c ConfigResource) GetElasticCommonData() (map[string]any, error) { return nil, nil }
diff --git a/internal/resources/fetching/fetchers/aws/monitoring_fetcher.go b/internal/resources/fetching/fetchers/aws/monitoring_fetcher.go
@@ -94,6 +94,10 @@ func (r MonitoringResource) GetData() any {
 	return r
 }
 
+func (r MonitoringResource) GetIds() []string {
+	return []string{}
+}
+
 func (r MonitoringResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	id := fmt.Sprintf("cloudtrail-%s", r.identity.Account)
 	return fetching.ResourceMetadata{
@@ -110,6 +114,10 @@ func (s SecurityHubResource) GetData() any {
 	return s
 }
 
+func (s SecurityHubResource) GetIds() []string {
+	return []string{s.GetResourceArn()}
+}
+
 func (s SecurityHubResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return fetching.ResourceMetadata{
 		ID:      s.GetResourceArn(),

diff --git a/internal/resources/fetching/fetchers/aws/network_fetcher.go b/internal/resources/fetching/fetchers/aws/network_fetcher.go
@@ -73,6 +73,10 @@ func (r NetworkResource) GetData() any {
 	return r.AwsResource
 }
 
+func (r NetworkResource) GetIds() []string {
+	return []string{r.GetResourceArn()}
+}
+
 func (r NetworkResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	identifier := r.GetResourceArn()
 	return fetching.ResourceMetadata{

diff --git a/internal/resources/fetching/fetchers/aws/rds_fetcher.go b/internal/resources/fetching/fetchers/aws/rds_fetcher.go
@@ -75,6 +75,10 @@ func (r RdsResource) GetData() any {
 	return r.dbInstance
 }
 
+func (r RdsResource) GetIds() []string {
+	return []string{r.dbInstance.GetResourceArn()}
+}
+
 func (r RdsResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return fetching.ResourceMetadata{
 		ID:      r.dbInstance.GetResourceArn(),

diff --git a/internal/resources/fetching/fetchers/aws/s3_fetcher.go b/internal/resources/fetching/fetchers/aws/s3_fetcher.go
@@ -72,6 +72,10 @@ func (r S3Resource) GetData() any {
 	return r.bucket
 }
 
+func (r S3Resource) GetIds() []string {
+	return []string{r.bucket.GetResourceArn()}
+}
+
 func (r S3Resource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return fetching.ResourceMetadata{
 		ID:      r.bucket.GetResourceArn(),

diff --git a/internal/resources/fetching/fetchers/azure/assets_fetcher.go b/internal/resources/fetching/fetchers/azure/assets_fetcher.go
@@ -174,6 +174,10 @@ func (r *AzureResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
+func (r *AzureResource) GetIds() []string {
+	return []string{r.Asset.Id}
+}
+
 func (r *AzureResource) GetElasticCommonData() (map[string]any, error) {
 	m := map[string]any{}
 

diff --git a/internal/resources/fetching/fetchers/azure/batch_fetcher.go b/internal/resources/fetching/fetchers/azure/batch_fetcher.go
@@ -124,6 +124,12 @@ func (r *AzureBatchResource) GetData() any {
 	return r.Assets
 }
 
+func (r *AzureBatchResource) GetIds() []string {
+	return lo.Map(r.Assets, func(item inventory.AzureAsset, _ int) string {
+		return item.Id
+	})
+}
+
 func (r *AzureBatchResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	// Assuming all batch in not empty includes assets of the same subscription
 	id := fmt.Sprintf("%s-%s", r.SubType, r.Subscription.ShortID)

diff --git a/internal/resources/fetching/fetchers/azure/locations_network_watcher_batch_fetcher.go b/internal/resources/fetching/fetchers/azure/locations_network_watcher_batch_fetcher.go
@@ -115,7 +115,7 @@ type NetworkWatchersBatchedByLocationResource struct {
 }
 
 func (r *NetworkWatchersBatchedByLocationResource) GetMetadata() (fetching.ResourceMetadata, error) {
-	id := fmt.Sprintf("%s-%s-%s", r.SubType, r.Location.Name, r.Subscription.ShortID)
+	id := r.buildId()
 	return fetching.ResourceMetadata{
 		ID:                   id,
 		Name:                 id,
@@ -126,10 +126,19 @@ func (r *NetworkWatchersBatchedByLocationResource) GetMetadata() (fetching.Resou
 	}, nil
 }
 
+func (r *NetworkWatchersBatchedByLocationResource) buildId() string {
+	id := fmt.Sprintf("%s-%s-%s", r.SubType, r.Location.Name, r.Subscription.ShortID)
+	return id
+}
+
 func (r *NetworkWatchersBatchedByLocationResource) GetData() any {
 	return r
 }
 
+func (r *NetworkWatchersBatchedByLocationResource) GetIds() []string {
+	return []string{r.buildId()}
+}
+
 func (r *NetworkWatchersBatchedByLocationResource) GetElasticCommonData() (map[string]any, error) {
 	return nil, nil
 }
diff --git a/internal/resources/fetching/fetchers/gcp/assets_fetcher.go b/internal/resources/fetching/fetchers/gcp/assets_fetcher.go
@@ -131,6 +131,10 @@ func (r *GcpAsset) GetData() any {
 	return r.ExtendedAsset.Asset
 }
 
+func (r *GcpAsset) GetIds() []string {
+	return []string{r.ExtendedAsset.Name}
+}
+
 func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {
 	var region string
 

diff --git a/internal/resources/fetching/fetchers/gcp/log_sink_fetcher.go b/internal/resources/fetching/fetchers/gcp/log_sink_fetcher.go
@@ -82,7 +82,7 @@ func (f *GcpLogSinkFetcher) Stop() {
 }
 
 func (g *GcpLoggingAsset) GetMetadata() (fetching.ResourceMetadata, error) {
-	id := fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+	id := g.buildId()
 	return fetching.ResourceMetadata{
 		ID:                   id,
 		Type:                 g.Type,
@@ -93,10 +93,18 @@ func (g *GcpLoggingAsset) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
+func (g *GcpLoggingAsset) buildId() string {
+	return fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+}
+
 func (g *GcpLoggingAsset) GetData() any {
 	return g.Asset
 }
 
+func (g *GcpLoggingAsset) GetIds() []string {
+	return []string{g.buildId()}
+}
+
 func (g *GcpLoggingAsset) GetElasticCommonData() (map[string]any, error) {
 	return nil, nil
 }
diff --git a/internal/resources/fetching/fetchers/gcp/monitoring_fetcher.go b/internal/resources/fetching/fetchers/gcp/monitoring_fetcher.go
@@ -87,7 +87,7 @@ func (f *GcpMonitoringFetcher) Stop() {
 }
 
 func (g *GcpMonitoringAsset) GetMetadata() (fetching.ResourceMetadata, error) {
-	id := fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+	id := g.buildId()
 	return fetching.ResourceMetadata{
 		ID:                   id,
 		Type:                 g.Type,
@@ -98,10 +98,18 @@ func (g *GcpMonitoringAsset) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
+func (g *GcpMonitoringAsset) buildId() string {
+	return fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+}
+
 func (g *GcpMonitoringAsset) GetData() any {
 	return g.Asset
 }
 
+func (g *GcpMonitoringAsset) GetIds() []string {
+	return []string{g.buildId()}
+}
+
 func (g *GcpMonitoringAsset) GetElasticCommonData() (map[string]any, error) {
 	return nil, nil
 }
diff --git a/internal/resources/fetching/fetchers/gcp/policies_fetcher.go b/internal/resources/fetching/fetchers/gcp/policies_fetcher.go
@@ -81,7 +81,7 @@ func (f *GcpPoliciesFetcher) Stop() {
 }
 
 func (g *GcpPoliciesAsset) GetMetadata() (fetching.ResourceMetadata, error) {
-	id := fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+	id := g.buildId()
 	return fetching.ResourceMetadata{
 		ID:                   id,
 		Type:                 g.Type,
@@ -92,10 +92,19 @@ func (g *GcpPoliciesAsset) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
+func (g *GcpPoliciesAsset) buildId() string {
+	id := fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+	return id
+}
+
 func (g *GcpPoliciesAsset) GetData() any {
 	return g.Asset.Policies
 }
 
+func (g *GcpPoliciesAsset) GetIds() []string {
+	return []string{g.buildId()}
+}
+
 func (g *GcpPoliciesAsset) GetElasticCommonData() (map[string]any, error) {
 	return nil, nil
 }
diff --git a/internal/resources/fetching/fetchers/gcp/service_usage_fetcher.go b/internal/resources/fetching/fetchers/gcp/service_usage_fetcher.go
@@ -82,7 +82,7 @@ func (f *GcpServiceUsageFetcher) Stop() {
 }
 
 func (g *GcpServiceUsageAsset) GetMetadata() (fetching.ResourceMetadata, error) {
-	id := fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+	id := g.buildId()
 	return fetching.ResourceMetadata{
 		ID:                   id,
 		Type:                 g.Type,
@@ -93,10 +93,18 @@ func (g *GcpServiceUsageAsset) GetMetadata() (fetching.ResourceMetadata, error)
 	}, nil
 }
 
+func (g *GcpServiceUsageAsset) buildId() string {
+	return fmt.Sprintf("%s-%s", g.subType, g.Asset.CloudAccount.AccountId)
+}
+
 func (g *GcpServiceUsageAsset) GetData() any {
 	return g.Asset
 }
 
+func (g *GcpServiceUsageAsset) GetIds() []string {
+	return []string{g.buildId()}
+}
+
 func (g *GcpServiceUsageAsset) GetElasticCommonData() (map[string]any, error) {
 	return nil, nil
 }
diff --git a/internal/resources/fetching/fetchers/k8s/file_system_fetcher.go b/internal/resources/fetching/fetchers/k8s/file_system_fetcher.go
@@ -182,6 +182,10 @@ func (r FSResource) GetData() any {
 	return r.EvalResource
 }
 
+func (r FSResource) GetIds() []string {
+	return nil
+}
+
 func (r FSResource) GetElasticCommonData() (map[string]any, error) {
 	m := map[string]any{}
 

diff --git a/internal/resources/fetching/fetchers/k8s/kube_provider.go b/internal/resources/fetching/fetchers/k8s/kube_provider.go
@@ -72,6 +72,10 @@ func (r K8sResource) GetData() any {
 	return r.Data
 }
 
+func (r K8sResource) GetIds() []string {
+	return nil
+}
+
 func (r K8sResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	k8sObj := reflect.Indirect(reflect.ValueOf(r.Data))
 	k8sObjMeta := getK8sObjectMeta(r.log, k8sObj)

diff --git a/internal/resources/fetching/fetchers/k8s/process_fetcher.go b/internal/resources/fetching/fetchers/k8s/process_fetcher.go
@@ -279,6 +279,10 @@ func (res ProcResource) GetData() any {
 	return res.EvalResource
 }
 
+func (res ProcResource) GetIds() []string {
+	return nil
+}
+
 func (res ProcResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return fetching.ResourceMetadata{
 		ID:      res.EvalResource.PID + res.EvalResource.Stat.StartTime,