Fix localstack integration test

elastic · Jul 10, 2023 · 4536076 · 4536076
1 parent 5a81733
commit 4536076
Show file tree

Hide file tree

Showing 10 changed files with 278 additions and 575 deletions.
diff --git a/x-pack/filebeat/docker-compose.yml b/x-pack/filebeat/docker-compose.yml
@@ -7,6 +7,7 @@ services:
     depends_on:
       elasticsearch: { condition: service_healthy }
       cometd: { condition: service_healthy }
+      localstack: { condition: service_healthy }
 
   elasticsearch:
     extends:
@@ -26,13 +27,12 @@ services:
       - 8080:8080
 
   localstack:
-    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
     image: localstack/localstack
     hostname: localhost
     ports:
       - "4566:4566"            # LocalStack Gateway
     environment:
-      - SERVICES=sqs,sns,secretsmanager
+      - SERVICES=s3,sqs,sns,secretsmanager
       - DEBUG=1
       - DOCKER_HOST=unix:///var/run/docker.sock
       - HOST_TMP_FOLDER=${TMPDIR}

diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/localstack.tf b/x-pack/filebeat/input/awss3/_meta/terraform/localstack.tf
@@ -0,0 +1,90 @@
+provider "aws" {
+  alias = "localstack"
+  access_key                  = "bharat"
+  secret_key                  = "bharat"
+  region                      = "us-east-1"
+  s3_use_path_style           = true
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+//  s3_force_path_style = true
+
+  endpoints {
+    apigateway     = "http://localhost:4566"
+    apigatewayv2   = "http://localhost:4566"
+    cloudformation = "http://localhost:4566"
+    cloudwatch     = "http://localhost:4566"
+    dynamodb       = "http://localhost:4566"
+    ec2            = "http://localhost:4566"
+    es             = "http://localhost:4566"
+    elasticache    = "http://localhost:4566"
+    firehose       = "http://localhost:4566"
+    iam            = "http://localhost:4566"
+    kinesis        = "http://localhost:4566"
+    lambda         = "http://localhost:4566"
+    rds            = "http://localhost:4566"
+    redshift       = "http://localhost:4566"
+    route53        = "http://localhost:4566"
+    s3             = "http://localhost:4566"
+    secretsmanager = "http://localhost:4566"
+    ses            = "http://localhost:4566"
+    sns            = "http://localhost:4566"
+    sqs            = "http://localhost:4566"
+    ssm            = "http://localhost:4566"
+    stepfunctions  = "http://localhost:4566"
+    sts            = "http://localhost:4566"
+  }
+}
+
+resource "random_string" "random_localstack" {
+  length  = 6
+  special = false
+  upper   = false
+}
+
+resource "aws_s3_bucket" "filebeat-integtest-localstack" {
+  provider = aws.localstack
+  bucket        = "filebeat-s3-integtest-localstack-${random_string.random_localstack.result}"
+  force_destroy = true
+}
+
+resource "aws_sqs_queue" "filebeat-integtest-localstack" {
+  provider = aws.localstack
+  name   = "filebeat-sqs-integtest-localstack-${random_string.random_localstack.result}"
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Id": "sqspolicy",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:SendMessage",
+      "Resource": "arn:aws:sqs:*:*:filebeat-sqs-integtest-localstack-${random_string.random_localstack.result}",
+      "Condition": {
+        "ArnEquals": { "aws:SourceArn": "${aws_s3_bucket.filebeat-integtest-localstack.arn}" }
+      }
+    }
+  ]
+}
+POLICY
+
+  depends_on = [
+    aws_s3_bucket.filebeat-integtest-localstack,
+  ]
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification-localstack" {
+  provider = aws.localstack
+  bucket = aws_s3_bucket.filebeat-integtest-localstack.id
+
+  queue {
+    queue_arn = aws_sqs_queue.filebeat-integtest-localstack.arn
+    events    = ["s3:ObjectCreated:*"]
+  }
+
+  depends_on = [
+    aws_s3_bucket.filebeat-integtest-localstack,
+    aws_sqs_queue.filebeat-integtest-localstack,
+  ]
+}
diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/main.tf b/x-pack/filebeat/input/awss3/_meta/terraform/main.tf
@@ -8,38 +8,15 @@ terraform {
 }
 
 provider "aws" {
-  access_key                  = "bharat"
-  secret_key                  = "bharat"
-  region                      = "us-east-1"
-  s3_use_path_style           = true
-  skip_credentials_validation = true
-  skip_metadata_api_check     = true
-  skip_requesting_account_id  = true
-
-  endpoints {
-    apigateway     = "http://localhost:4566"
-    apigatewayv2   = "http://localhost:4566"
-    cloudformation = "http://localhost:4566"
-    cloudwatch     = "http://localhost:4566"
-    dynamodb       = "http://localhost:4566"
-    ec2            = "http://localhost:4566"
-    es             = "http://localhost:4566"
-    elasticache    = "http://localhost:4566"
-    firehose       = "http://localhost:4566"
-    iam            = "http://localhost:4566"
-    kinesis        = "http://localhost:4566"
-    lambda         = "http://localhost:4566"
-    rds            = "http://localhost:4566"
-    redshift       = "http://localhost:4566"
-    route53        = "http://localhost:4566"
-    s3             = "http://localhost:4566"
-    secretsmanager = "http://localhost:4566"
-    ses            = "http://localhost:4566"
-    sns            = "http://localhost:4566"
-    sqs            = "http://localhost:4566"
-    ssm            = "http://localhost:4566"
-    stepfunctions  = "http://localhost:4566"
-    sts            = "http://localhost:4566"
+  region = var.aws_region
+  default_tags {
+    tags = {
+      environment  = var.ENVIRONMENT
+      repo         = var.REPO
+      branch       = var.BRANCH
+      build        = var.BUILD_ID
+      created_date = var.CREATED_DATE
+    }
   }
 }
 

diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/outputs.tf b/x-pack/filebeat/input/awss3/_meta/terraform/outputs.tf
@@ -9,3 +9,14 @@ resource "local_file" "secrets" {
   filename        = "${path.module}/outputs.yml"
   file_permission = "0644"
 }
+
+resource "local_file" "secrets-localstack" {
+  content = yamlencode({
+    "queue_url" : aws_sqs_queue.filebeat-integtest-localstack.url
+    "aws_region" : aws_s3_bucket.filebeat-integtest-localstack.region
+    "bucket_name" : aws_s3_bucket.filebeat-integtest-localstack.id
+  })
+  filename        = "${path.module}/outputs-localstack.yml"
+  file_permission = "0644"
+}
+
diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go
@@ -100,7 +100,7 @@ func (c *config) Validate() error {
 	}
 
 	if c.AWSConfig.FIPSEnabled && c.NonAWSBucketName != "" {
-		return errors.New("fips_enabled cannot be used with a non-AWS S3 bucket.")
+		return errors.New("fips_enabled cannot be used with a non-AWS S3 bucket")
 	}
 	if c.PathStyle && c.NonAWSBucketName == "" {
 		return errors.New("path_style can only be used when polling non-AWS S3 services")

diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go
@@ -248,34 +248,29 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli
 		if in.config.AWSConfig.FIPSEnabled {
 			o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled
 		}
-		o.UsePathStyle = true
+		o.UsePathStyle = in.config.PathStyle
 	})
 
-	// regionName, err := getRegionForBucket(cancelCtx, s3Client, bucketName)
-	// if err != nil {
-	// 	return nil, fmt.Errorf("failed to get AWS region for bucket: %w", err)
-	// }
-
-	// originalAwsConfigRegion := in.awsConfig.Region
-
-	// ctx.Logger.Errorf("**ORIGINAL*** %s", originalAwsConfigRegion)
-
-	// in.awsConfig.Region = regionName
+	regionName, err := getRegionForBucket(cancelCtx, s3Client, bucketName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get AWS region for bucket: %w", err)
+	}
 
-	// ctx.Logger.Errorf("***** %s", regionName)
+	originalAwsConfigRegion := in.awsConfig.Region
+	in.awsConfig.Region = regionName
 
-	// if regionName != originalAwsConfigRegion {
-	// 	s3Client = s3.NewFromConfig(in.awsConfig, func(o *s3.Options) {
-	// 		if in.config.NonAWSBucketName != "" {
-	// 			o.EndpointResolver = nonAWSBucketResolver{endpoint: in.config.AWSConfig.Endpoint}
-	// 		}
+	if regionName != originalAwsConfigRegion {
+		s3Client = s3.NewFromConfig(in.awsConfig, func(o *s3.Options) {
+			if in.config.NonAWSBucketName != "" {
+				o.EndpointResolver = nonAWSBucketResolver{endpoint: in.config.AWSConfig.Endpoint}
+			}
 
-	// 		if in.config.AWSConfig.FIPSEnabled {
-	// 			o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled
-	// 		}
-	// 		o.UsePathStyle = true
-	// 	})
-	// }
+			if in.config.AWSConfig.FIPSEnabled {
+				o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled
+			}
+			o.UsePathStyle = in.config.PathStyle
+		})
+	}
 
 	s3API := &awsS3API{
 		client: s3Client,