feat(sources): New amqp (Rabbitmq) Source/Sink

.github/workflows/integration-test.yml

@@ -55,6 +55,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - test: 'amqp'
           - test: 'apex'
           - test: 'aws'
           - test: 'axiom'

Cargo.toml

@@ -198,6 +198,9 @@ prost-types = { version = "0.10.1", default-features = false, optional = true }
 goauth = { version = "0.13.1", optional = true }
 smpl_jwt = { version = "0.7.1", default-features = false, optional = true }
 
+# AMQP
+lapin = { version = "2.1.1", default-features = false, optional = true }
+
 # API
 async-graphql = { version = "4.0.13", default-features = false, optional = true, features = ["chrono"] }
 async-graphql-warp = { version = "4.0.13", default-features = false, optional = true }
@@ -441,6 +444,7 @@ enrichment-tables-geoip = ["dep:maxminddb"]
 # Sources
 sources = ["sources-logs", "sources-metrics"]
 sources-logs = [
+  "sources-amqp",
   "sources-aws_kinesis_firehose",
   "sources-aws_s3",
   "sources-aws_sqs",
@@ -483,6 +487,7 @@ sources-metrics = [
   "sources-vector",
 ]
 
+sources-amqp = ["lapin"]
 sources-apache_metrics = []
 sources-aws_ecs_metrics = []
 sources-aws_kinesis_firehose = ["dep:base64", "dep:infer"]
@@ -576,6 +581,7 @@ transforms-throttle = ["dep:governor"]
 # Sinks
 sinks = ["sinks-logs", "sinks-metrics"]
 sinks-logs = [
+  "sinks-amqp",
   "sinks-apex",
   "sinks-aws_cloudwatch_logs",
   "sinks-aws_kinesis_firehose",
@@ -630,6 +636,7 @@ sinks-metrics = [
   "sinks-splunk_hec"
 ]
 
+sinks-amqp = ["lapin"]
 sinks-apex = []
 sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs"]
 sinks-aws_cloudwatch_metrics = ["aws-core", "dep:aws-sdk-cloudwatch"]
@@ -692,6 +699,7 @@ nightly = []
 
 # Testing-related features
 all-integration-tests = [
+  "amqp-integration-tests",
   "apex-integration-tests",
   "aws-integration-tests",
   "axiom-integration-tests",
@@ -727,6 +735,8 @@ all-integration-tests = [
   "dnstap-integration-tests",
 ]
 
+amqp-integration-tests = ["sources-amqp", "sinks-amqp"]
+
 aws-integration-tests = [
   "aws-cloudwatch-logs-integration-tests",
   "aws-cloudwatch-metrics-integration-tests",
@@ -876,4 +886,3 @@ name = "codecs"
 path = "benches/codecs/main.rs"
 harness = false
 required-features = ["codecs-benches"]
-

Makefile

@@ -334,7 +334,7 @@ test-enterprise: ## Runs enterprise related behavioral tests
 
 .PHONY: test-integration
 test-integration: ## Runs all integration tests
-test-integration: test-integration-apex test-integration-aws test-integration-axiom test-integration-azure test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch
+test-integration: test-integration-amqp test-integration-apex test-integration-aws test-integration-axiom test-integration-azure test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch
 test-integration: test-integration-azure test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch
 test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-humio test-integration-http-scrape test-integration-influxdb
 test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats

scripts/integration/docker-compose.amqp.yml

@@ -0,0 +1,36 @@
+version: "3"
+
+services:
+  rabbitmq:
+    image: docker.io/rabbitmq:3.8
+    ports:
+      - 5672:5672
+  runner:
+    build:
+      context:  ${PWD}
+      dockerfile: scripts/integration/Dockerfile
+      args:
+        - RUST_VERSION=${RUST_VERSION}
+    working_dir: /code
+    command:
+      - "cargo"
+      - "nextest"
+      - "run"
+      - "--no-fail-fast"
+      - "--no-default-features"
+      - "--features"
+      - "amqp-integration-tests"
+      - "--lib"
+      - "${FILTER:-::amqp::}"
+    depends_on:
+      - rabbitmq
+    volumes:
+      - ${PWD}:/code
+      - target:/code/target
+      - cargogit:/usr/local/cargo/git
+      - cargoregistry:/usr/local/cargo/registry
+
+volumes:
+  target: {}
+  cargogit: {}
+  cargoregistry: {}