Skip to content

Commit

Permalink
modularize the antithesis docker compose (#338) (#341)
Browse files Browse the repository at this point in the history
  • Loading branch information
agavra authored Sep 5, 2024
1 parent d311fb1 commit 77020a3
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 94 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/github-e2e-manual.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
permissions:
id-token: write # This is required for requesting the JWT
contents: write # This is required for actions/checkout (read) and publishing tags (write)
uses: responsivedev/responsive-pub/.github/workflows/github-e2e.yaml@main
uses: ./.github/workflows/github-e2e.yaml
with:
test-duration: ${{ inputs.test-duration }}
secrets: inherit
2 changes: 2 additions & 0 deletions .github/workflows/github-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,11 @@ jobs:
working-directory: ./kafka-client-examples/e2e-test/docker/antithesis
env:
KAFKA_CLIENT_VERSION: latest
BACKEND: scylla
CONFIG_IMG_VERSION: ${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }}
run: |
cat config/docker-compose.yml.template | envsubst > config/docker-compose.yml
cp config/volumes/app-$BACKEND.properties config/volumes/app.properties
docker build . -t us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test-config:$CONFIG_IMG_VERSION
- name: "Push Config"
Expand Down
2 changes: 2 additions & 0 deletions kafka-client-examples/e2e-test/docker/antithesis/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from scratch
COPY config/docker-compose.yml /docker-compose.yml
COPY config/docker-compose-base.yml /docker-compose-base.yml
COPY config/docker-compose-scylla.yml /docker-compose-scylla.yml
COPY config/volumes/scylla.yaml volumes/scylla.yaml
COPY config/volumes/app.properties /volumes/app.properties
COPY config/volumes/log4j2.properties /volumes/log4j2.properties
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
version: '3'
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.3.0
container_name: zookeeper
hostname: zookeeper
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
volumes:
- ./volumes/zookeeper:/volumes/zookeeper:rw
- ./volumes/zookeeper/log:/volumes/zookeeper/log:rw
- ./volumes/zookeeper/data:/volumes/zookeeper/data:rw
networks:
backend:
ipv4_address: 10.0.0.12

broker:
image: confluentinc/cp-kafka:7.3.0
container_name: broker
hostname: broker
ports:
# To learn about configuring Kafka for access across networks see
# https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/
- "9092:9092"
depends_on:
- zookeeper
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_INTERNAL://broker:29092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000
volumes:
- ./volumes/kafka:/volumes/kafka:rw
- ./volumes/kafka/data:/volumes/kafka/data:rw
networks:
backend:
ipv4_address: 10.0.0.20

networks:
backend:
driver: bridge
ipam:
config:
- subnet: 10.0.0.0/24
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
version: '3'
include:
- docker-compose-base.yml
services:
scylla:
image: scylladb/scylla:5.4
container_name: scylla
hostname: scylla
command: --smp 1 --options-file /etc/scylla/scylla.yaml --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --seeds 10.0.0.30 --broadcast-address 10.0.0.30
ports:
- "9042:9042"
volumes:
- ./volumes/scylla/data:/var/lib/scylla/data:rw
- ./volumes/scylla.yaml:/etc/scylla/scylla.yaml
networks:
backend:
ipv4_address: 10.0.0.30

scylla2:
image: scylladb/scylla:5.4
container_name: scylla2
hostname: scylla2
command: --smp 1 --options-file /etc/scylla/scylla.yaml --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --seeds 10.0.0.30 --broadcast-address 10.0.0.31
ports:
- "9043:9042"
volumes:
- ./volumes/scylla2/data:/var/lib/scylla/data:rw
- ./volumes/scylla.yaml:/etc/scylla/scylla.yaml
networks:
backend:
ipv4_address: 10.0.0.31

scylla3:
image: scylladb/scylla:5.4
container_name: scylla3
hostname: scylla3
command: --smp 1 --options-file /etc/scylla/scylla.yaml --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --seeds 10.0.0.30 --broadcast-address 10.0.0.32
ports:
- "9044:9042"
volumes:
- ./volumes/scylla3/data:/var/lib/scylla/data:rw
- ./volumes/scylla.yaml:/etc/scylla/scylla.yaml
networks:
backend:
ipv4_address: 10.0.0.32
Original file line number Diff line number Diff line change
@@ -1,95 +1,14 @@
version: '3'
include:
- docker-compose-${BACKEND}.yml
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.3.0
container_name: zookeeper
hostname: zookeeper
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
volumes:
- ./volumes/zookeeper:/volumes/zookeeper:rw
- ./volumes/zookeeper/log:/volumes/zookeeper/log:rw
- ./volumes/zookeeper/data:/volumes/zookeeper/data:rw
networks:
backend:
ipv4_address: 10.0.0.12

broker:
image: confluentinc/cp-kafka:7.3.0
container_name: broker
hostname: broker
ports:
# To learn about configuring Kafka for access across networks see
# https://www.confluent.io/blog/kafka-client-cannot-connect-to-broker-on-aws-on-docker-etc/
- "9092:9092"
depends_on:
- zookeeper
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_INTERNAL://broker:29092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000
volumes:
- ./volumes/kafka:/volumes/kafka:rw
- ./volumes/kafka/data:/volumes/kafka/data:rw
networks:
backend:
ipv4_address: 10.0.0.20

scylla:
image: scylladb/scylla:5.4
container_name: scylla
hostname: scylla
command: --smp 1 --options-file /etc/scylla/scylla.yaml --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --seeds 10.0.0.30 --broadcast-address 10.0.0.30
ports:
- "9042:9042"
volumes:
- ./volumes/scylla/data:/var/lib/scylla/data:rw
- ./volumes/scylla.yaml:/etc/scylla/scylla.yaml
networks:
backend:
ipv4_address: 10.0.0.30

scylla2:
image: scylladb/scylla:5.4
container_name: scylla2
hostname: scylla2
command: --smp 1 --options-file /etc/scylla/scylla.yaml --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --seeds 10.0.0.30 --broadcast-address 10.0.0.31
ports:
- "9043:9042"
volumes:
- ./volumes/scylla2/data:/var/lib/scylla/data:rw
- ./volumes/scylla.yaml:/etc/scylla/scylla.yaml
networks:
backend:
ipv4_address: 10.0.0.31

scylla3:
image: scylladb/scylla:5.4
container_name: scylla3
hostname: scylla3
command: --smp 1 --options-file /etc/scylla/scylla.yaml --memory 750M --overprovisioned 1 --api-address 0.0.0.0 --seeds 10.0.0.30 --broadcast-address 10.0.0.32
ports:
- "9044:9042"
volumes:
- ./volumes/scylla3/data:/var/lib/scylla/data:rw
- ./volumes/scylla.yaml:/etc/scylla/scylla.yaml
networks:
backend:
ipv4_address: 10.0.0.32

application1:
image: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test:${KAFKA_CLIENT_VERSION}
container_name: application1
hostname: application1
depends_on:
- broker
- scylla
- ${BACKEND}
environment:
NAME: e2e
INPUT_TOPIC: input
Expand All @@ -113,7 +32,7 @@ services:
hostname: application2
depends_on:
- broker
- scylla
- ${BACKEND}
environment:
NAME: e2e
INPUT_TOPIC: input
Expand All @@ -137,7 +56,7 @@ services:
hostname: driver
depends_on:
- broker
- scylla
- ${BACKEND}
environment:
NAME: e2e
INPUT_TOPIC: input
Expand All @@ -152,10 +71,3 @@ services:
networks:
backend:
ipv4_address: 10.0.0.50

networks:
backend:
driver: bridge
ipam:
config:
- subnet: 10.0.0.0/24

0 comments on commit 77020a3

Please sign in to comment.