From 4aef70a379948fc06ea6911a7eda28f702b504b3 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Fri, 28 Jul 2023 14:45:40 -0700 Subject: [PATCH 1/3] add issue platform infra --- docker-compose.yml | 9 +++++++++ install/create-kafka-topics.sh | 2 +- sentry/sentry.conf.example.py | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b7195959a8..6619a2311e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -275,6 +275,9 @@ services: snuba-replays-consumer: <<: *snuba_defaults command: consumer --storage replays --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + snuba-issue-occurrence-consumer: + <<: *snuba_defaults + command: consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset @@ -345,6 +348,9 @@ services: ingest-replay-recordings: <<: *sentry_defaults command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings + ingest-occurrences: + <<: *sentry_defaults + command: run consumer ingest-occurrences --consumer-group ingest-occurrences ingest-profiles: <<: *sentry_defaults command: run consumer --no-strict-offset-reset ingest-profiles --consumer-group ingest-profiles @@ -354,6 +360,9 @@ services: post-process-forwarder-transactions: <<: *sentry_defaults command: run consumer post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group + post-process-forwarder-issue-platform: + <<: *sentry_defaults + command: run consumer post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group generic_events_group subscription-consumer-events: <<: *sentry_defaults command: run consumer events-subscription-results --consumer-group query-subscription-consumer diff --git a/install/create-kafka-topics.sh b/install/create-kafka-topics.sh index 6ce5157f2f..36b73655f9 100644 --- a/install/create-kafka-topics.sh +++ b/install/create-kafka-topics.sh @@ -3,7 +3,7 @@ echo "${_group}Creating additional Kafka topics ..." # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) EXISTING_KAFKA_TOPICS=$($dcr -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) -NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles" +NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles generic-events ingest-occurrences" for topic in $NEEDED_KAFKA_TOPICS; do if ! echo "$EXISTING_KAFKA_TOPICS" | grep -qE "(^| )$topic( |$)"; then $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index c8ae1ff9a8..2988ecf76b 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -269,6 +269,7 @@ def get_internal_network(): "organizations:performance-view", "organizations:advanced-search", "organizations:session-replay", + "organizations:issue-platform", "organizations:profiling", "projects:custom-inbound-filters", "projects:data-forwarding", From fffacaf66702293cc47d96a279ff1cbe779c7a89 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Mon, 31 Jul 2023 13:50:04 -0700 Subject: [PATCH 2/3] use right commit log for issue platform --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6619a2311e..abf71a535a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -362,7 +362,7 @@ services: command: run consumer post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group post-process-forwarder-issue-platform: <<: *sentry_defaults - command: run consumer post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group generic_events_group + command: run consumer post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group subscription-consumer-events: <<: *sentry_defaults command: run consumer events-subscription-results --consumer-group query-subscription-consumer From 8649ba261c36387b8785d874ce7e15607b2f6145 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Mon, 31 Jul 2023 13:50:39 -0700 Subject: [PATCH 3/3] remove generic metrics topic creation, it's done by default --- install/create-kafka-topics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/create-kafka-topics.sh b/install/create-kafka-topics.sh index 36b73655f9..7d8a6dd575 100644 --- a/install/create-kafka-topics.sh +++ b/install/create-kafka-topics.sh @@ -3,7 +3,7 @@ echo "${_group}Creating additional Kafka topics ..." # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) EXISTING_KAFKA_TOPICS=$($dcr -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) -NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles generic-events ingest-occurrences" +NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles ingest-occurrences" for topic in $NEEDED_KAFKA_TOPICS; do if ! echo "$EXISTING_KAFKA_TOPICS" | grep -qE "(^| )$topic( |$)"; then $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092