[SB Pyamqp] stress updates

sdk/eventhub/azure-eventhub/stress/Dockerfile38

@@ -2,7 +2,7 @@
 # public OSS users should simply leave this argument blank or ignore its presence entirely
 ARG REGISTRY="mcr.microsoft.com/mirror/docker/library/"
 FROM ${REGISTRY}python:3.8-slim-buster
-# RUN apt-get -y update && apt-get -y install git
+RUN apt-get -y update && apt-get -y install git
 
 WORKDIR /app
 

sdk/eventhub/azure-eventhub/stress/scenarios-matrix.yaml

@@ -2,27 +2,35 @@ matrix:
   image:
     - Dockerfile38
   scenarios:
-    event-async:
-      testTarget: event-async
-    event-sync:
-      testTarget: event-sync
-    batch-async:
-      testTarget: batch-async
-    batch-sync:
-      testTarget: batch-sync
-    bplistsync:
-      testTarget:  bplistsync
-    bpasync:
-      testTarget:   bpasync
-    bplistasync:
-      testTarget: bplistasync
-    bpsync:
-      testTarget:  bpsync
-    syncwebsockets:
-      testTarget:  syncwebsockets
-    asyncwebsockets:
-      testTarget: asyncwebsockets
-    sync-batch-web:
-      testTarget: sync-batch-web
-    async-batch-web:
-      testTarget: async-batch-web
+    send:
+      testTarget: send
+    consume:
+      testTarget: consume
+    send-async:
+      testTarget: send-async
+    consume-async:
+      testTarget: consume-async
+    # event-async:
+    #   testTarget: event-async
+    # event-sync:
+    #   testTarget: event-sync
+    # batch-async:
+    #   testTarget: batch-async
+    # batch-sync:
+    #   testTarget: batch-sync
+    # bplistsync:
+    #   testTarget:  bplistsync
+    # bpasync:
+    #   testTarget:   bpasync
+    # bplistasync:
+    #   testTarget: bplistasync
+    # bpsync:
+    #   testTarget:  bpsync
+    # syncwebsockets:
+    #   testTarget:  syncwebsockets
+    # asyncwebsockets:
+    #   testTarget: asyncwebsockets
+    # sync-batch-web:
+    #   testTarget: sync-batch-web
+    # async-batch-web:
+    #   testTarget: async-batch-web

sdk/eventhub/azure-eventhub/stress/scripts/dev_requirement.txt

@@ -1,5 +1,5 @@
 psutil
-azure-eventhub
+git+https://github.com/l0lawrence/azure-sdk-for-python.git@sb_to_eh2#subdirectory=sdk/eventhub/azure-eventhub&egg=azure-eventhub
 azure-eventhub-checkpointstoreblob
 azure-eventhub-checkpointstoreblob-aio
 azure-servicebus==7.8.1

sdk/eventhub/azure-eventhub/stress/scripts/logger.py

@@ -22,19 +22,10 @@ def get_base_logger(log_filename, logger_name, level=logging.INFO, print_console
             console_handler.setFormatter(formatter)
             logger.addHandler(console_handler)
 
-    if log_filename:
-        logger_file_handler = RotatingFileHandler(
-            log_filename,
-            maxBytes=log_file_max_bytes,
-            backupCount=log_file_backup_count
-        )
-        logger_file_handler.setFormatter(formatter)
-        logger.addHandler(logger_file_handler)
-
     return logger
 
 
-def get_logger(log_filename, logger_name, level=logging.INFO, print_console=False, log_format=None,
+def get_logger(log_filename, logger_name, level=logging.DEBUG, print_console=False, log_format=None,
                log_file_max_bytes=20 * 1024 * 1024, log_file_backup_count=3):
     stress_logger = logging.getLogger(logger_name)
     stress_logger.setLevel(level)
@@ -44,38 +35,11 @@ def get_logger(log_filename, logger_name, level=logging.INFO, print_console=Fals
     uamqp_logger.setLevel(level)
 
     formatter = log_format or logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
-    if print_console:
-        console_handler = logging.StreamHandler(stream=sys.stdout)
-        console_handler.setFormatter(formatter)
-        if not eventhub_logger.handlers:
-            eventhub_logger.addHandler(console_handler)
-        if not uamqp_logger.handlers:
-            uamqp_logger.addHandler(console_handler)
-        if not stress_logger.handlers:
-            stress_logger.addHandler(console_handler)
-
-    if log_filename:
-        eventhub_file_handler = RotatingFileHandler(
-            "eventhub_" + log_filename,
-            maxBytes=log_file_max_bytes,
-            backupCount=log_file_backup_count
-        )
-        uamqp_file_handler = RotatingFileHandler(
-            "uamqp_" + log_filename,
-            maxBytes=log_file_max_bytes,
-            backupCount=log_file_backup_count
-        )
-        stress_file_handler = RotatingFileHandler(
-            log_filename,
-            maxBytes=log_file_max_bytes,
-            backupCount=log_file_backup_count
-        )
-        eventhub_file_handler.setFormatter(formatter)
-        uamqp_file_handler.setFormatter(formatter)
-        stress_file_handler.setFormatter(formatter)
-        eventhub_logger.addHandler(eventhub_file_handler)
-        uamqp_logger.addHandler(uamqp_file_handler)
-        stress_logger.addHandler(stress_file_handler)
+    console_handler = logging.FileHandler(log_filename)
+    console_handler.setFormatter(formatter)
+    eventhub_logger.addHandler(console_handler)
+    uamqp_logger.addHandler(console_handler)
+    stress_logger.addHandler(console_handler)
 
     return stress_logger
 

sdk/eventhub/azure-eventhub/stress/templates/testjob.yaml

@@ -13,54 +13,66 @@ spec:
         limits:
           memory: "2000Mi"
           cpu: "1"
-
-      {{ if eq .Stress.testTarget "event-async" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200   & python azure_eventhub_consumer_stress_async.py --duration 259200    ']
+      {{ if eq .Stress.testTarget "send" }}
+      command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200 --log_filename $DEBUG_SHARE/output  ']
       {{- end -}}
-
-      {{ if eq .Stress.testTarget "event-sync" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200    & python azure_eventhub_consumer_stress_sync.py --duration 259200   ']
+      {{ if eq .Stress.testTarget "consume" }}
+      command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_consumer_stress_sync.py --duration 259200   --log_filename $DEBUG_SHARE/output   ']
+      {{- end -}} 
+      {{ if eq .Stress.testTarget "send-async" }}
+      command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200 --log_filename $DEBUG_SHARE/output    ']
       {{- end -}}
+      {{ if eq .Stress.testTarget "consume-async" }}
+      command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output &&  python azure_eventhub_consumer_stress_async.py --duration 259200 --log_filename $DEBUG_SHARE/output     ']
+      {{- end -}}   
 
-      {{ if eq .Stress.testTarget "batch-async" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_list_async --duration 259200    & python azure_eventhub_consumer_stress_async.py --duration 259200   ']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "event-async" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_async.py --duration 259200 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "batch-sync" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_list_sync --duration 259200    & python azure_eventhub_consumer_stress_sync.py --duration 259200   ']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "event-sync" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200  --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_sync.py --duration 259200  --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "bplistsync" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_list_sync --duration 259200  --buffered_mode & python azure_eventhub_consumer_stress_sync.py --duration 259200   ']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "batch-async" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_list_async --duration 259200 --log_filename $DEBUG_SHARE/output  --pyamqp_logging_enable & python azure_eventhub_consumer_stress_async.py --duration 259200  --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable ']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "bpasync" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200  --buffered_mode & python azure_eventhub_consumer_stress_async.py --duration 259200']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "batch-sync" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_list_sync --duration 259200  --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_sync.py --duration 259200 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable ']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "bplistasync" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_list_async --duration 259200  --buffered_mode & python azure_eventhub_consumer_stress_async.py --duration 259200   ']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "bplistsync" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_list_sync --duration 259200  --buffered_mode --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_sync.py --duration 259200  --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "bpsync" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200  --buffered_mode & python azure_eventhub_consumer_stress_sync.py --duration 259200']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "bpasync" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200  --buffered_mode --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_async.py --duration 259200 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "syncwebsockets" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200 --transport_type 1 & python azure_eventhub_consumer_stress_sync.py --duration 259200 --transport_type 1']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "bplistasync" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_list_async --duration 259200  --buffered_mode --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_async.py --duration 259200  --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable ']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "asyncwebsockets" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200  --transport_type 1 & python azure_eventhub_consumer_stress_async.py --duration 259200 --transport_type 1']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "bpsync" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200  --buffered_mode --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_sync.py --duration 259200 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "sync-batch-web" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_list_sync --duration 259200 --transport_type 1 & python azure_eventhub_consumer_stress_sync.py --duration 259200 --transport_type 1']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "syncwebsockets" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_sync --duration 259200 --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_sync.py --duration 259200 --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
-      {{ if eq .Stress.testTarget "async-batch-web" }}
-      command: ['bash', '-c', 'python azure_eventhub_producer_stress.py -m stress_send_list_async --duration 259200  --transport_type 1 & python azure_eventhub_consumer_stress_async.py --duration 259200 --transport_type 1']
-      {{- end -}}
+      # {{ if eq .Stress.testTarget "asyncwebsockets" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_async --duration 259200  --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_async.py --duration 259200 --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
+
+      # {{ if eq .Stress.testTarget "sync-batch-web" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_list_sync --duration 259200 --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_sync.py --duration 259200 --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
+
+      # {{ if eq .Stress.testTarget "async-batch-web" }}
+      # command: ['bash', '-c', 'mkdir -p $DEBUG_SHARE && cat > $DEBUG_SHARE/output && python azure_eventhub_producer_stress.py -m stress_send_list_async --duration 259200  --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable & python azure_eventhub_consumer_stress_async.py --duration 259200 --transport_type 1 --log_filename $DEBUG_SHARE/output --pyamqp_logging_enable']
+      # {{- end -}}
 
       {{- include "stress-test-addons.container-env" . | nindent 6 }}
 {{- end -}}

sdk/servicebus/azure-servicebus/stress/.helmignore

@@ -0,0 +1,6 @@
+stress
+stress.exe
+.env
+Dockerfile
+*.py
+*.txt

sdk/servicebus/azure-servicebus/stress/Chart.lock

@@ -2,5 +2,5 @@ dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
   version: 0.2.0
-digest: sha256:59fff3930e78c4ca9f9c0120433c7695d31db63f36ac61d50abcc91b1f1835a0
-generated: "2022-11-19T01:30:02.403917379Z"
+digest: sha256:53cbe4c0fed047f6c611523bd34181b21a310e7a3a21cb14f649bb09e4a77648
+generated: "2023-03-14T09:57:20.6731895-07:00"

sdk/servicebus/azure-servicebus/stress/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-name: python-servicebus-stress-test
+name: py-sb-stress-test
 description: python service bus stress test.
 version: 0.1.2
 appVersion: v0.2
@@ -9,5 +9,5 @@ annotations:
 
 dependencies:
 - name: stress-test-addons
-  version: ~0.2.0
-  repository: https://stresstestcharts.blob.core.windows.net/helm/
+  version: 0.2.0
+  repository: "@stress-test-charts"

sdk/servicebus/azure-servicebus/stress/Dockerfile

@@ -2,10 +2,11 @@
 # public OSS users should simply leave this argument blank or ignore its presence entirely
 ARG REGISTRY="mcr.microsoft.com/mirror/docker/library/"
 FROM ${REGISTRY}python:3.8-slim-buster
+RUN apt-get -y update && apt-get -y install git
 
 WORKDIR /app
 
 COPY ./scripts /app/stress/scripts
 
 WORKDIR /app/stress/scripts
-RUN pip3 install -r dev_requirements.txt
+RUN pip install -r dev_requirements.txt

sdk/servicebus/azure-servicebus/stress/scenarios-matrix.yaml

@@ -1,7 +1,64 @@
-displayNames:
 matrix:
   image:
     - Dockerfile
   scenarios:
-    sbStress:
-      testTarget: servicebus
+    sender:
+      testTarget: sender  
+    receiver:
+      testTarget: receiver
+    iterator:
+      testTarget: iterator
+    senderasync:
+      testTarget: senderasync
+    receiverasync:
+      testTarget: senderasync
+    iteratorasync:
+      testTarget: senderasync
+    iteratorasyncnobatch:
+      testTarget: iteratorasyncnobatch
+    # queue:
+    #   testTarget: queue
+    # aqueue:
+    #  testTarget: aqueue
+    # queuepull:
+    #   testTarget: queuepull
+    # aqueuepull:
+    #  testTarget: aqueuepull
+    # batch:
+    #   testTarget: batch
+    # abatch:
+    #   testTarget: abatch
+    # queuew:
+    #   testTarget: queuew
+    # aqueuew:
+    #  testTarget: aqueuew
+    # queuepullw:
+    #   testTarget: queuepullw
+    # aqueuepullw:
+    #  testTarget: aqueuepullw
+    # batchw:
+    #   testTarget: batchw
+    # abatchw:
+    #   testTarget: abatchw
+    # memray:
+    #   testTarget: memray
+    # sendreceiveslow:
+    #   testTarget: send_receive_slow
+    # receivedelete:
+    #   testTarget: receive_delete
+    # unsettledmessage:
+    #   testTarget: unsettled_message
+    # largebatch:
+    #   testTarget: large_batch
+    # pullreceivetimeout:
+    #   testTarget: pull_receive_timeout
+    # longrenew:
+    #   testTarget: long_renew
+    # queuepeek:
+    #   testTarget: queue_peek
+    # queueclosereopen:
+    #   testTarget: queue_close_reopen
+    # droppedmessages:
+    #   testTarget: dropped_messages
+    #sbStress:
+    #  testTarget: servicebus

sdk/servicebus/azure-servicebus/stress/scripts/dev_requirements.txt

@@ -1,6 +1,8 @@
-aiohttp>=3.0; python_version >= '3.5'
+aiohttp>=3.0
 opencensus-ext-azure
 psutil
-pytest
-azure-servicebus
+
 python-dotenv
+websocket-client
+
+git+https://github.com/kashifkhan/azure-sdk-for-python.git@sb_pyamqp_mem#subdirectory=sdk/servicebus/azure-servicebus&egg=azure-servicebus