diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 8809d87c24..3b4b0a7f87 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -38,11 +38,13 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libssl-dev \ locales \ make \ + odbc-postgresql \ openssl \ python2-dev \ python3-dev \ python3-pip \ - odbc-postgresql \ + tzdata \ + unixodbc-dev \ unzip \ wget \ zip \ @@ -58,6 +60,9 @@ RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psq # Set the locale RUN locale-gen --no-purge en_US.UTF-8 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 +ENV TZ="Etc/UTC" +RUN ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata # Use root user ENV HOME /root diff --git a/.github/containers/requirements.txt b/.github/containers/requirements.txt index 54f3c39d00..27fa6624b1 100644 --- a/.github/containers/requirements.txt +++ b/.github/containers/requirements.txt @@ -1,5 +1,5 @@ pip setuptools wheel -virtualenv<20.22.1 +virtualenv<20.22.0 tox \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5ab093f19..f8e5182434 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,10 +36,9 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - # - gearman + - gearman - grpc #- kafka - - libcurl - memcached - mongodb - mysql @@ -61,6 +60,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -117,48 +117,19 @@ jobs: ] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - name: Get Environments - id: get-envs + - name: Fetch git tags run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 - - grpc: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -169,7 +140,7 @@ jobs: - name: Test run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto env: TOX_PARALLEL_NO_SPINNER: 1 PY_COLORS: 0 @@ -181,7 +152,7 @@ jobs: path: ./**/.coverage.* retention-days: 1 - libcurl: + grpc: env: TOTAL_GROUPS: 1 @@ -191,17 +162,19 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - # Special case packages - - name: Install libcurl-dev + - name: Fetch git tags run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -212,7 +185,7 @@ jobs: - name: Test run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + tox -vv -e ${{ steps.get-envs.outputs.envs }} env: TOX_PARALLEL_NO_SPINNER: 1 PY_COLORS: 0 @@ -234,6 +207,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -253,15 +230,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - name: Install odbc driver for postgresql + - name: Fetch git tags run: | - sudo apt-get update - sudo sudo apt-get install odbc-postgresql - sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini - sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini - sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -294,6 +267,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -316,7 +293,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -349,6 +330,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -366,7 +351,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -399,6 +388,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -418,7 +411,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -451,6 +448,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -468,7 +469,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -501,6 +506,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -519,7 +528,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -542,77 +555,85 @@ jobs: path: ./**/.coverage.* retention-days: 1 - #kafka: - # env: - # TOTAL_GROUPS: 4 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1, 2, 3, 4] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # zookeeper: - # image: bitnami/zookeeper:3.7 - # env: - # ALLOW_ANONYMOUS_LOGIN: yes - - # ports: - # - 2181:2181 - - # kafka: - # image: bitnami/kafka:3.2 - # ports: - # - 8080:8080 - # - 8081:8081 - # env: - # ALLOW_PLAINTEXT_LISTENER: yes - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true - # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 - # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, - # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT - # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 - - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + # kafka: + # env: + # TOTAL_GROUPS: 4 + + # strategy: + # fail-fast: false + # matrix: + # group-number: [1, 2, 3, 4] + + # runs-on: ubuntu-20.04 + # container: + # image: ghcr.io/${{ github.repository }}-ci:latest + # options: >- + # --add-host=host.docker.internal:host-gateway + # timeout-minutes: 30 + + # services: + # zookeeper: + # image: bitnami/zookeeper:3.7 + # env: + # ALLOW_ANONYMOUS_LOGIN: yes + + # ports: + # - 2181:2181 + + # kafka: + # image: bitnami/kafka:3.2 + # ports: + # - 8080:8080 + # - 8081:8081 + # env: + # ALLOW_PLAINTEXT_LISTENER: yes + # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 + # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, + # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT + # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 + + # steps: + # - uses: actions/checkout@v3 + + # - name: Fetch git tags + # run: | + # git config --global --add safe.directory "$GITHUB_WORKSPACE" + # git fetch --tags origin + + # # Special case packages + # - name: Install librdkafka-dev + # run: | + # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name + # sudo apt-get update + # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + # sudo apt-get install -y wget gnupg2 software-properties-common + # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - + # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" + # sudo apt-get update + # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 mongodb: env: @@ -624,6 +645,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -641,7 +666,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -674,10 +703,14 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: - es07: + elasticsearch: image: elasticsearch:7.17.8 env: "discovery.type": "single-node" @@ -693,7 +726,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -726,10 +763,14 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: - es08: + elasticsearch: image: elasticsearch:8.6.0 env: "xpack.security.enabled": "false" @@ -746,7 +787,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -769,51 +814,59 @@ jobs: path: ./**/.coverage.* retention-days: 1 - # gearman: - # env: - # TOTAL_GROUPS: 1 + gearman: + env: + TOTAL_GROUPS: 1 - # strategy: - # fail-fast: false - # matrix: - # group-number: [1] + strategy: + fail-fast: false + matrix: + group-number: [1] - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 + runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 - # services: - # gearman: - # image: artefactual/gearmand - # ports: - # - 4730:4730 - # # Set health checks to wait until gearman has started - # options: >- - # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 + services: + gearman: + image: artefactual/gearmand + ports: + - 8080:4730 + # Set health checks to wait until gearman has started + options: >- + --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix + steps: + - uses: actions/checkout@v3 - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index 5c3961b026..df95db0290 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3029,21 +3029,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - # _process_module_definition( - # "gearman.client", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_client", - # ) - # _process_module_definition( - # "gearman.connection_manager", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_connection_manager", - # ) - # _process_module_definition( - # "gearman.worker", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_worker", - # ) + _process_module_definition( + "gearman.client", + "newrelic.hooks.application_gearman", + "instrument_gearman_client", + ) + _process_module_definition( + "gearman.connection_manager", + "newrelic.hooks.application_gearman", + "instrument_gearman_connection_manager", + ) + _process_module_definition( + "gearman.worker", + "newrelic.hooks.application_gearman", + "instrument_gearman_worker", + ) _process_module_definition( "botocore.endpoint", diff --git a/tests/application_gearman/test_gearman.py b/tests/application_gearman/test_gearman.py index 7ddc13fdc3..5dda4ef47e 100644 --- a/tests/application_gearman/test_gearman.py +++ b/tests/application_gearman/test_gearman.py @@ -20,14 +20,16 @@ import gearman from newrelic.api.background_task import background_task +from testing_support.db_settings import gearman_settings worker_thread = None worker_event = threading.Event() gm_client = None -GEARMAND_HOST = os.environ.get("GEARMAND_PORT_4730_TCP_ADDR", "localhost") -GEARMAND_PORT = os.environ.get("GEARMAND_PORT_4730_TCP_PORT", "4730") +GEARMAND_SETTINGS = gearman_settings()[0] +GEARMAND_HOST = GEARMAND_SETTINGS["host"] +GEARMAND_PORT = GEARMAND_SETTINGS["port"] GEARMAND_ADDR = "%s:%s" % (GEARMAND_HOST, GEARMAND_PORT) diff --git a/tests/datastore_elasticsearch/test_connection.py b/tests/datastore_elasticsearch/test_connection.py index 2e888af9b5..9e8f17b4c1 100644 --- a/tests/datastore_elasticsearch/test_connection.py +++ b/tests/datastore_elasticsearch/test_connection.py @@ -36,7 +36,7 @@ def test_connection_default(): else: conn = Connection(**HOST) - assert conn._nr_host_port == ("localhost", ES_SETTINGS["port"]) + assert conn._nr_host_port == (ES_SETTINGS["host"], ES_SETTINGS["port"]) @SKIP_IF_V7 diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index c7c35935f8..bda3180622 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -29,25 +29,15 @@ def postgresql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "postgres" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 5432 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "localhost", - "port": base_port + instance_num, + "user": "postgres", + "password": "postgres", + "name": "postgres", + "host": host, + "port": 8080 + instance_num, "table_name": "postgres_table_" + str(os.getpid()), } for instance_num in range(instances) @@ -66,25 +56,15 @@ def mysql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "python_agent" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 3306 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 1 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "127.0.0.1", - "port": base_port + instance_num, + "user": "python_agent", + "password": "python_agent", + "name": "python_agent", + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -103,17 +83,12 @@ def redis_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 6379 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] @@ -131,17 +106,12 @@ def memcached_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 11211 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -160,15 +130,10 @@ def mongodb_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 27017 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ - {"host": "127.0.0.1", "port": base_port + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} + {"host": host, "port": 8080 + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} for instance_num in range(instances) ] return settings @@ -185,17 +150,12 @@ def elasticsearch_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9200 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": str(base_port + instance_num), + "host": host, + "port": str(8080 + instance_num), "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -214,17 +174,12 @@ def solr_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 8983 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -243,13 +198,12 @@ def rabbitmq_settings(): 2. Github Actions """ + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 1 - base_port = 5672 - settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 5672 + instance_num, } for instance_num in range(instances) ] @@ -267,17 +221,35 @@ def kafka_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9092 + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 + settings = [ + { + "host": host, + "port": 8080 + instance_num, + } + for instance_num in range(instances) + ] + return settings + +def gearman_settings(): + """Return a list of dict of settings for connecting to kafka. + + Will return the correct settings, depending on which of the environments it + is running in. It attempts to set variables in the following order, where + later environments override earlier ones. + + 1. Local + 2. Github Actions + """ + + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 1 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] diff --git a/tests/testing_support/validators/validate_tt_collector_json.py b/tests/testing_support/validators/validate_tt_collector_json.py index 85e3932806..28c9e93a39 100644 --- a/tests/testing_support/validators/validate_tt_collector_json.py +++ b/tests/testing_support/validators/validate_tt_collector_json.py @@ -135,7 +135,7 @@ def _check_params_and_start_time(node): if segment_name.startswith("Datastore"): for key in datastore_params: assert key in params, key - assert params[key] == datastore_params[key] + assert params[key] == datastore_params[key], "Expected %s. Got %s." % (datastore_params[key], params[key]) for key in datastore_forgone_params: assert key not in params, key diff --git a/tox.ini b/tox.ini index 58f3815bac..0ac7331298 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ ; framework_aiohttp-aiohttp01: aiohttp<2 ; framework_aiohttp-aiohttp0202: aiohttp<2.3 ; 3. Python version required. Uses the standard tox definitions. (https://tox.readthedocs.io/en/latest/config.html#tox-environments) -; Examples: py27,py37,py38,py39,pypy,pypy37 +; Examples: py27,py37,py38,py39,pypy27,pypy37 ; 4. Library and version (Optional). Used when testing multiple versions of the library, and may be omitted when only testing a single version. ; Versions should be specified with 2 digits per version number, so <3 becomes 02 and <3.5 becomes 0304. latest and master are also acceptable versions. ; Examples: uvicorn03, CherryPy0302, uvicornlatest @@ -43,10 +43,10 @@ requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py37,py38,py310,py311}, + python-adapter_gevent-{py27,py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -55,81 +55,94 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy37}-without_extensions, + python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy27,pypy37}-without_extensions, + python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy37}-without_extensions, - python-application_celery-{py37,py38,py39,py310,py311,pypy37}, + python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy27,pypy37}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + gearman-application_gearman-{py27,pypy27}, + python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, + python-component_tastypie-{py27,pypy27}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, + mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, - postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, - memcached-datastore_pylibmc-{py37}, - memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, - mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, - solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, - redis-datastore_redis-{py37,py38,pypy37}-redis03, + postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest + postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest + memcached-datastore_pylibmc-{py27,py37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, - redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, - python-external_boto3-{py37,py38,py39,py310,py311}-boto01, + solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_http-{py37,py38,py39,py310,py311}, - python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, - python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, + python-external_feedparser-py27-feedparser{05,06}, + python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py37,py38,py39,py310,py311,pypy37}, - python-external_urllib3-{py37}-urllib3{0109}, - python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-py27-bottle{0008,0009,0010}, + python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, + python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{py37}-Django0108, + python-framework_django-{pypy27,py27}-Django0103, + python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27}-flask0012, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, + grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{py38}-Pyramid0104, - python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, + python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -137,26 +150,27 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, - libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, + python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py37,py311}-genshilatest, - python-template_mako-{py37,py310,py311}, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py27,py37,py311}-genshilatest + python-template_mako-{py27,py37,py310,py311} [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py27,pypy27}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -187,8 +201,8 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 application_celery: celery<6.0 - application_celery-py{py37,37}: importlib-metadata<5.0 - ; application_gearman: gearman<3.0.0 + application_celery-{py37,pypy37}: importlib-metadata<5.0 + application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -205,7 +219,7 @@ deps = component_graphqlserver: markupsafe<2.1 component_graphqlserver: jinja2<3.1 component_tastypie-tastypie0143: django-tastypie<0.14.4 - component_tastypie-{py27,pypy}-tastypie0143: django<1.12 + component_tastypie-{py27,pypy27}-tastypie0143: django<1.12 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie @@ -237,10 +251,9 @@ deps = datastore_redis-redislatest: redis datastore_redis-redis0400: redis<4.1 datastore_redis-redis03: redis<4.0 - datastore_redis-{py27,pypy}: rb + datastore_redis-{py27,pypy27}: rb datastore_aioredis-redislatest: redis datastore_aioredis-aioredislatest: aioredis - datastore_aioredis-aioredis01: aioredis<2 datastore_aredis-aredislatest: aredis datastore_solrpy-solrpy00: solrpy<1.0 datastore_solrpy-solrpy01: solrpy<2.0 @@ -355,7 +368,7 @@ deps = messagebroker_pika-pika0.13: pika<0.14 messagebroker_pika-pikalatest: pika messagebroker_pika: tornado<5 - messagebroker_pika-{py27,pypy}: enum34 + messagebroker_pika-{py27,pypy27}: enum34 messagebroker_confluentkafka-confluentkafkalatest: confluent-kafka messagebroker_confluentkafka-confluentkafka0108: confluent-kafka<1.9 messagebroker_confluentkafka-confluentkafka0107: confluent-kafka<1.8 @@ -376,9 +389,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - libcurl: PYCURL_SSL_LIBRARY=openssl - libcurl: LDFLAGS=-L/usr/local/opt/openssl/lib - libcurl: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado: PYCURL_SSL_LIBRARY=openssl + framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -393,7 +406,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* @@ -416,7 +429,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - ; application_gearman: tests/application_gearman + application_gearman: tests/application_gearman component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver @@ -489,6 +502,7 @@ source = newrelic source = newrelic/ .tox/**/site-packages/newrelic/ + /__w/**/site-packages/newrelic/ [coverage:html] directory = ${TOX_ENV_DIR-.}/htmlcov