From 89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 11:57:11 -0700 Subject: [PATCH 01/25] Remove non-root user --- .github/containers/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 65fe441047..8809d87c24 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -59,11 +59,8 @@ RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psq RUN locale-gen --no-purge en_US.UTF-8 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 -# Set user to non-root -ENV HOME /home/github -RUN groupadd -g 1000 github && \ - useradd -m -d "${HOME}" -s /bin/bash -u 1000 -g 1000 github -USER 1000:1000 +# Use root user +ENV HOME /root WORKDIR "${HOME}" # Install pyenv From e430abb4259b13d5389d62af3ca357cd93cf92e7 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 12:38:14 -0700 Subject: [PATCH 02/25] Test new CI image --- .github/workflows/tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fbfae2e552..b91ddbd4b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -117,7 +117,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 steps: @@ -155,7 +155,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 steps: @@ -193,7 +193,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -246,7 +246,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -302,7 +302,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -353,7 +353,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -406,7 +406,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -457,7 +457,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -581,7 +581,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -632,7 +632,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -685,7 +685,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -739,7 +739,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: From 4fffb311216bd710ca2c3557fcf3bbc7562e9771 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 12:47:22 -0700 Subject: [PATCH 03/25] Change pypy to pypy27 in tox. --- tox.ini | 88 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/tox.ini b/tox.ini index 0703d39968..d3ad596c85 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ ; framework_aiohttp-aiohttp01: aiohttp<2 ; framework_aiohttp-aiohttp0202: aiohttp<2.3 ; 3. Python version required. Uses the standard tox definitions. (https://tox.readthedocs.io/en/latest/config.html#tox-environments) -; Examples: py27,py37,py38,py39,pypy,pypy37 +; Examples: py27,py37,py38,py39,pypy27,pypy37 ; 4. Library and version (Optional). Used when testing multiple versions of the library, and may be omitted when only testing a single version. ; Versions should be specified with 2 digits per version number, so <3 becomes 02 and <3.5 becomes 0304. latest and master are also acceptable versions. ; Examples: uvicorn03, CherryPy0302, uvicornlatest @@ -55,94 +55,94 @@ envlist = python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy,pypy37}-without_extensions, + python-agent_features-{pypy27,pypy37}-without_extensions, python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy,pypy37}-without_extensions, - python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - gearman-application_gearman-{py27,pypy}, + python-agent_unittests-{pypy27,pypy37}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + gearman-application_gearman-{py27,pypy27}, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, - python-component_flask_rest-{py27,pypy}-flaskrestx051, + python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, - python-component_tastypie-{py27,pypy}-tastypie0143, + python-component_tastypie-{py27,pypy27}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-cross_agent-pypy-without_extensions, + python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{pypy,py27,py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-memcached01, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest - postgres-datastore_psycopg2cffi-{py27,pypy,py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest memcached-datastore_pylibmc-{py27,py37}, - memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy,pypy37}-pymongo04, - mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - redis-datastore_redis-{py27,py37,py38,pypy,pypy37}-redis03, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - solr-datastore_solrpy-{py27,pypy}-solrpy{00,01}, - python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, python-external_feedparser-py27-feedparser{05,06}, - python-external_http-{py27,py37,py38,py39,py310,py311,pypy}, - python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_urllib3-{py27,py37,pypy}-urllib3{0109}, - python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-urllib3latest, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, python-framework_bottle-py27-bottle{0008,0009,0010}, python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, - python-framework_bottle-pypy-bottle{0008,0009,0010,0011,0012}, + python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{pypy,py27}-Django0103, - python-framework_django-{pypy,py27,py37}-Django0108, + python-framework_django-{pypy27,py27}-Django0103, + python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py27,py37,py38,py39,pypy,pypy37}-falcon0103, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{pypy,py27}-flask0012, - python-framework_flask-{pypy,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27}-flask0012, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py27,py37,py38,py39,pypy,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-graphql02, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{pypy,py27,py38}-Pyramid0104, - python-framework_pyramid-{pypy,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, + python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -150,18 +150,18 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy,pypy37}-pika0.13, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{pypy,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, python-template_genshi-{py27,py37,py311}-genshilatest python-template_mako-{py27,py37,py310,py311} @@ -170,7 +170,7 @@ envlist = deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 - {py27,pypy}: pytest==4.6.11 + {py27,pypy27}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -201,7 +201,7 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 application_celery: celery<6.0 - application_celery-py{py37,37}: importlib-metadata<5.0 + application_celery-{py37,pypy37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 @@ -219,7 +219,7 @@ deps = component_graphqlserver: markupsafe<2.1 component_graphqlserver: jinja2<3.1 component_tastypie-tastypie0143: django-tastypie<0.14.4 - component_tastypie-{py27,pypy}-tastypie0143: django<1.12 + component_tastypie-{py27,pypy27}-tastypie0143: django<1.12 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie @@ -251,7 +251,7 @@ deps = datastore_redis-redislatest: redis datastore_redis-redis0400: redis<4.1 datastore_redis-redis03: redis<4.0 - datastore_redis-{py27,pypy}: rb + datastore_redis-{py27,pypy27}: rb datastore_aioredis-redislatest: redis datastore_aioredis-aioredislatest: aioredis datastore_aioredis-aioredis01: aioredis<2 @@ -369,7 +369,7 @@ deps = messagebroker_pika-pika0.13: pika<0.14 messagebroker_pika-pikalatest: pika messagebroker_pika: tornado<5 - messagebroker_pika-{py27,pypy}: enum34 + messagebroker_pika-{py27,pypy27}: enum34 messagebroker_confluentkafka-confluentkafkalatest: confluent-kafka messagebroker_confluentkafka-confluentkafka0108: confluent-kafka<1.9 messagebroker_confluentkafka-confluentkafka0107: confluent-kafka<1.8 From 25add74086168f8935f73f5f5173b21c61f562b6 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 13:12:28 -0700 Subject: [PATCH 04/25] Fix checkout logic --- .github/workflows/tests.yml | 98 +++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b91ddbd4b0..b92f3dfd91 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,6 +60,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -122,6 +123,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -160,6 +163,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -213,6 +218,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -269,6 +276,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -320,6 +329,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -373,6 +384,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -424,6 +437,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -476,6 +491,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -498,7 +515,7 @@ jobs: path: ./**/.coverage.* retention-days: 1 - #kafka: + # kafka: # env: # TOTAL_GROUPS: 4 @@ -534,41 +551,42 @@ jobs: # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + # - uses: actions/checkout@v3 + # with: + # fetch-depth: 0 + + # # Special case packages + # - name: Install librdkafka-dev + # run: | + # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name + # sudo apt-get update + # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + # sudo apt-get install -y wget gnupg2 software-properties-common + # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - + # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" + # sudo apt-get update + # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 mongodb: env: @@ -599,6 +617,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -652,6 +672,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -706,6 +728,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs @@ -756,6 +780,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Get Environments id: get-envs From 479b7648864002bdfa3abb40f6caee211dc574a4 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 14:28:02 -0700 Subject: [PATCH 05/25] Fetch git tags properly --- .github/workflows/tests.yml | 84 ++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b92f3dfd91..5649ba1370 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,8 +123,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -163,8 +166,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -218,8 +224,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -276,8 +285,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -329,8 +341,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -384,8 +399,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -437,8 +455,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -491,8 +512,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -617,8 +641,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -672,8 +699,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -728,8 +758,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -780,8 +813,11 @@ jobs: steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs From d062a6eedffdfff832f20a09daf055d505873d6c Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 14:40:31 -0700 Subject: [PATCH 06/25] Pin tox requires --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index d3ad596c85..fa792fc9e1 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,7 @@ ; - python-adapter_gevent-py27 [tox] +requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, From 430bc25704f8b6a0c231025cd2a28f554762793a Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 15:47:58 -0700 Subject: [PATCH 07/25] Adjust default db settings for github actions --- tests/testing_support/db_settings.py | 132 +++++++++------------------ 1 file changed, 41 insertions(+), 91 deletions(-) diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index c7c35935f8..570970df57 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -29,25 +29,15 @@ def postgresql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "postgres" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 5432 - + host = "postgres" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "localhost", - "port": base_port + instance_num, + "user": "postgres", + "password": "postgres", + "name": "postgres", + "host": host, + "port": 8080 + instance_num, "table_name": "postgres_table_" + str(os.getpid()), } for instance_num in range(instances) @@ -66,25 +56,15 @@ def mysql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "python_agent" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 3306 - + host = "mysql" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "127.0.0.1", - "port": base_port + instance_num, + "user": "python_agent", + "password": "python_agent", + "name": "python_agent", + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -103,17 +83,12 @@ def redis_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 6379 - + host = "redis" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] @@ -131,17 +106,12 @@ def memcached_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 11211 - + host = "rabbitmq" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -160,15 +130,10 @@ def mongodb_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 27017 - + host = "rabbitmq" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ - {"host": "127.0.0.1", "port": base_port + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} + {"host": host, "port": 8080 + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} for instance_num in range(instances) ] return settings @@ -185,17 +150,12 @@ def elasticsearch_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9200 - + host = "elasticsearch" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": str(base_port + instance_num), + "host": host, + "port": str(8080 + instance_num), "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -214,17 +174,12 @@ def solr_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 8983 - + host = "solr" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -243,13 +198,12 @@ def rabbitmq_settings(): 2. Github Actions """ + host = "rabbitmq" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 1 - base_port = 5672 - settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 5672 + instance_num, } for instance_num in range(instances) ] @@ -267,17 +221,13 @@ def kafka_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9092 - + + host = "kafka" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] From 877b380430802f0ed7c62a7b941d21145c894226 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 16:19:58 -0700 Subject: [PATCH 08/25] Rename elasticsearch services --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5649ba1370..41d4d25e1b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -683,7 +683,7 @@ jobs: timeout-minutes: 30 services: - es07: + elasticsearch: image: elasticsearch:7.17.8 env: "discovery.type": "single-node" @@ -741,7 +741,7 @@ jobs: timeout-minutes: 30 services: - es08: + elasticsearch: image: elasticsearch:8.6.0 env: "xpack.security.enabled": "false" From 88c13c7193b0bfa7e3a1256ac9c11b4fee822844 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:07:15 -0700 Subject: [PATCH 09/25] Reset to new pipelines --- .github/workflows/tests.yml | 189 +++++++++++++++--------------------- newrelic/config.py | 30 +++--- tox.ini | 123 ++++++++++++----------- 3 files changed, 161 insertions(+), 181 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 015061785c..41d4d25e1b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,10 +36,9 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - # - gearman + - gearman - grpc #- kafka - - libcurl - memcached - mongodb - mysql @@ -118,11 +117,12 @@ jobs: ] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -160,11 +160,12 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -192,49 +193,6 @@ jobs: path: ./**/.coverage.* retention-days: 1 - libcurl: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - # Special case packages - - name: Install libcurl-dev - run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev - - - name: Get Environments - id: get-envs - run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 - postgres: env: TOTAL_GROUPS: 2 @@ -245,6 +203,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -264,15 +224,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - - name: Install odbc driver for postgresql - run: | - sudo apt-get update - sudo sudo apt-get install odbc-postgresql - sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini - sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini - sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini - name: Fetch git tags run: | @@ -310,6 +261,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -332,7 +285,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -370,6 +322,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -387,7 +341,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -425,6 +378,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -444,7 +399,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -482,6 +436,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -499,7 +455,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -537,6 +492,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -555,7 +512,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -666,6 +622,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -683,7 +641,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -721,6 +678,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -740,7 +699,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -778,6 +736,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 timeout-minutes: 30 services: @@ -798,7 +758,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Fetch git tags run: | @@ -826,51 +785,57 @@ jobs: path: ./**/.coverage.* retention-days: 1 - # gearman: - # env: - # TOTAL_GROUPS: 1 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # gearman: - # image: artefactual/gearmand - # ports: - # - 4730:4730 - # # Set health checks to wait until gearman has started - # options: >- - # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + gearman: + env: + TOTAL_GROUPS: 1 + + strategy: + fail-fast: false + matrix: + group-number: [1] + + runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + timeout-minutes: 30 + + services: + gearman: + image: artefactual/gearmand + ports: + - 4730:4730 + # Set health checks to wait until gearman has started + options: >- + --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index 5c3961b026..df95db0290 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3029,21 +3029,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - # _process_module_definition( - # "gearman.client", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_client", - # ) - # _process_module_definition( - # "gearman.connection_manager", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_connection_manager", - # ) - # _process_module_definition( - # "gearman.worker", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_worker", - # ) + _process_module_definition( + "gearman.client", + "newrelic.hooks.application_gearman", + "instrument_gearman_client", + ) + _process_module_definition( + "gearman.connection_manager", + "newrelic.hooks.application_gearman", + "instrument_gearman_connection_manager", + ) + _process_module_definition( + "gearman.worker", + "newrelic.hooks.application_gearman", + "instrument_gearman_worker", + ) _process_module_definition( "botocore.endpoint", diff --git a/tox.ini b/tox.ini index 6b1e11407d..fa792fc9e1 100644 --- a/tox.ini +++ b/tox.ini @@ -43,10 +43,10 @@ requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py37,py38,py310,py311}, + python-adapter_gevent-{py27,py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -55,81 +55,95 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy37}-without_extensions, + python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy27,pypy37}-without_extensions, + python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy37}-without_extensions, - python-application_celery-{py37,py38,py39,py310,py311,pypy37}, + python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy27,pypy37}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + gearman-application_gearman-{py27,pypy27}, + python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, + python-component_tastypie-{py27,pypy27}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, + mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, - postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, - memcached-datastore_pylibmc-{py37}, - memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, - mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, - solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, - redis-datastore_redis-{py37,py38,pypy37}-redis03, + postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest + postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest + memcached-datastore_pylibmc-{py27,py37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, - python-external_boto3-{py37,py38,py39,py310,py311}-boto01, + solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_http-{py37,py38,py39,py310,py311}, - python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, - python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, + python-external_feedparser-py27-feedparser{05,06}, + python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py37,py38,py39,py310,py311,pypy37}, - python-external_urllib3-{py37}-urllib3{0109}, - python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-py27-bottle{0008,0009,0010}, + python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, + python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{py37}-Django0108, + python-framework_django-{pypy27,py27}-Django0103, + python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27}-flask0012, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, + grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{py38}-Pyramid0104, - python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, + python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -137,26 +151,27 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, - libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, + python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py37,py311}-genshilatest, - python-template_mako-{py37,py310,py311}, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py27,py37,py311}-genshilatest + python-template_mako-{py27,py37,py310,py311} [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py27,pypy27}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -187,8 +202,8 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 application_celery: celery<6.0 - application_celery-py{py37,37}: importlib-metadata<5.0 - ; application_gearman: gearman<3.0.0 + application_celery-{py37,pypy37}: importlib-metadata<5.0 + application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -376,9 +391,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - libcurl: PYCURL_SSL_LIBRARY=openssl - libcurl: LDFLAGS=-L/usr/local/opt/openssl/lib - libcurl: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado: PYCURL_SSL_LIBRARY=openssl + framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -393,7 +408,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* @@ -416,7 +431,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - ; application_gearman: tests/application_gearman + application_gearman: tests/application_gearman component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From 0338cbc82794478a0e0471042ac2c9531bdafe29 Mon Sep 17 00:00:00 2001 From: TimPansino Date: Fri, 23 Jun 2023 00:09:19 +0000 Subject: [PATCH 10/25] [Mega-Linter] Apply linters fixes --- tests/testing_support/db_settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index 570970df57..87e1b20157 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -221,7 +221,6 @@ def kafka_settings(): 2. Github Actions """ - host = "kafka" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 2 settings = [ From cbf41f3dbbeed7eb1d2111e4825a3e8b5198184e Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:32:53 -0700 Subject: [PATCH 11/25] Fix timezone --- .github/containers/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 8809d87c24..54fec2aee6 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -38,11 +38,12 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libssl-dev \ locales \ make \ + odbc-postgresql \ openssl \ python2-dev \ python3-dev \ python3-pip \ - odbc-postgresql \ + tzdata \ unzip \ wget \ zip \ @@ -58,6 +59,9 @@ RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psq # Set the locale RUN locale-gen --no-purge en_US.UTF-8 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 +ENV TZ="Etc/UTC" +RUN ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata # Use root user ENV HOME /root From abe65c682a1a492a24a610edb0160fc47da14572 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:38:05 -0700 Subject: [PATCH 12/25] Fix docker networking --- .github/workflows/tests.yml | 173 ++++++++++++++++----------- tests/testing_support/db_settings.py | 20 ++-- 2 files changed, 112 insertions(+), 81 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 41d4d25e1b..e81983b337 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -119,6 +119,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: @@ -162,6 +164,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: @@ -205,6 +209,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -263,6 +269,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -324,6 +332,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -380,6 +390,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -438,6 +450,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -494,6 +508,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -540,77 +556,84 @@ jobs: retention-days: 1 # kafka: - # env: - # TOTAL_GROUPS: 4 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1, 2, 3, 4] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # zookeeper: - # image: bitnami/zookeeper:3.7 - # env: - # ALLOW_ANONYMOUS_LOGIN: yes - - # ports: - # - 2181:2181 - - # kafka: - # image: bitnami/kafka:3.2 - # ports: - # - 8080:8080 - # - 8081:8081 - # env: - # ALLOW_PLAINTEXT_LISTENER: yes - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true - # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 - # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, - # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT - # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 - - # steps: - # - uses: actions/checkout@v3 - # with: - # fetch-depth: 0 - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + # env: + # TOTAL_GROUPS: 4 + + # strategy: + # fail-fast: false + # matrix: + # group-number: [1, 2, 3, 4] + + # runs-on: ubuntu-20.04 + # container: + # image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + # options: >- + # --add-host=host.docker.internal:host-gateway + # timeout-minutes: 30 + + # services: + # zookeeper: + # image: bitnami/zookeeper:3.7 + # env: + # ALLOW_ANONYMOUS_LOGIN: yes + + # ports: + # - 2181:2181 + + # kafka: + # image: bitnami/kafka:3.2 + # ports: + # - 8080:8080 + # - 8081:8081 + # env: + # ALLOW_PLAINTEXT_LISTENER: yes + # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 + # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, + # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT + # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 + + # steps: + # - uses: actions/checkout@v3 + + # - name: Fetch git tags + # run: | + # git config --global --add safe.directory "$GITHUB_WORKSPACE" + # git fetch --tags origin + + # # Special case packages + # - name: Install librdkafka-dev + # run: | + # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name + # sudo apt-get update + # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + # sudo apt-get install -y wget gnupg2 software-properties-common + # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - + # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" + # sudo apt-get update + # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 mongodb: env: @@ -624,6 +647,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -680,6 +705,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -738,6 +765,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -797,6 +826,8 @@ jobs: runs-on: ubuntu-20.04 container: image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index 87e1b20157..d2b4b5b72a 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -29,7 +29,7 @@ def postgresql_settings(): 2. Github Actions """ - host = "postgres" if "GITHUB_ACTIONS" in os.environ else "localhost" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 2 settings = [ { @@ -56,8 +56,8 @@ def mysql_settings(): 2. Github Actions """ - host = "mysql" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" - instances = 2 + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 1 settings = [ { "user": "python_agent", @@ -83,7 +83,7 @@ def redis_settings(): 2. Github Actions """ - host = "redis" if "GITHUB_ACTIONS" in os.environ else "localhost" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 2 settings = [ { @@ -106,7 +106,7 @@ def memcached_settings(): 2. Github Actions """ - host = "rabbitmq" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" instances = 2 settings = [ { @@ -130,7 +130,7 @@ def mongodb_settings(): 2. Github Actions """ - host = "rabbitmq" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" instances = 2 settings = [ {"host": host, "port": 8080 + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} @@ -150,7 +150,7 @@ def elasticsearch_settings(): 2. Github Actions """ - host = "elasticsearch" if "GITHUB_ACTIONS" in os.environ else "localhost" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 2 settings = [ { @@ -174,7 +174,7 @@ def solr_settings(): 2. Github Actions """ - host = "solr" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" instances = 2 settings = [ { @@ -198,7 +198,7 @@ def rabbitmq_settings(): 2. Github Actions """ - host = "rabbitmq" if "GITHUB_ACTIONS" in os.environ else "localhost" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 1 settings = [ { @@ -221,7 +221,7 @@ def kafka_settings(): 2. Github Actions """ - host = "kafka" if "GITHUB_ACTIONS" in os.environ else "localhost" + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 2 settings = [ { From d46236be90e4000d54f4720e898e649e33c27e4d Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:38:47 -0700 Subject: [PATCH 13/25] Pin dev image to new sha --- .github/workflows/tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e81983b337..f28e3cefdd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -163,7 +163,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -208,7 +208,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -331,7 +331,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -389,7 +389,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -449,7 +449,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -507,7 +507,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -566,7 +566,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + # image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -646,7 +646,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -704,7 +704,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -764,7 +764,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -825,7 +825,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-89f5dd62bd2d9167e51962c305b9b57a1a7cdb75 + image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 From 6a234aff2521f72fd9608305fee941539a2e230e Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:44:06 -0700 Subject: [PATCH 14/25] Standardize gearman DB settings --- .github/workflows/tests.yml | 2 +- tests/application_gearman/test_gearman.py | 6 ++++-- tests/testing_support/db_settings.py | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f28e3cefdd..1037ac32c0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -834,7 +834,7 @@ jobs: gearman: image: artefactual/gearmand ports: - - 4730:4730 + - 8080:4730 # Set health checks to wait until gearman has started options: >- --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" diff --git a/tests/application_gearman/test_gearman.py b/tests/application_gearman/test_gearman.py index 7ddc13fdc3..db993b582b 100644 --- a/tests/application_gearman/test_gearman.py +++ b/tests/application_gearman/test_gearman.py @@ -20,14 +20,16 @@ import gearman from newrelic.api.background_task import background_task +from testing_support.db_settings import gearman_settings worker_thread = None worker_event = threading.Event() gm_client = None -GEARMAND_HOST = os.environ.get("GEARMAND_PORT_4730_TCP_ADDR", "localhost") -GEARMAND_PORT = os.environ.get("GEARMAND_PORT_4730_TCP_PORT", "4730") +GEARMAND_SETTINGS = gearman_settings() +GEARMAND_HOST = GEARMAND_SETTINGS["host"] +GEARMAND_PORT = GEARMAND_SETTINGS["port"] GEARMAND_ADDR = "%s:%s" % (GEARMAND_HOST, GEARMAND_PORT) diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index d2b4b5b72a..bda3180622 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -231,3 +231,26 @@ def kafka_settings(): for instance_num in range(instances) ] return settings + + +def gearman_settings(): + """Return a list of dict of settings for connecting to kafka. + + Will return the correct settings, depending on which of the environments it + is running in. It attempts to set variables in the following order, where + later environments override earlier ones. + + 1. Local + 2. Github Actions + """ + + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 1 + settings = [ + { + "host": host, + "port": 8080 + instance_num, + } + for instance_num in range(instances) + ] + return settings From 2e03372a24815d3ba8b45956e6aa627f4d48bb07 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:45:36 -0700 Subject: [PATCH 15/25] Fix elasticsearch settings bug --- tests/datastore_elasticsearch/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/datastore_elasticsearch/test_connection.py b/tests/datastore_elasticsearch/test_connection.py index 2e888af9b5..9e8f17b4c1 100644 --- a/tests/datastore_elasticsearch/test_connection.py +++ b/tests/datastore_elasticsearch/test_connection.py @@ -36,7 +36,7 @@ def test_connection_default(): else: conn = Connection(**HOST) - assert conn._nr_host_port == ("localhost", ES_SETTINGS["port"]) + assert conn._nr_host_port == (ES_SETTINGS["host"], ES_SETTINGS["port"]) @SKIP_IF_V7 From da338fc28fce6584e7ee1cacfe0066c0b76f7d4a Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:51:09 -0700 Subject: [PATCH 16/25] Fix gearman bug --- tests/application_gearman/test_gearman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/application_gearman/test_gearman.py b/tests/application_gearman/test_gearman.py index db993b582b..5dda4ef47e 100644 --- a/tests/application_gearman/test_gearman.py +++ b/tests/application_gearman/test_gearman.py @@ -27,7 +27,7 @@ gm_client = None -GEARMAND_SETTINGS = gearman_settings() +GEARMAND_SETTINGS = gearman_settings()[0] GEARMAND_HOST = GEARMAND_SETTINGS["host"] GEARMAND_PORT = GEARMAND_SETTINGS["port"] From 712926a261847335fb98ffa20a66ecdccbba88dd Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:51:17 -0700 Subject: [PATCH 17/25] Add missing odbc headers --- .github/containers/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 54fec2aee6..3b4b0a7f87 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -44,6 +44,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-dev \ python3-pip \ tzdata \ + unixodbc-dev \ unzip \ wget \ zip \ From 507d27a18fef37659d9a8f95a7fef6aa72397d34 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:54:02 -0700 Subject: [PATCH 18/25] Add more debug messages --- tests/testing_support/validators/validate_tt_collector_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testing_support/validators/validate_tt_collector_json.py b/tests/testing_support/validators/validate_tt_collector_json.py index 85e3932806..28c9e93a39 100644 --- a/tests/testing_support/validators/validate_tt_collector_json.py +++ b/tests/testing_support/validators/validate_tt_collector_json.py @@ -135,7 +135,7 @@ def _check_params_and_start_time(node): if segment_name.startswith("Datastore"): for key in datastore_params: assert key in params, key - assert params[key] == datastore_params[key] + assert params[key] == datastore_params[key], "Expected %s. Got %s." % (datastore_params[key], params[key]) for key in datastore_forgone_params: assert key not in params, key From d9a1ac20ae3a1dd1fdb53576ea6c34dcfbd4c6f1 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:55:58 -0700 Subject: [PATCH 19/25] Swap out dev ci image --- .github/workflows/tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1037ac32c0..0033f8633b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -163,7 +163,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -208,7 +208,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -331,7 +331,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -389,7 +389,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -449,7 +449,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -507,7 +507,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -566,7 +566,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + # image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -646,7 +646,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -704,7 +704,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -764,7 +764,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -825,7 +825,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-b1cb9004196da0d511c42a9a6248523748c324f0 + image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 From 907633f0ad01a0338c6131edb8b1e0ed89b57406 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 17:59:46 -0700 Subject: [PATCH 20/25] Fix required virtualenv version --- .github/containers/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/containers/requirements.txt b/.github/containers/requirements.txt index 54f3c39d00..27fa6624b1 100644 --- a/.github/containers/requirements.txt +++ b/.github/containers/requirements.txt @@ -1,5 +1,5 @@ pip setuptools wheel -virtualenv<20.22.1 +virtualenv<20.22.0 tox \ No newline at end of file From 1441bdbc206e398843251b36a80e8deb0102657f Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 18:01:00 -0700 Subject: [PATCH 21/25] Swap out dev ci image --- .github/workflows/tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0033f8633b..159426d4c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -163,7 +163,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -208,7 +208,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -331,7 +331,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -389,7 +389,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -449,7 +449,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -507,7 +507,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -566,7 +566,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + # image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -646,7 +646,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -704,7 +704,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -764,7 +764,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -825,7 +825,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-507d27a18fef37659d9a8f95a7fef6aa72397d34 + image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 From 967c82e82d2ea1ccf9f02309f506aa29c24d3a69 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 18:26:10 -0700 Subject: [PATCH 22/25] Swap out dev ci image --- .github/workflows/tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 159426d4c5..02643b5a3c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -163,7 +163,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -208,7 +208,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -331,7 +331,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -389,7 +389,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -449,7 +449,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -507,7 +507,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -566,7 +566,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + # image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -646,7 +646,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -704,7 +704,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -764,7 +764,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -825,7 +825,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-907633f0ad01a0338c6131edb8b1e0ed89b57406 + image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 From 014431cf9daa8610e8e221036563b470720e3499 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 22 Jun 2023 19:44:17 -0700 Subject: [PATCH 23/25] Remove aioredis v1 for EOL --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index fa792fc9e1..a28f2c878b 100644 --- a/tox.ini +++ b/tox.ini @@ -94,7 +94,6 @@ envlist = redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, - redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, @@ -255,7 +254,6 @@ deps = datastore_redis-{py27,pypy27}: rb datastore_aioredis-redislatest: redis datastore_aioredis-aioredislatest: aioredis - datastore_aioredis-aioredis01: aioredis<2 datastore_aredis-aredislatest: aredis datastore_solrpy-solrpy00: solrpy<1.0 datastore_solrpy-solrpy01: solrpy<2.0 From 3fa18ed30ee25d8e4e12809a1021e6e7c17ad176 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Fri, 23 Jun 2023 09:54:08 -0700 Subject: [PATCH 24/25] Add coverage paths for docker container --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index a28f2c878b..0ac7331298 100644 --- a/tox.ini +++ b/tox.ini @@ -502,6 +502,7 @@ source = newrelic source = newrelic/ .tox/**/site-packages/newrelic/ + /__w/**/site-packages/newrelic/ [coverage:html] directory = ${TOX_ENV_DIR-.}/htmlcov From 6e4c49627157178f8c1688e4a7c89d3670b6ae6b Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Fri, 23 Jun 2023 10:10:24 -0700 Subject: [PATCH 25/25] Unpin ci container --- .github/workflows/tests.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02643b5a3c..f8e5182434 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -163,7 +163,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -208,7 +208,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -331,7 +331,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -389,7 +389,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -449,7 +449,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -507,7 +507,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -566,7 +566,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + # image: ghcr.io/${{ github.repository }}-ci:latest # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -646,7 +646,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -704,7 +704,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -764,7 +764,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -825,7 +825,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:sha-1441bdbc206e398843251b36a80e8deb0102657f + image: ghcr.io/${{ github.repository }}-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30