chore(gunicorn): add tests (#4776)

Add testing support for gunicorn. While documentation has been provided for using ddtrace with gunicorn, there were no tests ensuring that the library actually works with it. ## Reviewer Checklist - [ ] Title is accurate. - [ ] Description motivates each change. - [ ] No unnecessary changes were introduced in this PR. - [ ] Avoid breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes unless absolutely necessary. - [ ] Tests provided or description of manual testing performed is included in the code or PR. - [ ] Release note has been added and follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/contributing.html#Release-Note-Guidelines), or else `changelog/no-changelog` label added. - [ ] All relevant GitHub issues are correctly linked. - [ ] Backports are identified and tagged with Mergifyio. Co-authored-by: Yun Kim <[email protected]> Co-authored-by: Yun Kim <[email protected]> Co-authored-by: Gabriele N. Tornetta <[email protected]>
DataDog · Dec 20, 2022 · e9f7a87 · e9f7a87
1 parent a9ee02f
commit e9f7a87
Show file tree

Hide file tree

Showing 14 changed files with 284 additions and 24 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -781,6 +781,13 @@ jobs:
           pattern: "grpc"
           snapshot: true
 
+  gunicorn:
+    <<: *machine_executor
+    steps:
+      - run_test:
+          pattern: "gunicorn"
+          snapshot: true
+
   httplib:
     <<: *machine_executor
     steps:
@@ -1170,6 +1177,7 @@ requires_tests: &requires_tests
     - gevent
     - graphql
     - grpc
+    - gunicorn
     - httplib
     - httpx
     - integration_agent
@@ -1269,6 +1277,7 @@ workflows:
       - graphene: *requires_base_venvs
       - graphql: *requires_base_venvs
       - grpc: *requires_base_venvs
+      - gunicorn: *requires_base_venvs
       - httplib: *requires_base_venvs
       - httpx: *requires_base_venvs
       - integration_agent: *requires_base_venvs

diff --git a/.circleci/dependencies.py b/.circleci/dependencies.py
@@ -68,6 +68,7 @@
     "graphene": "3.1.1",
     "graphql-core": "3.2.3",
     "grpcio": "1.51.1",
+    "gunicorn": "20.1.0",
     "httpretty": "1.1.4",
     "httpx": "0.23.1",
     "hypothesis": "6.60.0",

diff --git a/conftest.py b/conftest.py
@@ -69,7 +69,7 @@ def pytest_configure(config):
 # DEV: We can only ignore folders/modules, we cannot ignore individual files
 # DEV: We must wrap with `@pytest.mark.hookwrapper` to inherit from default (e.g. honor `--ignore`)
 #      https://github.com/pytest-dev/pytest/issues/846#issuecomment-122129189
-@pytest.mark.hookwrapper
+@pytest.hookimpl(hookwrapper=True)
 def pytest_ignore_collect(path, config):
     """
     Skip directories defining a required minimum Python version

diff --git a/ddtrace/contrib/gunicorn/__init__.py b/ddtrace/contrib/gunicorn/__init__.py
@@ -0,0 +1,23 @@
+"""
+``ddtrace`` supports `Gunicorn <https://gunicorn.org>`__.
+
+If the application is using the ``gevent`` worker class, ``gevent`` monkey patching must be performed before loading the
+``ddtrace`` library.
+
+There are different options to ensure this happens:
+
+- If using ``ddtrace-run``, set the environment variable ``DD_GEVENT_PATCH_ALL=1``.
+
+- Replace ``ddtrace-run`` by using ``import ddtrace.bootstrap.sitecustomize`` as the first import of the application.
+
+- Use a `post_worker_init <https://docs.gunicorn.org/en/stable/settings.html#post-worker-init>`_
+  hook to import ``ddtrace.bootstrap.sitecustomize``.
+"""
+
+
+def patch():
+    pass
+
+
+def unpatch():
+    pass
diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst
@@ -677,25 +677,3 @@ Example with uWSGI ini file:
 .. code-block:: bash
 
   uwsgi --ini uwsgi.ini
-
-
-.. _gunicorn:
-
-Gunicorn
---------
-
-``ddtrace`` supports `Gunicorn <https://gunicorn.org>`__.
-
-However, if you are using the ``gevent`` worker class, you have to make sure
-``gevent`` monkey patching is done before loading the ``ddtrace`` library.
-
-There are different options to make that happen:
-
-- If you rely on ``ddtrace-run``, you must set ``DD_GEVENT_PATCH_ALL=1`` in
-  your environment to have gevent patched first-thing.
-
-- Replace ``ddtrace-run`` by using ``import ddtrace.bootstrap.sitecustomize``
-  as the first import of your application.
-
-- Use a `post_worker_init <https://docs.gunicorn.org/en/stable/settings.html#post-worker-init>`_
-  hook to import ``ddtrace.bootstrap.sitecustomize``.
diff --git a/docs/index.rst b/docs/index.rst
@@ -101,6 +101,8 @@ contacting support.
 +--------------------------------------------------+---------------+----------------+
 | :ref:`graphql-core <graphql>`                    | >= 2.0.0      | Yes            |
 +--------------------------------------------------+---------------+----------------+
+| :ref:`gunicorn <gunicorn>`                       | >= 19.10.0    | No             |
++--------------------------------------------------+---------------+----------------+
 | :ref:`httplib`                                   | \*            | Yes            |
 +--------------------------------------------------+---------------+----------------+
 | :ref:`httpx`                                     | >= 0.14.0     | Yes            |

diff --git a/docs/integrations.rst b/docs/integrations.rst
@@ -203,6 +203,13 @@ Grpc
 .. automodule:: ddtrace.contrib.grpc
 
 
+.. _gunicorn:
+
+gunicorn
+^^^^^^^^
+.. automodule:: ddtrace.contrib.gunicorn
+
+
 .. _httplib:
 
 httplib

diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -86,6 +86,7 @@ graphene
 greenlet
 greenlets
 grpc
+gunicorn
 hostname
 http
 httplib
@@ -204,4 +205,4 @@ serverless
 cattrs
 IAST
 DES
-Blowfish
+Blowfish
diff --git a/riotfile.py b/riotfile.py
@@ -2586,6 +2586,22 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                 "molten": [">=0.6,<0.7", ">=0.7,<0.8", ">=1.0,<1.1", latest],
             },
         ),
+        Venv(
+            name="gunicorn",
+            command="pytest {cmdargs} tests/contrib/gunicorn",
+            pkgs={"requests": latest},
+            venvs=[
+                Venv(
+                    pys="2.7",
+                    # Gunicorn ended Python 2 support after 19.10.0
+                    pkgs={"gunicorn": "==19.10.0"},
+                ),
+                Venv(
+                    pys=select_pys(min_version="3.5"),
+                    pkgs={"gunicorn": ["==19.10.0", "==20.0.4", latest]},
+                ),
+            ],
+        ),
     ],
 )
 

diff --git a/tests/contrib/gunicorn/__init__.py b/tests/contrib/gunicorn/__init__.py
diff --git a/tests/contrib/gunicorn/simple_app.py b/tests/contrib/gunicorn/simple_app.py
@@ -0,0 +1,4 @@
+def app(_, start_response):
+    data = b"Hello, World!\n"
+    start_response("200 OK", [("Content-Type", "text/plain"), ("Content-Length", str(len(data)))])
+    return iter([data])
diff --git a/tests/contrib/gunicorn/test_gunicorn.py b/tests/contrib/gunicorn/test_gunicorn.py
@@ -0,0 +1,130 @@
+import os
+import subprocess
+import sys
+from typing import Dict
+from typing import NamedTuple
+
+import pytest
+import tenacity
+
+from ddtrace.internal.compat import stringify
+from tests.webclient import Client
+
+
+GunicornServerSettings = NamedTuple(
+    "GunicornServerSettings",
+    [
+        ("env", Dict[str, str]),
+        ("directory", str),
+        ("app_path", str),
+        ("num_workers", str),
+        ("worker_class", str),
+        ("bind", str),
+        ("use_ddtracerun", bool),
+        ("post_worker_init", str),
+    ],
+)
+
+
+_post_worker_init_ddtrace = """
+def post_worker_init(worker):
+    import ddtrace.bootstrap.sitecustomize
+"""
+
+
+def _gunicorn_settings_factory(
+    env=os.environ.copy(),  # type: Dict[str, str]
+    directory=os.getcwd(),  # type: str
+    app_path="tests.contrib.gunicorn.simple_app:app",  # type: str
+    num_workers="4",  # type: str
+    worker_class="sync",  # type: str
+    bind="0.0.0.0:8080",  # type: str
+    use_ddtracerun=True,  # type: bool
+    post_worker_init="",  # type: str
+):
+    # type: (...) -> GunicornServerSettings
+    """Factory for creating gunicorn settings with simple defaults if settings are not defined."""
+    return GunicornServerSettings(
+        env=env,
+        directory=directory,
+        app_path=app_path,
+        num_workers=num_workers,
+        worker_class=worker_class,
+        bind=bind,
+        use_ddtracerun=use_ddtracerun,
+        post_worker_init=post_worker_init,
+    )
+
+
+@pytest.fixture
+def gunicorn_server_settings():
+    yield _gunicorn_settings_factory()
+
+
+@pytest.fixture
+def gunicorn_server(gunicorn_server_settings, tmp_path):
+    cfg_file = tmp_path / "gunicorn.conf.py"
+    cfg = """
+{post_worker_init}
+
+workers = {num_workers}
+worker_class = "{worker_class}"
+bind = "{bind}"
+""".format(
+        post_worker_init=gunicorn_server_settings.post_worker_init,
+        bind=gunicorn_server_settings.bind,
+        num_workers=gunicorn_server_settings.num_workers,
+        worker_class=gunicorn_server_settings.worker_class,
+    )
+    cfg_file.write_text(stringify(cfg))
+    cmd = []
+    if gunicorn_server_settings.use_ddtracerun:
+        cmd = ["ddtrace-run"]
+    cmd += ["gunicorn", "--config", str(cfg_file), str(gunicorn_server_settings.app_path)]
+    print("Running %r with configuration file %s" % (" ".join(cmd), cfg))
+    p = subprocess.Popen(
+        cmd,
+        env=gunicorn_server_settings.env,
+        cwd=gunicorn_server_settings.directory,
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        close_fds=True,
+        preexec_fn=os.setsid,
+    )
+    try:
+        client = Client("http://%s" % gunicorn_server_settings.bind)
+        # Wait for the server to start up
+        try:
+            client.wait(max_tries=20, delay=0.5)
+        except tenacity.RetryError:
+            # if proc.returncode is not None:
+            # process failed
+            raise TimeoutError("Server failed to start, see stdout and stderr logs")
+
+        yield client
+
+        try:
+            client.get_ignored("/shutdown")
+        except Exception:
+            pass
+    finally:
+        p.terminate()
+        p.wait()
+
+
+def test_basic(gunicorn_server):
+    r = gunicorn_server.get("/")
+    assert r.status_code == 200
+    assert r.content == b"Hello, World!\n"
+
+
+@pytest.mark.snapshot(ignores=["meta.result_class"])
+@pytest.mark.parametrize(
+    "gunicorn_server_settings", [_gunicorn_settings_factory(app_path="tests.contrib.gunicorn.wsgi_mw_app:app")]
+)
+def test_traced_basic(gunicorn_server_settings, gunicorn_server):
+    # meta.result_class is listiterator vs list_iterator in PY2 vs PY3.
+    # Ignore this field to avoid having to create mostly duplicate snapshots in Python 2 and 3.
+    r = gunicorn_server.get("/")
+    assert r.status_code == 200
+    assert r.content == b"Hello, World!\n"
diff --git a/tests/contrib/gunicorn/wsgi_mw_app.py b/tests/contrib/gunicorn/wsgi_mw_app.py
@@ -0,0 +1,22 @@
+from ddtrace import tracer
+from ddtrace.contrib.wsgi import DDWSGIMiddleware
+from tests.webclient import PingFilter
+
+
+tracer.configure(
+    settings={
+        "FILTERS": [PingFilter()],
+    }
+)
+
+
+def simple_app(environ, start_response):
+    if environ["RAW_URI"] == "/shutdown":
+        tracer.shutdown()
+
+    data = b"Hello, World!\n"
+    start_response("200 OK", [("Content-Type", "text/plain"), ("Content-Length", str(len(data)))])
+    return iter([data])
+
+
+app = DDWSGIMiddleware(simple_app)
diff --git a/...ts/tests.contrib.gunicorn.test_gunicorn.test_traced_basic[gunicorn_server_settings0].json b/...ts/tests.contrib.gunicorn.test_gunicorn.test_traced_basic[gunicorn_server_settings0].json
@@ -0,0 +1,67 @@
+[[
+  {
+    "name": "wsgi.request",
+    "service": "wsgi",
+    "resource": "GET /",
+    "trace_id": 0,
+    "span_id": 1,
+    "parent_id": 0,
+    "type": "web",
+    "error": 0,
+    "meta": {
+      "_dd.p.dm": "-0",
+      "http.method": "GET",
+      "http.status_code": "200",
+      "http.status_msg": "OK",
+      "http.url": "http://0.0.0.0:8080/",
+      "runtime-id": "b73097ce1a54484f872936703dc77f48"
+    },
+    "metrics": {
+      "_dd.agent_psr": 1.0,
+      "_dd.top_level": 1,
+      "_dd.tracer_kr": 1.0,
+      "_sampling_priority_v1": 1,
+      "process_id": 28765
+    },
+    "duration": 198000,
+    "start": 1670886340849791000
+  },
+     {
+       "name": "wsgi.application",
+       "service": "wsgi",
+       "resource": "wsgi.application",
+       "trace_id": 0,
+       "span_id": 2,
+       "parent_id": 1,
+       "type": "",
+       "error": 0,
+       "duration": 49000,
+       "start": 1670886340849877000
+     },
+        {
+          "name": "wsgi.start_response",
+          "service": "wsgi",
+          "resource": "wsgi.start_response",
+          "trace_id": 0,
+          "span_id": 4,
+          "parent_id": 2,
+          "type": "web",
+          "error": 0,
+          "duration": 14000,
+          "start": 1670886340849902000
+        },
+     {
+       "name": "wsgi.response",
+       "service": "wsgi",
+       "resource": "wsgi.response",
+       "trace_id": 0,
+       "span_id": 3,
+       "parent_id": 1,
+       "type": "",
+       "error": 0,
+       "meta": {
+         "result_class": "list_iterator"
+       },
+       "duration": 44000,
+       "start": 1670886340849936000
+     }]]