Merge branch 'chore/module-better-legacy-support' of https://github.c…

…om/p403n1x87/dd-trace-py into chore/module-better-legacy-support

.github/workflows/pr-name.yml

@@ -18,7 +18,7 @@ jobs:
         run: |
           npm install @commitlint/lint @commitlint/load @commitlint/config-conventional @actions/core
       - name: Lint PR name
-        uses: actions/[email protected].0
+        uses: actions/[email protected].1
         with:
           script: |
             const load = require('@commitlint/load').default;

.github/workflows/set-target-milestone.yml

@@ -28,7 +28,7 @@ jobs:
           scripts/get-target-milestone.py
       - name: Update Pull Request
         if: steps.milestones.outputs.milestone != null
-        uses: actions/[email protected].0
+        uses: actions/[email protected].1
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |

.github/workflows/test_frameworks.yml

@@ -149,7 +149,7 @@ jobs:
           repository: tiangolo/fastapi
           ref: 0.75.0
           path: fastapi
-      - uses: actions/[email protected].6
+      - uses: actions/[email protected].7
         id: cache
         with:
           path: ${{ env.pythonLocation }}

ddtrace/appsec/_ddwaf.pyx

@@ -56,6 +56,7 @@ def version():
 
 
 cdef inline object _string_to_bytes(object string, const char **ptr, ssize_t *length):
+    ptr[0] = NULL
     if isinstance(string, six.binary_type):
         ptr[0] = PyBytes_AsString(string)
         length[0] = PyBytes_Size(string)

ddtrace/contrib/pymongo/__init__.py

@@ -23,6 +23,17 @@
     # Use a pin to specify metadata related to this client
     client = pymongo.MongoClient()
     pin = Pin.override(client, service="mongo-master")
+
+Global Configuration
+~~~~~~~~~~~~~~~~~~~~
+
+.. py:data:: ddtrace.config.pymongo["service"]
+   The service name reported by default for pymongo spans
+
+   The option can also be set with the ``DD_PYMONGO_SERVICE`` environment variable
+
+   Default: ``"pymongo"``
+
 """
 from ...internal.utils.importlib import require_modules
 

ddtrace/contrib/pymongo/client.py

@@ -63,7 +63,7 @@ def __init__(self, client=None, *args, **kwargs):
             client._topology = TracedTopology(client._topology)
 
         # Default Pin
-        ddtrace.Pin(service=mongox.SERVICE).onto(self)
+        ddtrace.Pin(service="pymongo").onto(self)
 
     def __setddpin__(self, pin):
         pin.onto(self._topology)

ddtrace/contrib/pymongo/patch.py

@@ -9,7 +9,6 @@
 
 from ...constants import SPAN_MEASURED_KEY
 from ...ext import SpanTypes
-from ...ext import mongo as mongox
 from ..trace_utils import unwrap as _u
 from .client import TracedMongoClient
 from .client import set_address_tags
@@ -18,7 +17,7 @@
 config._add(
     "pymongo",
     dict(
-        _default_service=mongox.SERVICE,
+        _default_service="pymongo",
     ),
 )
 

ddtrace/contrib/requests/connection.py

@@ -7,6 +7,7 @@
 from ...constants import ANALYTICS_SAMPLE_RATE_KEY
 from ...constants import SPAN_MEASURED_KEY
 from ...ext import SpanTypes
+from ...internal.compat import parse
 from ...internal.logger import get_logger
 from ...internal.utils import get_argument_value
 from ...propagation.http import HTTPPropagator
@@ -17,23 +18,17 @@
 
 def _extract_hostname(uri):
     # type: (str) -> str
-    end = len(uri)
-    j = uri.rfind("#", 0, end)
-    if j != -1:
-        end = j
-    j = uri.rfind("&", 0, end)
-    if j != -1:
-        end = j
-
-    start = uri.find("://", 0, end) + 3
-    i = uri.find("@", start, end) + 1
-    if i != 0:
-        start = i
-    j = uri.find("/", start, end)
-    if j != -1:
-        end = j
-
-    return uri[start:end]
+    parsed_uri = parse.urlparse(uri)
+    port = None
+    try:
+        port = parsed_uri.port
+    except ValueError:
+        # ValueError is raised in PY>3.5 when parsed_uri.port < 0 or parsed_uri.port > 65535
+        return "%s:?" % (parsed_uri.hostname,)
+
+    if port is not None:
+        return "%s:%s" % (parsed_uri.hostname, str(port))
+    return parsed_uri.hostname
 
 
 def _extract_query_string(uri):

ddtrace/contrib/trace_utils.py

@@ -49,6 +49,9 @@
 # starting a "new object" on the UI.
 NORMALIZE_PATTERN = re.compile(r"([^a-z0-9_\-:/]){1}")
 
+# Possible User Agent header.
+USER_AGENT_PATTERNS = {"HTTP_USER_AGENT", "User-Agent", "user-agent", "Http-User-Agent"}
+
 
 @cached()
 def _normalized_header_name(header_name):
@@ -102,13 +105,28 @@ def _store_headers(headers, span, integration_config, request_or_response):
         return
 
     for header_name, header_value in headers.items():
+        """config._header_tag_name gets an element of the dictionary in config.http._header_tags
+        which gets the value from DD_TRACE_HEADER_TAGS environment variable."""
         tag_name = integration_config._header_tag_name(header_name)
         if tag_name is None:
             continue
         # An empty tag defaults to a http.<request or response>.headers.<header name> tag
         span.set_tag(tag_name or _normalize_tag_name(request_or_response, header_name), header_value)
 
 
+def _get_request_header_user_agent(headers):
+    # type: (Dict[str, str], Span, IntegrationConfig, str) -> str
+    """Get user agent from request headers
+    :param headers: A dict of http headers to be stored in the span
+    :type headers: dict or list
+    """
+    for key_pattern in USER_AGENT_PATTERNS:
+        user_agent = headers.get(key_pattern)
+        if user_agent:
+            return user_agent
+    return ""
+
+
 def _store_request_headers(headers, span, integration_config):
     # type: (Dict[str, str], Span, IntegrationConfig) -> None
     """
@@ -289,8 +307,15 @@ def set_http_meta(
     if query is not None and integration_config.trace_query_string:
         span._set_str_tag(http.QUERY_STRING, query)
 
-    if request_headers is not None and integration_config.is_header_tracing_configured:
-        _store_request_headers(dict(request_headers), span, integration_config)
+    if request_headers:
+        user_agent = _get_request_header_user_agent(request_headers)
+        if user_agent:
+            span.set_tag(http.USER_AGENT, user_agent)
+
+        if integration_config.is_header_tracing_configured:
+            """We should store both http.<request_or_response>.headers.<header_name> and http.<key>. The last one
+            is the DD standardized tag for user-agent"""
+            _store_request_headers(dict(request_headers), span, integration_config)
 
     if response_headers is not None and integration_config.is_header_tracing_configured:
         _store_response_headers(dict(response_headers), span, integration_config)

ddtrace/ext/http.py

@@ -10,6 +10,7 @@
 URL = "http.url"
 METHOD = "http.method"
 STATUS_CODE = "http.status_code"
+USER_AGENT = "http.useragent"
 STATUS_MSG = "http.status_msg"
 QUERY_STRING = "http.query.string"
 RETRIES_REMAIN = "http.retries_remain"

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools >= 40.6.0", "setuptools_scm[toml] >=4,<6.1", "cython", "cmake >= 3.14", "ninja"]
+requires = ["setuptools >= 40.6.0,<64", "setuptools_scm[toml] >=4,<6.1", "cython", "cmake >= 3.14", "ninja"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

releasenotes/notes/asm-fix-python-2-error-reading-ddwaf-rules-d3653031f2ba84ba.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    ASM: fix Python 2 error reading WAF rules.

releasenotes/notes/feat-collect-user-agent-630b2c10f83ac063.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Collect user agent in normalized span tag ``http.useragent``.

releasenotes/notes/fix-split-by-domain-requests-978e1f51a5b3f77d.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    requests: fix split-by-domain service name when multiple ``@`` signs are present in the url

releasenotes/notes/pymongo-add-service-name-config-option-f4eb81539b2d39dc.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    add DD_PYMONGO_SERVICE configuration

tests/appsec/test_processor.py

@@ -3,7 +3,9 @@
 
 import pytest
 
+from ddtrace.appsec._ddwaf import DDWaf
 from ddtrace.appsec.processor import AppSecSpanProcessor
+from ddtrace.appsec.processor import DEFAULT_RULES
 from ddtrace.appsec.processor import _transform_headers
 from ddtrace.constants import USER_KEEP
 from ddtrace.contrib.trace_utils import set_http_meta
@@ -196,3 +198,9 @@ def test_appsec_span_rate_limit(tracer):
         assert span1.get_tag("_dd.appsec.json") is not None
         assert span2.get_tag("_dd.appsec.json") is not None
         assert span3.get_tag("_dd.appsec.json") is None
+
+
+def test_ddwaf_not_raises_exception():
+    with open(DEFAULT_RULES) as rules:
+        rules_json = json.loads(rules.read())
+        DDWaf(rules_json)

tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2.py

@@ -48,7 +48,7 @@ async def test_template_rendering_snapshot_patched_server(
     app, _ = patched_app_tracer_jinja
     Pin.override(aiohttp_jinja2, tracer=tracer)
     # Ignore meta.http.url tag as the port is not fixed on the server
-    with snapshot_context(ignores=["meta.http.url"]):
+    with snapshot_context(ignores=["meta.http.url", "meta.http.useragent"]):
         client = await aiohttp_client(app)
         # it should trace a template rendering
         request = await client.request("GET", "/template/")

tests/contrib/django/test_django_appsec.py

@@ -1,5 +1,6 @@
 import json
 
+from ddtrace.ext import http
 from ddtrace.internal import _context
 from ddtrace.internal.compat import urlencode
 from tests.appsec.test_processor import RULES_GOOD_PATH
@@ -206,3 +207,9 @@ def test_django_path_params(client, test_spans, tracer):
     assert path_params["month"] == "july"
     # django>=1.8,<1.9 returns string instead int
     assert int(path_params["year"]) == 2022
+
+
+def test_django_useragent(client, test_spans, tracer):
+    client.get("/?a=1&b&c=d", HTTP_USER_AGENT="test/1.2.3")
+    root_span = test_spans.spans[0]
+    assert root_span.get_tag(http.USER_AGENT) == "test/1.2.3"

tests/contrib/django/test_django_snapshots.py

@@ -173,6 +173,7 @@ def test_psycopg_query_default(client, snapshot_context, psycopg2_patched):
         "31": (3, 1, 0) <= django.VERSION < (3, 2, 0),
         "3x": django.VERSION >= (3, 2, 0),
     },
+    ignores=["meta.http.useragent"],
     token_override="tests.contrib.django.test_django_snapshots.test_asgi_200",
 )
 @pytest.mark.parametrize("django_asgi", ["application", "channels_application"])
@@ -184,7 +185,7 @@ def test_asgi_200(django_asgi):
 
 
 @pytest.mark.skipif(django.VERSION < (3, 0, 0), reason="ASGI not supported in django<3")
-@snapshot()
+@snapshot(ignores=["meta.http.useragent"])
 def test_asgi_200_simple_app():
     # The path simple-asgi-app/ routes to an ASGI Application that is not traced
     # This test should generate an empty snapshot
@@ -195,7 +196,7 @@ def test_asgi_200_simple_app():
 
 
 @pytest.mark.skipif(django.VERSION < (3, 0, 0), reason="ASGI not supported in django<3")
-@snapshot()
+@snapshot(ignores=["meta.http.useragent"])
 def test_asgi_200_traced_simple_app():
     with daphne_client("channels_application") as client:
         resp = client.get("/traced-simple-asgi-app/")
@@ -205,7 +206,7 @@ def test_asgi_200_traced_simple_app():
 
 @pytest.mark.skipif(django.VERSION < (3, 0, 0), reason="ASGI not supported in django<3")
 @snapshot(
-    ignores=["meta.error.stack"],
+    ignores=["meta.error.stack", "meta.http.useragent"],
     variants={
         "30": (3, 0, 0) <= django.VERSION < (3, 1, 0),
         "31": (3, 1, 0) <= django.VERSION < (3, 2, 0),
@@ -219,7 +220,7 @@ def test_asgi_500():
 
 
 @pytest.mark.skipif(django.VERSION < (3, 2, 0), reason="Only want to test with latest Django")
-@snapshot(ignores=["meta.error.stack", "meta.http.request.headers.user-agent"])
+@snapshot(ignores=["meta.error.stack", "meta.http.request.headers.user-agent", "meta.http.useragent"])
 def test_appsec_enabled():
     with daphne_client("application", additional_env={"DD_APPSEC_ENABLED": "true"}) as client:
         resp = client.get("/")
@@ -228,7 +229,7 @@ def test_appsec_enabled():
 
 
 @pytest.mark.skipif(django.VERSION < (3, 2, 0), reason="Only want to test with latest Django")
-@snapshot(ignores=["meta.error.stack", "meta.http.request.headers.user-agent"])
+@snapshot(ignores=["meta.error.stack", "meta.http.request.headers.user-agent", "meta.http.useragent"])
 def test_appsec_enabled_attack():
     with daphne_client("application", additional_env={"DD_APPSEC_ENABLED": "true"}) as client:
         resp = client.get("/.git")
@@ -237,6 +238,7 @@ def test_appsec_enabled_attack():
 
 @pytest.mark.skipif(django.VERSION < (3, 0, 0), reason="ASGI not supported in django<3")
 @snapshot(
+    ignores=["meta.http.useragent"],
     variants={
         "30": (3, 0, 0) <= django.VERSION < (3, 1, 0),
         "31": (3, 1, 0) <= django.VERSION < (3, 2, 0),
@@ -253,6 +255,7 @@ def test_templates_enabled():
 
 @pytest.mark.skipif(django.VERSION < (3, 0, 0), reason="ASGI not supported in django<3")
 @snapshot(
+    ignores=["meta.http.useragent"],
     variants={
         "30": (3, 0, 0) <= django.VERSION < (3, 1, 0),
         "31": (3, 1, 0) <= django.VERSION < (3, 2, 0),

tests/contrib/flask/test_flask_appsec.py

@@ -1,5 +1,6 @@
 import json
 
+from ddtrace.ext import http
 from ddtrace.internal import _context
 from ddtrace.internal.compat import urlencode
 from tests.appsec.test_processor import RULES_GOOD_PATH
@@ -112,6 +113,12 @@ def test_flask_cookie(self):
             "testingcookie_key": ["testingcookie_value"]
         }
 
+    def test_flask_useragent(self):
+        self.client.get("/", headers={"User-Agent": "test/1.2.3"})
+        spans = self.pop_spans()
+        root_span = spans[0]
+        assert root_span.get_tag(http.USER_AGENT) == "test/1.2.3"
+
     def test_flask_body_urlencoded(self):
         with override_global_config(dict(_appsec_enabled=True)):
             self.tracer._appsec_enabled = True

tests/contrib/flask/test_flask_snapshot.py

@@ -13,6 +13,11 @@
 from tests.webclient import Client
 
 
+DEFAULT_HEADERS = {
+    "User-Agent": "python-httpx/x.xx.x",
+}
+
+
 @pytest.fixture
 def flask_port():
     # type: () -> str
@@ -94,12 +99,12 @@ def flask_client(flask_command, flask_env, flask_port):
 )
 def test_flask_200(flask_client):
     # type: (Client) -> None
-    assert flask_client.get("/").status_code == 200
+    assert flask_client.get("/", headers=DEFAULT_HEADERS).status_code == 200
 
 
 @pytest.mark.snapshot(
     ignores=["meta.flask.version"], variants={"220": flask_version >= (2, 2, 0), "": flask_version < (2, 2, 0)}
 )
 def test_flask_stream(flask_client):
     # type: (Client) -> None
-    assert flask_client.get("/stream", stream=True).status_code == 200
+    assert flask_client.get("/stream", headers=DEFAULT_HEADERS, stream=True).status_code == 200