From 0813424ebf1f232d2da0b31b7f149c83b7827fbb Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Thu, 19 Dec 2024 13:07:12 +0100 Subject: [PATCH 01/10] docs: small improvements on why section (#699) --- docs/index.md | 4 ++-- docs/why-logfire/opentelemetry.md | 3 ++- docs/why-logfire/pydantic.md | 6 +++++- docs/why-logfire/python-centric.md | 7 ++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5ece4dc32..2e8d75a15 100644 --- a/docs/index.md +++ b/docs/index.md @@ -97,8 +97,8 @@ logfire auth 2. Configure your **Logfire** environment - ```bash title="in the terminal:" - LOGFIRE_TOKEN= + ```bash title="In the terminal:" + export LOGFIRE_TOKEN= ``` !!! info "" diff --git a/docs/why-logfire/opentelemetry.md b/docs/why-logfire/opentelemetry.md index fbdaef7cd..02aa44b2f 100644 --- a/docs/why-logfire/opentelemetry.md +++ b/docs/why-logfire/opentelemetry.md @@ -8,6 +8,7 @@ For example, we can instrument a simple FastAPI app with just 2 lines of code: ```py title="main.py" hl_lines="8 9 10" from datetime import date + import logfire from pydantic import BaseModel from fastapi import FastAPI @@ -38,7 +39,7 @@ We'll need the [FastAPI contrib package](../integrations/web-frameworks/fastapi. ```bash pip install 'logfire[fastapi]' fastapi uvicorn # (1)! -uvicorn fastapi_example:app # (2)! +uvicorn main:app # (2)! ``` 1. Install the `logfire` package with the `fastapi` extra, FastAPI, and uvicorn. diff --git a/docs/why-logfire/pydantic.md b/docs/why-logfire/pydantic.md index 1b837abe3..13c15fdf1 100644 --- a/docs/why-logfire/pydantic.md +++ b/docs/why-logfire/pydantic.md @@ -1,11 +1,14 @@ # Pydantic integration -Logfire has an out-of-the-box Pydantic integration that lets you understand the data passing through your Pydantic models and get analytics on validations. For existing Pydantic users, it delivers unparalleled insights into your usage of Pydantic models. +**Logfire** has an out-of-the-box **Pydantic** integration that lets you understand the data +passing through your Pydantic models and get analytics on validations. For existing Pydantic users, +it delivers unparalleled insights into your usage of Pydantic models. We can record Pydantic models directly: ```py from datetime import date + import logfire from pydantic import BaseModel @@ -28,6 +31,7 @@ Or we can record information about validations automatically: ```py from datetime import date + import logfire from pydantic import BaseModel diff --git a/docs/why-logfire/python-centric.md b/docs/why-logfire/python-centric.md index a6612931d..c5de4b9da 100644 --- a/docs/why-logfire/python-centric.md +++ b/docs/why-logfire/python-centric.md @@ -1,7 +1,9 @@ # Python-centric insights :material-snake: -Pydantic Logfire automatically instruments your code for minimal manual effort, provides exceptional insights into async code, offers detailed performance analytics, and displays Python objects the same as the interpreter. Pydantic Logfire gives you a clearer view into how your Python is running than any other observability tool. - +**Pydantic Logfire** automatically instruments your code for minimal manual effort, provides +exceptional insights into async code, offers detailed performance analytics, and displays Python +objects the same as the interpreter. **Pydantic Logfire** gives you a clearer view into how your +Python is running than any other observability tool. ## Rich display of Python objects @@ -9,7 +11,6 @@ Pydantic Logfire automatically instruments your code for minimal manual effort, In this example, you can see the parameters passed to a FastAPI endpoint formatted as a Python object. - ## Profiling Python code ![Logfire Auto-tracing screenshot](../images/logfire-screenshot-autotracing.png) From cac58a93f12e7210ccb78b909ffd5cbe92d536fd Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Thu, 19 Dec 2024 17:54:09 +0100 Subject: [PATCH 02/10] docs: set different logger level (#698) --- docs/integrations/logging.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/integrations/logging.md b/docs/integrations/logging.md index 4b80aeb50..dfb8e5e6d 100644 --- a/docs/integrations/logging.md +++ b/docs/integrations/logging.md @@ -21,4 +21,23 @@ logger.error("Hello %s!", "Fred") # 10:05:06.855 Hello Fred! ``` +## Oh no! Too many logs from... + +A common issue with logging is that it can be **too verbose**... Right? :sweat_smile: + +Don't worry! We are here to help you. + +In those cases, you can set the log level to a higher value to suppress logs that are less important. +Let's see an example with the [`apscheduler`](https://apscheduler.readthedocs.io/en/3.x/) logger: + +```py title="main.py" +import logging + +logger = logging.getLogger("apscheduler") +logger.setLevel(logging.WARNING) +``` + +In this example, we set the log level of the `apscheduler` logger to `WARNING`, which means that +only logs with a level of `WARNING` or higher will be emitted. + [logging]: https://docs.python.org/3/library/logging.html From 930fa8776bc9c92f29768dc2f6674b2be81b55e3 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Fri, 20 Dec 2024 11:21:39 +0200 Subject: [PATCH 03/10] Capture httpx response JSON bodies (#700) Co-authored-by: Marcelo Trylesinski --- logfire/_internal/integrations/httpx.py | 93 +++++++++- logfire/_internal/main.py | 6 + tests/otel_integrations/test_httpx.py | 214 +++++++++++++++++++++++- 3 files changed, 303 insertions(+), 10 deletions(-) diff --git a/logfire/_internal/integrations/httpx.py b/logfire/_internal/integrations/httpx.py index e85dad4f9..f9f7e5a56 100644 --- a/logfire/_internal/integrations/httpx.py +++ b/logfire/_internal/integrations/httpx.py @@ -7,6 +7,8 @@ import httpx +from logfire.propagate import attach_context, get_context + try: from opentelemetry.instrumentation.httpx import ( AsyncRequestHook, @@ -64,6 +66,7 @@ def instrument_httpx( capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, + capture_response_json_body: bool, **kwargs: Unpack[ClientKwargs], ) -> None: ... @@ -74,6 +77,7 @@ def instrument_httpx( capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, + capture_response_json_body: bool, **kwargs: Unpack[AsyncClientKwargs], ) -> None: ... @@ -84,6 +88,7 @@ def instrument_httpx( capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, + capture_response_json_body: bool, **kwargs: Unpack[HTTPXInstrumentKwargs], ) -> None: ... @@ -94,6 +99,7 @@ def instrument_httpx( capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, + capture_response_json_body: bool, **kwargs: Any, ) -> None: """Instrument the `httpx` module so that spans are automatically created for each request. @@ -108,6 +114,7 @@ def instrument_httpx( del kwargs # make sure only final_kwargs is used instrumentor = HTTPXClientInstrumentor() + logfire_instance = logfire_instance.with_settings(custom_scope_suffix='httpx') if client is None: request_hook = cast('RequestHook | None', final_kwargs.get('request_hook')) @@ -117,11 +124,15 @@ def instrument_httpx( final_kwargs['request_hook'] = make_request_hook( request_hook, capture_request_headers, capture_request_json_body ) - final_kwargs['response_hook'] = make_response_hook(response_hook, capture_response_headers) + final_kwargs['response_hook'] = make_response_hook( + response_hook, capture_response_headers, capture_response_json_body, logfire_instance + ) final_kwargs['async_request_hook'] = make_async_request_hook( async_request_hook, capture_request_headers, capture_request_json_body ) - final_kwargs['async_response_hook'] = make_async_response_hook(async_response_hook, capture_response_headers) + final_kwargs['async_response_hook'] = make_async_response_hook( + async_response_hook, capture_response_headers, capture_response_json_body, logfire_instance + ) instrumentor.instrument(**final_kwargs) else: @@ -130,13 +141,17 @@ def instrument_httpx( response_hook = cast('ResponseHook | AsyncResponseHook | None', final_kwargs.get('response_hook')) request_hook = make_async_request_hook(request_hook, capture_request_headers, capture_request_json_body) - response_hook = make_async_response_hook(response_hook, capture_response_headers) + response_hook = make_async_response_hook( + response_hook, capture_response_headers, capture_response_json_body, logfire_instance + ) else: request_hook = cast('RequestHook | None', final_kwargs.get('request_hook')) response_hook = cast('ResponseHook | None', final_kwargs.get('response_hook')) request_hook = make_request_hook(request_hook, capture_request_headers, capture_request_json_body) - response_hook = make_response_hook(response_hook, capture_response_headers) + response_hook = make_response_hook( + response_hook, capture_response_headers, capture_response_json_body, logfire_instance + ) tracer_provider = final_kwargs['tracer_provider'] instrumentor.instrument_client(client, tracer_provider, request_hook, response_hook) @@ -176,34 +191,96 @@ async def new_hook(span: Span, request: RequestInfo) -> None: return new_hook -def make_response_hook(hook: ResponseHook | None, should_capture_headers: bool) -> ResponseHook | None: - if not should_capture_headers and not hook: +def make_response_hook( + hook: ResponseHook | None, should_capture_headers: bool, should_capture_json: bool, logfire_instance: Logfire +) -> ResponseHook | None: + if not should_capture_headers and not should_capture_json and not hook: return None def new_hook(span: Span, request: RequestInfo, response: ResponseInfo) -> None: with handle_internal_errors(): if should_capture_headers: capture_response_headers(span, response) + if should_capture_json: + capture_response_json(logfire_instance, response, False) run_hook(hook, span, request, response) return new_hook def make_async_response_hook( - hook: ResponseHook | AsyncResponseHook | None, should_capture_headers: bool + hook: ResponseHook | AsyncResponseHook | None, + should_capture_headers: bool, + should_capture_json: bool, + logfire_instance: Logfire, ) -> AsyncResponseHook | None: - if not should_capture_headers and not hook: + if not should_capture_headers and not should_capture_json and not hook: return None async def new_hook(span: Span, request: RequestInfo, response: ResponseInfo) -> None: with handle_internal_errors(): if should_capture_headers: capture_response_headers(span, response) + if should_capture_json: + capture_response_json(logfire_instance, response, True) await run_async_hook(hook, span, request, response) return new_hook +def capture_response_json(logfire_instance: Logfire, response_info: ResponseInfo, is_async: bool) -> None: + headers = cast('httpx.Headers', response_info.headers) + if not headers.get('content-type', '').lower().startswith('application/json'): + return + + frame = inspect.currentframe().f_back.f_back # type: ignore + while frame: + response = frame.f_locals.get('response') + frame = frame.f_back + if isinstance(response, httpx.Response): # pragma: no branch + break + else: # pragma: no cover + return + + ctx = get_context() + attr_name = 'http.response.body.json' + + if is_async: # these two branches should be kept almost identical + original_aread = response.aread + + async def aread(*args: Any, **kwargs: Any): + try: + # Only log the body the first time it's read + return response.content + except httpx.ResponseNotRead: + pass + with attach_context(ctx), logfire_instance.span('Reading response body') as span: + content = await original_aread(*args, **kwargs) + span.set_attribute(attr_name, {}) # Set the JSON schema + # Set the attribute to the raw text so that the backend can parse it + span._span.set_attribute(attr_name, response.text) # type: ignore + return content + + response.aread = aread + else: + original_read = response.read + + def read(*args: Any, **kwargs: Any): + try: + # Only log the body the first time it's read + return response.content + except httpx.ResponseNotRead: + pass + with attach_context(ctx), logfire_instance.span('Reading response body') as span: + content = original_read(*args, **kwargs) + span.set_attribute(attr_name, {}) # Set the JSON schema + # Set the attribute to the raw text so that the backend can parse it + span._span.set_attribute(attr_name, response.text) # type: ignore + return content + + response.read = read + + async def run_async_hook(hook: Callable[P, Any] | None, *args: P.args, **kwargs: P.kwargs) -> None: if hook: result = hook(*args, **kwargs) diff --git a/logfire/_internal/main.py b/logfire/_internal/main.py index ed5d9e678..c0da4ad93 100644 --- a/logfire/_internal/main.py +++ b/logfire/_internal/main.py @@ -1165,6 +1165,7 @@ def instrument_httpx( capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, + capture_response_json_body: bool = False, **kwargs: Unpack[ClientKwargs], ) -> None: ... @@ -1175,6 +1176,7 @@ def instrument_httpx( capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, + capture_response_json_body: bool = False, **kwargs: Unpack[AsyncClientKwargs], ) -> None: ... @@ -1185,6 +1187,7 @@ def instrument_httpx( capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, + capture_response_json_body: bool = False, **kwargs: Unpack[HTTPXInstrumentKwargs], ) -> None: ... @@ -1194,6 +1197,7 @@ def instrument_httpx( capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, + capture_response_json_body: bool = False, **kwargs: Any, ) -> None: """Instrument the `httpx` module so that spans are automatically created for each request. @@ -1210,6 +1214,7 @@ def instrument_httpx( capture_request_headers: Set to `True` to capture all request headers. capture_response_headers: Set to `True` to capture all response headers. capture_request_json_body: Set to `True` to capture the request JSON body. + capture_response_json_body: Set to `True` to capture the response JSON body. **kwargs: Additional keyword arguments to pass to the OpenTelemetry `instrument` method, for future compatibility. """ from .integrations.httpx import instrument_httpx @@ -1221,6 +1226,7 @@ def instrument_httpx( capture_request_headers, capture_response_headers, capture_request_json_body=capture_request_json_body, + capture_response_json_body=capture_response_json_body, **kwargs, ) diff --git a/tests/otel_integrations/test_httpx.py b/tests/otel_integrations/test_httpx.py index 7a51f8e50..56eec775d 100644 --- a/tests/otel_integrations/test_httpx.py +++ b/tests/otel_integrations/test_httpx.py @@ -7,7 +7,7 @@ import httpx import pytest -from dirty_equals import IsDict +from dirty_equals import IsDict, IsStr from httpx import Request from inline_snapshot import snapshot from opentelemetry.instrumentation.httpx import RequestInfo, ResponseInfo @@ -24,7 +24,7 @@ # without needing to actually make a network request def create_transport() -> httpx.MockTransport: def handler(request: Request): - return httpx.Response(200, headers=request.headers) + return httpx.Response(200, headers=request.headers, stream=httpx.ByteStream(b'{"good": "response"}')) return httpx.MockTransport(handler) @@ -348,3 +348,213 @@ def test_missing_opentelemetry_dependency() -> None: You can install this with: pip install 'logfire[httpx]'\ """) + + +def test_httpx_client_capture_full(exporter: TestExporter): + with check_traceparent_header() as checker: + with httpx.Client(transport=create_transport()) as client: + logfire.instrument_httpx( + client, + capture_request_headers=True, + capture_request_json_body=True, + capture_response_headers=True, + capture_response_json_body=True, + ) + response = client.post('https://example.org/', json={'hello': 'world'}) + checker(response) + assert response.json() == {'good': 'response'} + assert response.read() == b'{"good": "response"}' + + assert exporter.exported_spans_as_dict() == snapshot( + [ + { + 'name': 'POST', + 'context': {'trace_id': 1, 'span_id': 3, 'is_remote': False}, + 'parent': {'trace_id': 1, 'span_id': 1, 'is_remote': False}, + 'start_time': 2000000000, + 'end_time': 3000000000, + 'attributes': { + 'http.method': 'POST', + 'http.request.method': 'POST', + 'http.url': 'https://example.org/', + 'url.full': 'https://example.org/', + 'http.host': 'example.org', + 'server.address': 'example.org', + 'network.peer.address': 'example.org', + 'logfire.span_type': 'span', + 'logfire.msg': 'POST /', + 'http.request.header.host': ('example.org',), + 'http.request.header.accept': ('*/*',), + 'http.request.header.accept-encoding': ('gzip, deflate',), + 'http.request.header.connection': ('keep-alive',), + 'http.request.header.user-agent': (IsStr(),), + 'http.request.header.content-length': (IsStr(),), + 'http.request.header.content-type': ('application/json',), + 'logfire.json_schema': '{"type":"object","properties":{"http.request.body.json":{"type":"object"}}}', + 'http.request.body.json': '{"hello":"world"}', + 'http.status_code': 200, + 'http.response.status_code': 200, + 'http.flavor': '1.1', + 'network.protocol.version': '1.1', + 'http.response.header.host': ('example.org',), + 'http.response.header.accept': ('*/*',), + 'http.response.header.accept-encoding': ('gzip, deflate',), + 'http.response.header.connection': ('keep-alive',), + 'http.response.header.user-agent': (IsStr(),), + 'http.response.header.content-length': (IsStr(),), + 'http.response.header.content-type': ('application/json',), + 'http.response.header.traceparent': ('00-00000000000000000000000000000001-0000000000000003-01',), + 'http.target': '/', + }, + }, + { + 'name': 'Reading response body', + 'context': {'trace_id': 1, 'span_id': 5, 'is_remote': False}, + 'parent': {'trace_id': 1, 'span_id': 3, 'is_remote': True}, + 'start_time': 4000000000, + 'end_time': 5000000000, + 'attributes': { + 'code.filepath': 'test_httpx.py', + 'code.function': 'test_httpx_client_capture_full', + 'code.lineno': 123, + 'logfire.msg_template': 'Reading response body', + 'logfire.msg': 'Reading response body', + 'logfire.span_type': 'span', + 'http.response.body.json': '{"good": "response"}', + 'logfire.json_schema': '{"type":"object","properties":{"http.response.body.json":{"type":"object"}}}', + }, + }, + { + 'name': 'test span', + 'context': {'trace_id': 1, 'span_id': 1, 'is_remote': False}, + 'parent': None, + 'start_time': 1000000000, + 'end_time': 6000000000, + 'attributes': { + 'code.filepath': 'test_httpx.py', + 'code.function': 'check_traceparent_header', + 'code.lineno': 123, + 'logfire.msg_template': 'test span', + 'logfire.msg': 'test span', + 'logfire.span_type': 'span', + }, + }, + ] + ) + + +async def test_async_httpx_client_capture_full(exporter: TestExporter): + with check_traceparent_header() as checker: + async with httpx.AsyncClient(transport=create_transport()) as client: + logfire.instrument_httpx( + client, + capture_request_headers=True, + capture_request_json_body=True, + capture_response_headers=True, + capture_response_json_body=True, + ) + response = await client.post('https://example.org/', json={'hello': 'world'}) + checker(response) + assert response.json() == {'good': 'response'} + assert await response.aread() == b'{"good": "response"}' + + assert exporter.exported_spans_as_dict() == snapshot( + [ + { + 'name': 'POST', + 'context': {'trace_id': 1, 'span_id': 3, 'is_remote': False}, + 'parent': {'trace_id': 1, 'span_id': 1, 'is_remote': False}, + 'start_time': 2000000000, + 'end_time': 3000000000, + 'attributes': { + 'http.method': 'POST', + 'http.request.method': 'POST', + 'http.url': 'https://example.org/', + 'url.full': 'https://example.org/', + 'http.host': 'example.org', + 'server.address': 'example.org', + 'network.peer.address': 'example.org', + 'logfire.span_type': 'span', + 'logfire.msg': 'POST /', + 'http.request.header.host': ('example.org',), + 'http.request.header.accept': ('*/*',), + 'http.request.header.accept-encoding': ('gzip, deflate',), + 'http.request.header.connection': ('keep-alive',), + 'http.request.header.user-agent': (IsStr(),), + 'http.request.header.content-length': (IsStr(),), + 'http.request.header.content-type': ('application/json',), + 'logfire.json_schema': '{"type":"object","properties":{"http.request.body.json":{"type":"object"}}}', + 'http.request.body.json': '{"hello":"world"}', + 'http.status_code': 200, + 'http.response.status_code': 200, + 'http.flavor': '1.1', + 'network.protocol.version': '1.1', + 'http.response.header.host': ('example.org',), + 'http.response.header.accept': ('*/*',), + 'http.response.header.accept-encoding': ('gzip, deflate',), + 'http.response.header.connection': ('keep-alive',), + 'http.response.header.user-agent': (IsStr(),), + 'http.response.header.content-length': (IsStr(),), + 'http.response.header.content-type': ('application/json',), + 'http.response.header.traceparent': ('00-00000000000000000000000000000001-0000000000000003-01',), + 'http.target': '/', + }, + }, + { + 'name': 'Reading response body', + 'context': {'trace_id': 1, 'span_id': 5, 'is_remote': False}, + 'parent': {'trace_id': 1, 'span_id': 3, 'is_remote': True}, + 'start_time': 4000000000, + 'end_time': 5000000000, + 'attributes': { + 'code.filepath': 'test_httpx.py', + 'code.function': 'test_async_httpx_client_capture_full', + 'code.lineno': 123, + 'logfire.msg_template': 'Reading response body', + 'logfire.msg': 'Reading response body', + 'logfire.span_type': 'span', + 'http.response.body.json': '{"good": "response"}', + 'logfire.json_schema': '{"type":"object","properties":{"http.response.body.json":{"type":"object"}}}', + }, + }, + { + 'name': 'test span', + 'context': {'trace_id': 1, 'span_id': 1, 'is_remote': False}, + 'parent': None, + 'start_time': 1000000000, + 'end_time': 6000000000, + 'attributes': { + 'code.filepath': 'test_httpx.py', + 'code.function': 'check_traceparent_header', + 'code.lineno': 123, + 'logfire.msg_template': 'test span', + 'logfire.msg': 'test span', + 'logfire.span_type': 'span', + }, + }, + ] + ) + + +def test_httpx_client_capture_json_response_checks_header(exporter: TestExporter): + with httpx.Client(transport=create_transport()) as client: + logfire.instrument_httpx(client, capture_response_json_body=True) + response = client.post('https://example.org/', content=b'hello') + assert response.json() == {'good': 'response'} + + spans = exporter.exported_spans_as_dict() + assert len(spans) == 1 + assert spans[0]['name'] == 'POST' + assert 'http.response.body.json' not in str(spans) + + +async def test_httpx_async_client_capture_json_response_checks_header(exporter: TestExporter): + async with httpx.AsyncClient(transport=create_transport()) as client: + logfire.instrument_httpx(client, capture_response_json_body=True) + response = await client.post('https://example.org/', content=b'hello') + assert response.json() == {'good': 'response'} + + spans = exporter.exported_spans_as_dict() + assert len(spans) == 1 + assert spans[0]['name'] == 'POST' + assert 'http.response.body.json' not in str(spans) From 5eced1cf4b83f200b2ed34876986284dd3fadb75 Mon Sep 17 00:00:00 2001 From: David Montague <35119617+dmontagu@users.noreply.github.com> Date: Fri, 20 Dec 2024 02:24:34 -0700 Subject: [PATCH 04/10] Use `OPEN_SPANS` and custom `record_exception` logic for all spans (#696) Co-authored-by: Alex Hall --- logfire-api/logfire_api/_internal/config.pyi | 6 +- logfire-api/logfire_api/_internal/main.pyi | 10 +-- logfire-api/logfire_api/_internal/tracer.pyi | 15 +++- logfire/_internal/backfill.py | 6 +- logfire/_internal/config.py | 9 +- logfire/_internal/main.py | 94 ++++---------------- logfire/_internal/tracer.py | 76 ++++++++++++++-- 7 files changed, 110 insertions(+), 106 deletions(-) diff --git a/logfire-api/logfire_api/_internal/config.pyi b/logfire-api/logfire_api/_internal/config.pyi index 26c03b6a0..f7fa80468 100644 --- a/logfire-api/logfire_api/_internal/config.pyi +++ b/logfire-api/logfire_api/_internal/config.pyi @@ -12,11 +12,11 @@ from .exporters.quiet_metrics import QuietMetricExporter as QuietMetricExporter from .exporters.remove_pending import RemovePendingSpansExporter as RemovePendingSpansExporter from .exporters.test import TestExporter as TestExporter from .integrations.executors import instrument_executors as instrument_executors -from .main import FastLogfireSpan as FastLogfireSpan, Logfire as Logfire, LogfireSpan as LogfireSpan +from .main import Logfire as Logfire from .metrics import ProxyMeterProvider as ProxyMeterProvider from .scrubbing import BaseScrubber as BaseScrubber, NOOP_SCRUBBER as NOOP_SCRUBBER, Scrubber as Scrubber, ScrubbingOptions as ScrubbingOptions from .stack_info import warn_at_user_stacklevel as warn_at_user_stacklevel -from .tracer import PendingSpanProcessor as PendingSpanProcessor, ProxyTracerProvider as ProxyTracerProvider +from .tracer import OPEN_SPANS as OPEN_SPANS, PendingSpanProcessor as PendingSpanProcessor, ProxyTracerProvider as ProxyTracerProvider from .utils import SeededRandomIdGenerator as SeededRandomIdGenerator, UnexpectedResponse as UnexpectedResponse, ensure_data_dir_exists as ensure_data_dir_exists, handle_internal_errors as handle_internal_errors, read_toml_file as read_toml_file, suppress_instrumentation as suppress_instrumentation from _typeshed import Incomplete from dataclasses import dataclass @@ -30,9 +30,7 @@ from opentelemetry.sdk.trace.id_generator import IdGenerator from pathlib import Path from typing import Any, Callable, Literal, Sequence, TypedDict from typing_extensions import Self, Unpack -from weakref import WeakSet -OPEN_SPANS: WeakSet[LogfireSpan | FastLogfireSpan] CREDENTIALS_FILENAME: str COMMON_REQUEST_HEADERS: Incomplete PROJECT_NAME_PATTERN: str diff --git a/logfire-api/logfire_api/_internal/main.pyi b/logfire-api/logfire_api/_internal/main.pyi index 208cc278b..7d54718e2 100644 --- a/logfire-api/logfire_api/_internal/main.pyi +++ b/logfire-api/logfire_api/_internal/main.pyi @@ -6,9 +6,9 @@ import requests from . import async_ as async_ from ..version import VERSION as VERSION from .auto_trace import AutoTraceModule as AutoTraceModule, install_auto_tracing as install_auto_tracing -from .config import GLOBAL_CONFIG as GLOBAL_CONFIG, LogfireConfig as LogfireConfig, OPEN_SPANS as OPEN_SPANS +from .config import GLOBAL_CONFIG as GLOBAL_CONFIG, LogfireConfig as LogfireConfig from .config_params import PydanticPluginRecordValues as PydanticPluginRecordValues -from .constants import ATTRIBUTES_JSON_SCHEMA_KEY as ATTRIBUTES_JSON_SCHEMA_KEY, ATTRIBUTES_LOG_LEVEL_NUM_KEY as ATTRIBUTES_LOG_LEVEL_NUM_KEY, ATTRIBUTES_MESSAGE_KEY as ATTRIBUTES_MESSAGE_KEY, ATTRIBUTES_MESSAGE_TEMPLATE_KEY as ATTRIBUTES_MESSAGE_TEMPLATE_KEY, ATTRIBUTES_SAMPLE_RATE_KEY as ATTRIBUTES_SAMPLE_RATE_KEY, ATTRIBUTES_SPAN_TYPE_KEY as ATTRIBUTES_SPAN_TYPE_KEY, ATTRIBUTES_TAGS_KEY as ATTRIBUTES_TAGS_KEY, ATTRIBUTES_VALIDATION_ERROR_KEY as ATTRIBUTES_VALIDATION_ERROR_KEY, DISABLE_CONSOLE_KEY as DISABLE_CONSOLE_KEY, LEVEL_NUMBERS as LEVEL_NUMBERS, LevelName as LevelName, NULL_ARGS_KEY as NULL_ARGS_KEY, OTLP_MAX_INT_SIZE as OTLP_MAX_INT_SIZE, log_level_attributes as log_level_attributes +from .constants import ATTRIBUTES_JSON_SCHEMA_KEY as ATTRIBUTES_JSON_SCHEMA_KEY, ATTRIBUTES_LOG_LEVEL_NUM_KEY as ATTRIBUTES_LOG_LEVEL_NUM_KEY, ATTRIBUTES_MESSAGE_KEY as ATTRIBUTES_MESSAGE_KEY, ATTRIBUTES_MESSAGE_TEMPLATE_KEY as ATTRIBUTES_MESSAGE_TEMPLATE_KEY, ATTRIBUTES_SAMPLE_RATE_KEY as ATTRIBUTES_SAMPLE_RATE_KEY, ATTRIBUTES_SPAN_TYPE_KEY as ATTRIBUTES_SPAN_TYPE_KEY, ATTRIBUTES_TAGS_KEY as ATTRIBUTES_TAGS_KEY, DISABLE_CONSOLE_KEY as DISABLE_CONSOLE_KEY, LEVEL_NUMBERS as LEVEL_NUMBERS, LevelName as LevelName, NULL_ARGS_KEY as NULL_ARGS_KEY, OTLP_MAX_INT_SIZE as OTLP_MAX_INT_SIZE, log_level_attributes as log_level_attributes from .formatter import logfire_format as logfire_format, logfire_format_with_magic as logfire_format_with_magic from .instrument import instrument as instrument from .integrations.asgi import ASGIApp as ASGIApp, ASGIInstrumentKwargs as ASGIInstrumentKwargs @@ -29,7 +29,7 @@ from .json_encoder import logfire_json_dumps as logfire_json_dumps from .json_schema import JsonSchemaProperties as JsonSchemaProperties, attributes_json_schema as attributes_json_schema, attributes_json_schema_properties as attributes_json_schema_properties, create_json_schema as create_json_schema from .metrics import ProxyMeterProvider as ProxyMeterProvider from .stack_info import get_user_stack_info as get_user_stack_info -from .tracer import ProxyTracerProvider as ProxyTracerProvider +from .tracer import ProxyTracerProvider as ProxyTracerProvider, record_exception as record_exception, set_exception_status as set_exception_status from .utils import SysExcInfo as SysExcInfo, get_version as get_version, handle_internal_errors as handle_internal_errors, log_internal_error as log_internal_error, uniquify_sequence as uniquify_sequence from django.http import HttpRequest as HttpRequest, HttpResponse as HttpResponse from fastapi import FastAPI @@ -1034,7 +1034,7 @@ class LogfireSpan(ReadableSpan): def message(self) -> str: ... @message.setter def message(self, message: str): ... - def end(self) -> None: + def end(self, end_time: int | None = None) -> None: """Sets the current time as the span's end time. The span's end time is the wall time at which the operation finished. @@ -1093,7 +1093,7 @@ class NoopSpan: def is_recording(self) -> bool: ... AttributesValueType = TypeVar('AttributesValueType', bound=Any | otel_types.AttributeValue) -def user_attributes(attributes: dict[str, Any]) -> dict[str, otel_types.AttributeValue]: +def prepare_otlp_attributes(attributes: dict[str, Any]) -> dict[str, otel_types.AttributeValue]: """Prepare attributes for sending to OpenTelemetry. This will convert any non-OpenTelemetry compatible types to JSON. diff --git a/logfire-api/logfire_api/_internal/tracer.pyi b/logfire-api/logfire_api/_internal/tracer.pyi index 4dedab6e3..bf8f7e75b 100644 --- a/logfire-api/logfire_api/_internal/tracer.pyi +++ b/logfire-api/logfire_api/_internal/tracer.pyi @@ -1,6 +1,7 @@ import opentelemetry.trace as trace_api from .config import LogfireConfig as LogfireConfig -from .constants import ATTRIBUTES_MESSAGE_KEY as ATTRIBUTES_MESSAGE_KEY, ATTRIBUTES_PENDING_SPAN_REAL_PARENT_KEY as ATTRIBUTES_PENDING_SPAN_REAL_PARENT_KEY, ATTRIBUTES_SAMPLE_RATE_KEY as ATTRIBUTES_SAMPLE_RATE_KEY, ATTRIBUTES_SPAN_TYPE_KEY as ATTRIBUTES_SPAN_TYPE_KEY, PENDING_SPAN_NAME_SUFFIX as PENDING_SPAN_NAME_SUFFIX +from .constants import ATTRIBUTES_MESSAGE_KEY as ATTRIBUTES_MESSAGE_KEY, ATTRIBUTES_PENDING_SPAN_REAL_PARENT_KEY as ATTRIBUTES_PENDING_SPAN_REAL_PARENT_KEY, ATTRIBUTES_SAMPLE_RATE_KEY as ATTRIBUTES_SAMPLE_RATE_KEY, ATTRIBUTES_SPAN_TYPE_KEY as ATTRIBUTES_SPAN_TYPE_KEY, ATTRIBUTES_VALIDATION_ERROR_KEY as ATTRIBUTES_VALIDATION_ERROR_KEY, PENDING_SPAN_NAME_SUFFIX as PENDING_SPAN_NAME_SUFFIX, log_level_attributes as log_level_attributes +from .utils import handle_internal_errors as handle_internal_errors from _typeshed import Incomplete from dataclasses import dataclass from opentelemetry import context as context_api @@ -13,7 +14,9 @@ from opentelemetry.trace.status import Status, StatusCode from opentelemetry.util import types as otel_types from threading import Lock from typing import Any, Callable, Mapping, Sequence -from weakref import WeakKeyDictionary +from weakref import WeakKeyDictionary, WeakSet + +OPEN_SPANS: WeakSet[_LogfireWrappedSpan] @dataclass class ProxyTracerProvider(TracerProvider): @@ -32,11 +35,12 @@ class ProxyTracerProvider(TracerProvider): def resource(self) -> Resource: ... def force_flush(self, timeout_millis: int = 30000) -> bool: ... -@dataclass -class _MaybeDeterministicTimestampSpan(trace_api.Span, ReadableSpan): +@dataclass(eq=False) +class _LogfireWrappedSpan(trace_api.Span, ReadableSpan): """Span that overrides end() to use a timestamp generator if one was provided.""" span: Span ns_timestamp_generator: Callable[[], int] + def __post_init__(self) -> None: ... def end(self, end_time: int | None = None) -> None: ... def get_span_context(self) -> SpanContext: ... def set_attributes(self, attributes: dict[str, otel_types.AttributeValue]) -> None: ... @@ -85,3 +89,6 @@ def should_sample(span_context: SpanContext, attributes: Mapping[str, otel_types This is used to sample spans that are not sampled by the OTEL sampler. """ def get_sample_rate_from_attributes(attributes: otel_types.Attributes) -> float | None: ... +def record_exception(span: trace_api.Span, exception: BaseException, *, attributes: otel_types.Attributes = None, timestamp: int | None = None, escaped: bool = False) -> None: + """Similar to the OTEL SDK Span.record_exception method, with our own additions.""" +def set_exception_status(span: trace_api.Span, exception: BaseException): ... diff --git a/logfire/_internal/backfill.py b/logfire/_internal/backfill.py index 724d8009f..e4fd1189b 100644 --- a/logfire/_internal/backfill.py +++ b/logfire/_internal/backfill.py @@ -24,7 +24,7 @@ ) from .exporters.file import FileSpanExporter from .formatter import logfire_format -from .main import user_attributes +from .main import prepare_otlp_attributes from .scrubbing import Scrubber try: @@ -150,7 +150,7 @@ def write(self, data: Union[Log, Span]) -> None: ) else: parent_context = None # pragma: no cover - otlp_attributes = user_attributes(data.attributes) + otlp_attributes = prepare_otlp_attributes(data.attributes) if data.formatted_msg is None: # pragma: no cover formatted_message = logfire_format(data.msg_template, data.attributes, self.scrubber) @@ -196,7 +196,7 @@ def write(self, data: Union[Log, Span]) -> None: start_timestamp = data.start_timestamp if start_timestamp.tzinfo is None: # pragma: no branch start_timestamp = start_timestamp.replace(tzinfo=timezone.utc) - otlp_attributes = user_attributes(data.log_attributes) + otlp_attributes = prepare_otlp_attributes(data.log_attributes) if data.formatted_msg is None: # pragma: no branch formatted_message = logfire_format(data.msg_template, data.log_attributes, self.scrubber) else: # pragma: no cover diff --git a/logfire/_internal/config.py b/logfire/_internal/config.py index fbe70dd29..0fb86c8c6 100644 --- a/logfire/_internal/config.py +++ b/logfire/_internal/config.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, Any, Callable, Literal, Sequence, TypedDict, cast from urllib.parse import urljoin from uuid import uuid4 -from weakref import WeakSet import requests from opentelemetry import trace @@ -87,7 +86,7 @@ from .metrics import ProxyMeterProvider from .scrubbing import NOOP_SCRUBBER, BaseScrubber, Scrubber, ScrubbingOptions from .stack_info import warn_at_user_stacklevel -from .tracer import PendingSpanProcessor, ProxyTracerProvider +from .tracer import OPEN_SPANS, PendingSpanProcessor, ProxyTracerProvider from .utils import ( SeededRandomIdGenerator, UnexpectedResponse, @@ -98,10 +97,8 @@ ) if TYPE_CHECKING: - from .main import FastLogfireSpan, Logfire, LogfireSpan + from .main import Logfire -# NOTE: this WeakSet is the reason that FastLogfireSpan.__slots__ has a __weakref__ slot. -OPEN_SPANS: WeakSet[LogfireSpan | FastLogfireSpan] = WeakSet() CREDENTIALS_FILENAME = 'logfire_credentials.json' """Default base URL for the Logfire API.""" @@ -948,7 +945,7 @@ def _exit_open_spans(): # type: ignore[reportUnusedFunction] # pragma: no cove # Registering this callback here after the OTEL one means that this runs first. # Otherwise OTEL would log an error "Already shutdown, dropping span." for span in list(OPEN_SPANS): - span.__exit__(None, None, None) + span.end() self._initialized = True diff --git a/logfire/_internal/main.py b/logfire/_internal/main.py index c0da4ad93..f11b06bd5 100644 --- a/logfire/_internal/main.py +++ b/logfire/_internal/main.py @@ -4,7 +4,6 @@ import inspect import json import sys -import traceback import warnings from functools import cached_property from time import time @@ -26,15 +25,14 @@ import opentelemetry.trace as trace_api from opentelemetry.metrics import CallbackT, Counter, Histogram, UpDownCounter from opentelemetry.sdk.trace import ReadableSpan, Span -from opentelemetry.semconv.trace import SpanAttributes -from opentelemetry.trace import SpanContext, StatusCode, Tracer +from opentelemetry.trace import SpanContext, Tracer from opentelemetry.util import types as otel_types from typing_extensions import LiteralString, ParamSpec from ..version import VERSION from . import async_ from .auto_trace import AutoTraceModule, install_auto_tracing -from .config import GLOBAL_CONFIG, OPEN_SPANS, LogfireConfig +from .config import GLOBAL_CONFIG, LogfireConfig from .config_params import PydanticPluginRecordValues from .constants import ( ATTRIBUTES_JSON_SCHEMA_KEY, @@ -44,7 +42,6 @@ ATTRIBUTES_SAMPLE_RATE_KEY, ATTRIBUTES_SPAN_TYPE_KEY, ATTRIBUTES_TAGS_KEY, - ATTRIBUTES_VALIDATION_ERROR_KEY, DISABLE_CONSOLE_KEY, LEVEL_NUMBERS, NULL_ARGS_KEY, @@ -63,7 +60,7 @@ ) from .metrics import ProxyMeterProvider from .stack_info import get_user_stack_info -from .tracer import ProxyTracerProvider +from .tracer import ProxyTracerProvider, record_exception, set_exception_status from .utils import get_version, handle_internal_errors, log_internal_error, uniquify_sequence if TYPE_CHECKING: @@ -107,12 +104,6 @@ ExcInfo = Union[SysExcInfo, BaseException, bool, None] -try: - from pydantic import ValidationError -except ImportError: # pragma: no cover - ValidationError = None - - class Logfire: """The main logfire class.""" @@ -194,7 +185,7 @@ def _span( merged_attributes[ATTRIBUTES_MESSAGE_TEMPLATE_KEY] = msg_template merged_attributes[ATTRIBUTES_MESSAGE_KEY] = log_message - otlp_attributes = user_attributes(merged_attributes) + otlp_attributes = prepare_otlp_attributes(merged_attributes) if json_schema_properties := attributes_json_schema_properties(attributes): otlp_attributes[ATTRIBUTES_JSON_SCHEMA_KEY] = attributes_json_schema(json_schema_properties) @@ -250,7 +241,7 @@ def _instrument_span_with_args( attributes[ATTRIBUTES_MESSAGE_KEY] = logfire_format(msg_template, function_args, self._config.scrubber) if json_schema_properties := attributes_json_schema_properties(function_args): # pragma: no branch attributes[ATTRIBUTES_JSON_SCHEMA_KEY] = attributes_json_schema(json_schema_properties) - attributes.update(user_attributes(function_args)) + attributes.update(prepare_otlp_attributes(function_args)) return self._fast_span(name, attributes) except Exception: # pragma: no cover log_internal_error() @@ -687,7 +678,7 @@ def log( msg = merged_attributes[ATTRIBUTES_MESSAGE_KEY] = str(msg) msg_template = str(msg_template) - otlp_attributes = user_attributes(merged_attributes) + otlp_attributes = prepare_otlp_attributes(merged_attributes) otlp_attributes = { ATTRIBUTES_SPAN_TYPE_KEY: 'log', **log_level_attributes(level), @@ -726,12 +717,12 @@ def log( if isinstance(exc_info, tuple): exc_info = exc_info[1] if isinstance(exc_info, BaseException): - _record_exception(span, exc_info) + record_exception(span, exc_info) if otlp_attributes[ATTRIBUTES_LOG_LEVEL_NUM_KEY] >= LEVEL_NUMBERS['error']: # type: ignore # Set the status description to the exception message. # OTEL only lets us set the description when the status code is ERROR, # which we only want to do when the log level is error. - _set_exception_status(span, exc_info) + set_exception_status(span, exc_info) elif exc_info is not None: # pragma: no cover raise TypeError(f'Invalid type for exc_info: {exc_info.__class__.__name__}') @@ -1931,20 +1922,17 @@ def shutdown(self, timeout_millis: int = 30_000, flush: bool = True) -> bool: # class FastLogfireSpan: """A simple version of `LogfireSpan` optimized for auto-tracing.""" - # __weakref__ is needed for the OPEN_SPANS WeakSet. - __slots__ = ('_span', '_token', '__weakref__') + __slots__ = ('_span', '_token') def __init__(self, span: trace_api.Span) -> None: self._span = span self._token = context_api.attach(trace_api.set_span_in_context(self._span)) - OPEN_SPANS.add(self) def __enter__(self) -> FastLogfireSpan: return self @handle_internal_errors() def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: Any) -> None: - OPEN_SPANS.remove(self) context_api.detach(self._token) _exit_span(self._span, exc_value) self._span.end() @@ -1986,8 +1974,6 @@ def __enter__(self) -> LogfireSpan: if self._token is None: # pragma: no branch self._token = context_api.attach(trace_api.set_span_in_context(self._span)) - OPEN_SPANS.add(self) - return self @handle_internal_errors() @@ -1995,8 +1981,6 @@ def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseExceptio if self._token is None: # pragma: no cover return - OPEN_SPANS.remove(self) - context_api.detach(self._token) self._token = None @@ -2029,7 +2013,7 @@ def message(self) -> str: def message(self, message: str): self._set_attribute(ATTRIBUTES_MESSAGE_KEY, message) - def end(self) -> None: + def end(self, end_time: int | None = None) -> None: """Sets the current time as the span's end time. The span's end time is the wall time at which the operation finished. @@ -2047,7 +2031,7 @@ def end(self) -> None: ATTRIBUTES_JSON_SCHEMA_KEY, attributes_json_schema(self._json_schema_properties) ) - self._span.end() + self._span.end(end_time) @handle_internal_errors() def set_attribute(self, key: str, value: Any) -> None: @@ -2093,7 +2077,7 @@ def record_exception( if not self._span.is_recording(): return - _record_exception( + record_exception( self._span, exception, attributes=attributes, @@ -2187,61 +2171,13 @@ def _exit_span(span: trace_api.Span, exception: BaseException | None) -> None: # record exception if present # isinstance is to ignore BaseException if isinstance(exception, Exception): - _record_exception(span, exception, escaped=True) - - -def _set_exception_status(span: trace_api.Span, exception: BaseException): - span.set_status( - trace_api.Status( - status_code=StatusCode.ERROR, - description=f'{exception.__class__.__name__}: {exception}', - ) - ) - - -@handle_internal_errors() -def _record_exception( - span: trace_api.Span, - exception: BaseException, - *, - attributes: otel_types.Attributes = None, - timestamp: int | None = None, - escaped: bool = False, -) -> None: - """Similar to the OTEL SDK Span.record_exception method, with our own additions.""" - # From https://opentelemetry.io/docs/specs/semconv/attributes-registry/exception/ - # `escaped=True` means that the exception is escaping the scope of the span. - # This means we know that the exception hasn't been handled, - # so we can set the OTEL status and the log level to error. - if escaped: - _set_exception_status(span, exception) - span.set_attributes(log_level_attributes('error')) - - attributes = {**(attributes or {})} - if ValidationError is not None and isinstance(exception, ValidationError): - # insert a more detailed breakdown of pydantic errors - try: - err_json = exception.json(include_url=False) - except TypeError: # pragma: no cover - # pydantic v1 - err_json = exception.json() - span.set_attribute(ATTRIBUTES_VALIDATION_ERROR_KEY, err_json) - attributes[ATTRIBUTES_VALIDATION_ERROR_KEY] = err_json - - if exception is not sys.exc_info()[1]: - # OTEL's record_exception uses `traceback.format_exc()` which is for the current exception, - # ignoring the passed exception. - # So we override the stacktrace attribute with the correct one. - stacktrace = ''.join(traceback.format_exception(type(exception), exception, exception.__traceback__)) - attributes[SpanAttributes.EXCEPTION_STACKTRACE] = stacktrace - - span.record_exception(exception, attributes=attributes, timestamp=timestamp, escaped=escaped) + record_exception(span, exception, escaped=True) AttributesValueType = TypeVar('AttributesValueType', bound=Union[Any, otel_types.AttributeValue]) -def user_attributes(attributes: dict[str, Any]) -> dict[str, otel_types.AttributeValue]: +def prepare_otlp_attributes(attributes: dict[str, Any]) -> dict[str, otel_types.AttributeValue]: """Prepare attributes for sending to OpenTelemetry. This will convert any non-OpenTelemetry compatible types to JSON. @@ -2288,7 +2224,7 @@ def set_user_attributes_on_raw_span(span: Span, attributes: dict[str, Any]) -> N if not span.is_recording(): return - otlp_attributes = user_attributes(attributes) + otlp_attributes = prepare_otlp_attributes(attributes) if json_schema_properties := attributes_json_schema_properties(attributes): # pragma: no branch existing_properties = JsonSchemaProperties({}) existing_json_schema_str = (span.attributes or {}).get(ATTRIBUTES_JSON_SCHEMA_KEY) diff --git a/logfire/_internal/tracer.py b/logfire/_internal/tracer.py index d4b86979f..5d7ad8410 100644 --- a/logfire/_internal/tracer.py +++ b/logfire/_internal/tracer.py @@ -1,9 +1,11 @@ from __future__ import annotations +import sys +import traceback from dataclasses import dataclass, field from threading import Lock from typing import TYPE_CHECKING, Any, Callable, Mapping, Sequence, cast -from weakref import WeakKeyDictionary +from weakref import WeakKeyDictionary, WeakSet import opentelemetry.trace as trace_api from opentelemetry import context as context_api @@ -17,6 +19,7 @@ ) from opentelemetry.sdk.trace.id_generator import IdGenerator from opentelemetry.semconv.resource import ResourceAttributes +from opentelemetry.semconv.trace import SpanAttributes from opentelemetry.trace import Link, NonRecordingSpan, Span, SpanContext, SpanKind, Tracer, TracerProvider from opentelemetry.trace.propagation import get_current_span from opentelemetry.trace.status import Status, StatusCode @@ -27,12 +30,23 @@ ATTRIBUTES_PENDING_SPAN_REAL_PARENT_KEY, ATTRIBUTES_SAMPLE_RATE_KEY, ATTRIBUTES_SPAN_TYPE_KEY, + ATTRIBUTES_VALIDATION_ERROR_KEY, PENDING_SPAN_NAME_SUFFIX, + log_level_attributes, ) +from .utils import handle_internal_errors if TYPE_CHECKING: from .config import LogfireConfig +try: + from pydantic import ValidationError +except ImportError: # pragma: no cover + ValidationError = None + + +OPEN_SPANS: WeakSet[_LogfireWrappedSpan] = WeakSet() + @dataclass class ProxyTracerProvider(TracerProvider): @@ -100,14 +114,18 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: return True # pragma: no cover -@dataclass -class _MaybeDeterministicTimestampSpan(trace_api.Span, ReadableSpan): +@dataclass(eq=False) +class _LogfireWrappedSpan(trace_api.Span, ReadableSpan): """Span that overrides end() to use a timestamp generator if one was provided.""" span: Span ns_timestamp_generator: Callable[[], int] + def __post_init__(self): + OPEN_SPANS.add(self) + def end(self, end_time: int | None = None) -> None: + OPEN_SPANS.discard(self) self.span.end(end_time or self.ns_timestamp_generator()) def get_span_context(self) -> SpanContext: @@ -151,7 +169,7 @@ def record_exception( escaped: bool = False, ) -> None: timestamp = timestamp or self.ns_timestamp_generator() - return self.span.record_exception(exception, attributes, timestamp, escaped) + record_exception(self.span, exception, attributes=attributes, timestamp=timestamp, escaped=escaped) if not TYPE_CHECKING: # pragma: no branch # for ReadableSpan @@ -211,7 +229,7 @@ def start_span( ), ) ) - return _MaybeDeterministicTimestampSpan( + return _LogfireWrappedSpan( span, ns_timestamp_generator=self.provider.config.advanced.ns_timestamp_generator, ) @@ -314,3 +332,51 @@ def get_sample_rate_from_attributes(attributes: otel_types.Attributes) -> float if not attributes: # pragma: no cover return None return cast('float | None', attributes.get(ATTRIBUTES_SAMPLE_RATE_KEY)) + + +@handle_internal_errors() +def record_exception( + span: trace_api.Span, + exception: BaseException, + *, + attributes: otel_types.Attributes = None, + timestamp: int | None = None, + escaped: bool = False, +) -> None: + """Similar to the OTEL SDK Span.record_exception method, with our own additions.""" + # From https://opentelemetry.io/docs/specs/semconv/attributes-registry/exception/ + # `escaped=True` means that the exception is escaping the scope of the span. + # This means we know that the exception hasn't been handled, + # so we can set the OTEL status and the log level to error. + if escaped: + set_exception_status(span, exception) + span.set_attributes(log_level_attributes('error')) + + attributes = {**(attributes or {})} + if ValidationError is not None and isinstance(exception, ValidationError): + # insert a more detailed breakdown of pydantic errors + try: + err_json = exception.json(include_url=False) + except TypeError: # pragma: no cover + # pydantic v1 + err_json = exception.json() + span.set_attribute(ATTRIBUTES_VALIDATION_ERROR_KEY, err_json) + attributes[ATTRIBUTES_VALIDATION_ERROR_KEY] = err_json + + if exception is not sys.exc_info()[1]: + # OTEL's record_exception uses `traceback.format_exc()` which is for the current exception, + # ignoring the passed exception. + # So we override the stacktrace attribute with the correct one. + stacktrace = ''.join(traceback.format_exception(type(exception), exception, exception.__traceback__)) + attributes[SpanAttributes.EXCEPTION_STACKTRACE] = stacktrace + + span.record_exception(exception, attributes=attributes, timestamp=timestamp, escaped=escaped) + + +def set_exception_status(span: trace_api.Span, exception: BaseException): + span.set_status( + trace_api.Status( + status_code=StatusCode.ERROR, + description=f'{exception.__class__.__name__}: {exception}', + ) + ) From 5349ca718970be7d53925fbfb1276eb0df03c0d1 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Fri, 20 Dec 2024 11:48:01 +0200 Subject: [PATCH 05/10] Prevent logged warning when span is double-ended at interpreter shutdown (#704) --- logfire/_internal/config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/logfire/_internal/config.py b/logfire/_internal/config.py index 0fb86c8c6..09df01941 100644 --- a/logfire/_internal/config.py +++ b/logfire/_internal/config.py @@ -944,8 +944,15 @@ def _exit_open_spans(): # type: ignore[reportUnusedFunction] # pragma: no cove # OTEL registers its own atexit callback in the tracer/meter providers to shut them down. # Registering this callback here after the OTEL one means that this runs first. # Otherwise OTEL would log an error "Already shutdown, dropping span." + # The reason that spans may be lingering open is that they're in suspended generator frames. + # Apart from here, they will be ended when the generator is garbage collected + # as the interpreter shuts down, but that's too late. for span in list(OPEN_SPANS): + # TODO maybe we should be recording something about what happened here? span.end() + # Interpreter shutdown may trigger another call to .end(), + # which would log a warning "Calling end() on an ended span." + span.end = lambda *_, **__: None # type: ignore self._initialized = True From 0f46d55513cfb37bc6d9ce74fe4d9caba7b5cda9 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Fri, 20 Dec 2024 12:00:35 +0200 Subject: [PATCH 06/10] Release v2.9.0 (#706) --- CHANGELOG.md | 6 ++++++ .../logfire_api/_internal/integrations/httpx.pyi | 12 +++++++----- logfire-api/logfire_api/_internal/main.pyi | 6 +++--- logfire-api/pyproject.toml | 2 +- pyproject.toml | 2 +- uv.lock | 6 +++--- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d86a0b8ab..6e10161b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Release Notes +## [v2.9.0] (2024-12-20) + +* Capture httpx response JSON bodies by @alexmojaki in [#700](https://github.com/pydantic/logfire/pull/700) +* Use end-at-shutdown and custom `record_exception` logic for all spans by @dmontagu in [#696](https://github.com/pydantic/logfire/pull/696) + ## [v2.8.0] (2024-12-18) * Add `capture_(request|response)_headers` ([#671](https://github.com/pydantic/logfire/pull/671)) and `capture_request_json_body` ([#682](https://github.com/pydantic/logfire/pull/682)) to `instrument_httpx` by @Kludex @@ -480,3 +485,4 @@ First release from new repo! [v2.7.0]: https://github.com/pydantic/logfire/compare/v2.6.2...v2.7.0 [v2.7.1]: https://github.com/pydantic/logfire/compare/v2.7.0...v2.7.1 [v2.8.0]: https://github.com/pydantic/logfire/compare/v2.7.1...v2.8.0 +[v2.9.0]: https://github.com/pydantic/logfire/compare/v2.8.0...v2.9.0 diff --git a/logfire-api/logfire_api/_internal/integrations/httpx.pyi b/logfire-api/logfire_api/_internal/integrations/httpx.pyi index 22f5f2951..58b6e67a7 100644 --- a/logfire-api/logfire_api/_internal/integrations/httpx.pyi +++ b/logfire-api/logfire_api/_internal/integrations/httpx.pyi @@ -2,6 +2,7 @@ import httpx from logfire import Logfire as Logfire from logfire._internal.main import set_user_attributes_on_raw_span as set_user_attributes_on_raw_span from logfire._internal.utils import handle_internal_errors as handle_internal_errors +from logfire.propagate import attach_context as attach_context, get_context as get_context from opentelemetry.instrumentation.httpx import AsyncRequestHook, AsyncResponseHook, RequestHook, RequestInfo, ResponseHook, ResponseInfo from opentelemetry.trace import Span from typing import Any, Callable, Literal, ParamSpec, TypeVar, TypedDict, Unpack, overload @@ -29,15 +30,16 @@ AsyncHook = TypeVar('AsyncHook', AsyncRequestHook, AsyncResponseHook) P = ParamSpec('P') @overload -def instrument_httpx(logfire_instance: Logfire, client: httpx.Client, capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, **kwargs: Unpack[ClientKwargs]) -> None: ... +def instrument_httpx(logfire_instance: Logfire, client: httpx.Client, capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, capture_response_json_body: bool, **kwargs: Unpack[ClientKwargs]) -> None: ... @overload -def instrument_httpx(logfire_instance: Logfire, client: httpx.AsyncClient, capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, **kwargs: Unpack[AsyncClientKwargs]) -> None: ... +def instrument_httpx(logfire_instance: Logfire, client: httpx.AsyncClient, capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, capture_response_json_body: bool, **kwargs: Unpack[AsyncClientKwargs]) -> None: ... @overload -def instrument_httpx(logfire_instance: Logfire, client: None, capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, **kwargs: Unpack[HTTPXInstrumentKwargs]) -> None: ... +def instrument_httpx(logfire_instance: Logfire, client: None, capture_request_headers: bool, capture_response_headers: bool, capture_request_json_body: bool, capture_response_json_body: bool, **kwargs: Unpack[HTTPXInstrumentKwargs]) -> None: ... def make_request_hook(hook: RequestHook | None, should_capture_headers: bool, should_capture_json: bool) -> RequestHook | None: ... def make_async_request_hook(hook: AsyncRequestHook | RequestHook | None, should_capture_headers: bool, should_capture_json: bool) -> AsyncRequestHook | None: ... -def make_response_hook(hook: ResponseHook | None, should_capture_headers: bool) -> ResponseHook | None: ... -def make_async_response_hook(hook: ResponseHook | AsyncResponseHook | None, should_capture_headers: bool) -> AsyncResponseHook | None: ... +def make_response_hook(hook: ResponseHook | None, should_capture_headers: bool, should_capture_json: bool, logfire_instance: Logfire) -> ResponseHook | None: ... +def make_async_response_hook(hook: ResponseHook | AsyncResponseHook | None, should_capture_headers: bool, should_capture_json: bool, logfire_instance: Logfire) -> AsyncResponseHook | None: ... +def capture_response_json(logfire_instance: Logfire, response_info: ResponseInfo, is_async: bool) -> None: ... async def run_async_hook(hook: Callable[P, Any] | None, *args: P.args, **kwargs: P.kwargs) -> None: ... def run_hook(hook: Callable[P, Any] | None, *args: P.args, **kwargs: P.kwargs) -> None: ... def capture_response_headers(span: Span, response: ResponseInfo) -> None: ... diff --git a/logfire-api/logfire_api/_internal/main.pyi b/logfire-api/logfire_api/_internal/main.pyi index 7d54718e2..007cdec57 100644 --- a/logfire-api/logfire_api/_internal/main.pyi +++ b/logfire-api/logfire_api/_internal/main.pyi @@ -551,11 +551,11 @@ class Logfire: def instrument_asyncpg(self, **kwargs: Unpack[AsyncPGInstrumentKwargs]) -> None: """Instrument the `asyncpg` module so that spans are automatically created for each query.""" @overload - def instrument_httpx(self, client: httpx.Client, capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, **kwargs: Unpack[ClientKwargs]) -> None: ... + def instrument_httpx(self, client: httpx.Client, capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, capture_response_json_body: bool = False, **kwargs: Unpack[ClientKwargs]) -> None: ... @overload - def instrument_httpx(self, client: httpx.AsyncClient, capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, **kwargs: Unpack[AsyncClientKwargs]) -> None: ... + def instrument_httpx(self, client: httpx.AsyncClient, capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, capture_response_json_body: bool = False, **kwargs: Unpack[AsyncClientKwargs]) -> None: ... @overload - def instrument_httpx(self, client: None = None, capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, **kwargs: Unpack[HTTPXInstrumentKwargs]) -> None: ... + def instrument_httpx(self, client: None = None, capture_request_headers: bool = False, capture_response_headers: bool = False, capture_request_json_body: bool = False, capture_response_json_body: bool = False, **kwargs: Unpack[HTTPXInstrumentKwargs]) -> None: ... def instrument_celery(self, **kwargs: Any) -> None: """Instrument `celery` so that spans are automatically created for each task. diff --git a/logfire-api/pyproject.toml b/logfire-api/pyproject.toml index f4c82dc4c..184935288 100644 --- a/logfire-api/pyproject.toml +++ b/logfire-api/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "logfire-api" -version = "2.8.0" +version = "2.9.0" description = "Shim for the Logfire SDK which does nothing unless Logfire is installed" authors = [ { name = "Pydantic Team", email = "engineering@pydantic.dev" }, diff --git a/pyproject.toml b/pyproject.toml index aefc328e8..5b8b64adf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "logfire" -version = "2.8.0" +version = "2.9.0" description = "The best Python observability tool! 🪵🔥" requires-python = ">=3.8" authors = [ diff --git a/uv.lock b/uv.lock index 3ee07e803..c16023b7c 100644 --- a/uv.lock +++ b/uv.lock @@ -436,7 +436,7 @@ name = "cffi" version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser" }, + { name = "pycparser", marker = "platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621 } wheels = [ @@ -1386,7 +1386,7 @@ wheels = [ [[package]] name = "logfire" -version = "2.8.0" +version = "2.9.0" source = { editable = "." } dependencies = [ { name = "executing" }, @@ -1661,7 +1661,7 @@ docs = [ [[package]] name = "logfire-api" -version = "2.8.0" +version = "2.9.0" source = { editable = "logfire-api" } [package.metadata] From a630aeded4209a30354e06f9482bf7574a399027 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Fri, 20 Dec 2024 11:06:23 +0100 Subject: [PATCH 07/10] docs: turn into single page (#703) --- docs/{get-started/traces.md => concepts.md} | 4 +- docs/guides/onboarding-checklist/index.md | 2 +- docs/index.md | 4 +- docs/why-logfire/index.md | 66 ----- docs/why-logfire/opentelemetry.md | 56 ----- docs/why-logfire/pydantic.md | 56 ----- docs/why-logfire/python-centric.md | 18 -- docs/why-logfire/simplicity.md | 19 -- docs/why-logfire/sql.md | 45 ---- docs/why.md | 265 ++++++++++++++++++++ mkdocs.yml | 24 +- 11 files changed, 282 insertions(+), 277 deletions(-) rename docs/{get-started/traces.md => concepts.md} (92%) delete mode 100644 docs/why-logfire/index.md delete mode 100644 docs/why-logfire/opentelemetry.md delete mode 100644 docs/why-logfire/pydantic.md delete mode 100644 docs/why-logfire/python-centric.md delete mode 100644 docs/why-logfire/simplicity.md delete mode 100644 docs/why-logfire/sql.md create mode 100644 docs/why.md diff --git a/docs/get-started/traces.md b/docs/concepts.md similarity index 92% rename from docs/get-started/traces.md rename to docs/concepts.md index 98e168d16..99541c0df 100644 --- a/docs/get-started/traces.md +++ b/docs/concepts.md @@ -29,7 +29,7 @@ with logfire.span('counting size of {cwd=}', cwd=cwd): logfire.info('total size of {cwd} is {size} bytes', cwd=cwd, size=total_size) ``` -![Counting size of loaded files screenshot](../images/logfire-screenshot-first-steps-load-files.png) +![Counting size of loaded files screenshot](images/logfire-screenshot-first-steps-load-files.png) --- @@ -55,7 +55,7 @@ with logfire.span('Asking the user for their {question}', question='birthday'): 2. Attempt to extract a date from the user input. If any exception is raised, the outer span will include the details of the exception. 3. This will log for example `dob=2000-01-01 age=datetime.timedelta(days=8838)` with `debug` level. -![Logfire hello world screenshot](../images/index/logfire-screenshot-hello-world-age.png) +![Logfire hello world screenshot](images/index/logfire-screenshot-hello-world-age.png) --- diff --git a/docs/guides/onboarding-checklist/index.md b/docs/guides/onboarding-checklist/index.md index 1dd373056..fa8346eb3 100644 --- a/docs/guides/onboarding-checklist/index.md +++ b/docs/guides/onboarding-checklist/index.md @@ -8,7 +8,7 @@ fix bugs, analyze user behavior, and make data-driven decisions. !!! note If you aren't familiar with traces and spans, start with the - [Tracing with Spans](../../get-started/traces.md) page. + [Tracing with Spans](../../concepts.md) page. #### Logfire Onboarding Checklist diff --git a/docs/index.md b/docs/index.md index 2e8d75a15..f5ec4b2eb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ From the team behind **Pydantic**, **Logfire** is a new type of observability pl the same belief as our open source library — that the most powerful tools can be easy to use. **Logfire** is built on OpenTelemetry, and supports monitoring your application from any language, -with particularly great support for Python! [Read more](why-logfire/index.md). +with particularly great support for Python! [Read more](why.md). ## Getting Started @@ -130,7 +130,7 @@ logfire auth Ready to keep going? -- Read about [Tracing with Spans](get-started/traces.md) +- Read about [Concepts](concepts.md) - Complete the [Onboarding Checklist](guides/onboarding-checklist/index.md) More topics to explore... diff --git a/docs/why-logfire/index.md b/docs/why-logfire/index.md deleted file mode 100644 index dc4e33e42..000000000 --- a/docs/why-logfire/index.md +++ /dev/null @@ -1,66 +0,0 @@ -# Introducing Pydantic Logfire - -From the team behind Pydantic, **Logfire** is an observability platform built on the same belief as our open source library — that the most powerful tools can be easy to use. - -## What sets Logfire apart - -
- -- :rocket:{ .lg .middle } __Simplicity and Power__ - - --- - - Logfire's dashboard is simple relative to the power it provides, ensuring your entire engineering team will actually use it. Time-to-first-log should be less than 5 minutes. - - [:octicons-arrow-right-24: Read more](simplicity.md) - -- :snake:{ .lg .middle } __Python-centric Insights__ - - --- - - From rich display of **Python objects**, to **event-loop telemetry**, to **profiling Python code & database queries**, Logfire gives you unparalleled visibility into your Python application's behavior. - - [:octicons-arrow-right-24: Read more](python-centric.md) - -- :simple-pydantic:{ .lg .middle } __Pydantic Integration__ - - --- - - Understand the data flowing through your Pydantic models and get built-in analytics on validations. - - Pydantic Logfire helps you instrument your applications with less code, less time, and better understanding. - - [:octicons-arrow-right-24: Read more](pydantic.md) - -- :telescope:{ .lg .middle } __OpenTelemetry__ - - --- - - Logfire is an opinionated wrapper around OpenTelemetry, allowing you to leverage existing tooling, infrastructure, and instrumentation for many common Python packages, and enabling support for virtually any language. - - [:octicons-arrow-right-24: Read more](opentelemetry.md) - -- :simple-instructure:{ .lg .middle } __Structured Data__ - - --- - - Include your Python objects in Logfire calls (lists, dict, dataclasses, Pydantic models, DataFrames, and more), and it'll end up as structured data in our platform ready to be queried. - - [:octicons-arrow-right-24: Read more](sql.md) - -- :abacus:{ .lg .middle } __SQL__ - - --- - - Query your data using standard SQL — all the control and (for many) nothing new to learn. Using SQL also means you can query your data with existing BI tools and database querying libraries. - - [:octicons-arrow-right-24: Read more](sql.md) - -
- - -## Find the needle in a _stack trace_ - -We understand Python and its peculiarities. Pydantic Logfire was crafted by Python developers, for Python developers, addressing the unique challenges and opportunities of the Python environment. It's not just about having data; it's about having the *right* data, presented in ways that make sense for Python applications. - -![Logfire FastAPI screenshot](../images/index/logfire-screenshot-fastapi-200.png) diff --git a/docs/why-logfire/opentelemetry.md b/docs/why-logfire/opentelemetry.md deleted file mode 100644 index 02aa44b2f..000000000 --- a/docs/why-logfire/opentelemetry.md +++ /dev/null @@ -1,56 +0,0 @@ -# OpenTelemetry under the hood :telescope: - -Because **Pydantic Logfire** is built on [OpenTelemetry](https://opentelemetry.io/), you can -use a wealth of existing tooling and infrastructure, including -[instrumentation for many common Python packages](https://opentelemetry-python-contrib.readthedocs.io/en/latest/index.html). Logfire also supports cross-language data integration and data export to any OpenTelemetry-compatible backend or proxy. - -For example, we can instrument a simple FastAPI app with just 2 lines of code: - -```py title="main.py" hl_lines="8 9 10" -from datetime import date - -import logfire -from pydantic import BaseModel -from fastapi import FastAPI - -app = FastAPI() - -logfire.configure() -logfire.instrument_fastapi(app) # (1)! -# Here you'd instrument any other library that you use. (2) - - -class User(BaseModel): - name: str - country_code: str - dob: date - - -@app.post('/') -async def add_user(user: User): - # we would store the user here - return {'message': f'{user.name} added'} -``` - -1. In addition to [configuring logfire](../reference/configuration.md) this line is all you need to instrument a FastAPI app with Logfire. The same applies to most other popular Python web frameworks. -2. The [integrations](../integrations/index.md) page has more information on how to instrument other parts of your app. Run the [inspect](../reference/cli.md#inspect-inspect) command for package suggestions. - -We'll need the [FastAPI contrib package](../integrations/web-frameworks/fastapi.md), FastAPI itself and uvicorn installed to run this: - -```bash -pip install 'logfire[fastapi]' fastapi uvicorn # (1)! -uvicorn main:app # (2)! -``` - -1. Install the `logfire` package with the `fastapi` extra, FastAPI, and uvicorn. -2. Run the FastAPI app with uvicorn. - -This will give you information on the HTTP request and details of results from successful input validations: - -![Logfire FastAPI 200 response screenshot](../images/index/logfire-screenshot-fastapi-200.png) - -And, importantly, details of failed input validations: - -![Logfire FastAPI 422 response screenshot](../images/index/logfire-screenshot-fastapi-422.png) - -In the example above, we can see the FastAPI arguments failing (`user` is null when it should always be populated). This demonstrates type-checking from Pydantic used out-of-the-box in FastAPI. diff --git a/docs/why-logfire/pydantic.md b/docs/why-logfire/pydantic.md deleted file mode 100644 index 13c15fdf1..000000000 --- a/docs/why-logfire/pydantic.md +++ /dev/null @@ -1,56 +0,0 @@ -# Pydantic integration - -**Logfire** has an out-of-the-box **Pydantic** integration that lets you understand the data -passing through your Pydantic models and get analytics on validations. For existing Pydantic users, -it delivers unparalleled insights into your usage of Pydantic models. - -We can record Pydantic models directly: - -```py -from datetime import date - -import logfire -from pydantic import BaseModel - -logfire.configure() - -class User(BaseModel): - name: str - country_code: str - dob: date - -user = User(name='Anne', country_code='USA', dob='2000-01-01') -logfire.info('user processed: {user!r}', user=user) # (1)! -``` - -1. This will show `user processed: User(name='Anne', country_code='US', dob=datetime.date(2000, 1, 1))`, but also allow you to see a "pretty" view of the model within the Logfire Platform. - -![Logfire pydantic manual screenshot](../images/index/logfire-screenshot-pydantic-manual.png) - -Or we can record information about validations automatically: - -```py -from datetime import date - -import logfire -from pydantic import BaseModel - -logfire.configure() -logfire.instrument_pydantic() # (1)! - -class User(BaseModel): - name: str - country_code: str - dob: date - -User(name='Anne', country_code='USA', dob='2000-01-01') # (2)! -User(name='Ben', country_code='USA', dob='2000-02-02') -User(name='Charlie', country_code='GBR', dob='1990-03-03') -``` - -1. This configuration means details about all Pydantic model validations will be recorded. You can also record details about validation failures only, or just metrics; see the [pydantic plugin docs](../integrations/pydantic.md). -2. Since we've enabled the Pydantic Plugin, all Pydantic validations will be recorded in Logfire. - -Learn more about the [Pydantic Plugin here](../integrations/pydantic.md). - -![Logfire pydantic plugin screenshot](../images/index/logfire-screenshot-pydantic-plugin.png) diff --git a/docs/why-logfire/python-centric.md b/docs/why-logfire/python-centric.md deleted file mode 100644 index c5de4b9da..000000000 --- a/docs/why-logfire/python-centric.md +++ /dev/null @@ -1,18 +0,0 @@ -# Python-centric insights :material-snake: - -**Pydantic Logfire** automatically instruments your code for minimal manual effort, provides -exceptional insights into async code, offers detailed performance analytics, and displays Python -objects the same as the interpreter. **Pydantic Logfire** gives you a clearer view into how your -Python is running than any other observability tool. - -## Rich display of Python objects - -![Logfire FastAPI screenshot](../images/logfire-screenshot-fastapi-arguments.png) - -In this example, you can see the parameters passed to a FastAPI endpoint formatted as a Python object. - -## Profiling Python code - -![Logfire Auto-tracing screenshot](../images/logfire-screenshot-autotracing.png) - -In this simple app example, you can see every interaction the user makes with the web app automatically traced to the Live view using the [Auto-tracing method](../guides/onboarding-checklist/add-auto-tracing.md). diff --git a/docs/why-logfire/simplicity.md b/docs/why-logfire/simplicity.md deleted file mode 100644 index 60c642271..000000000 --- a/docs/why-logfire/simplicity.md +++ /dev/null @@ -1,19 +0,0 @@ -# Simplicity and Power :rocket: - -Emulating the Pydantic library's philosophy, Pydantic Logfire offers an -intuitive start for beginners while providing the depth experts desire. It's the same balance of ease, sophistication, -and productivity, reimagined for observability. - -Within a few minutes you'll have your first logs: - -![Logfire hello world screenshot](../images/index/logfire-screenshot-hello-world-age.png) - - -This might look similar to simple logging, but it's much more powerful — you get: - -- **Structured data** from your logs -- **Nested logs & traces** to contextualize what you're viewing -- **Custom-built platform** to view your data, with no configuration required -- **Pretty display** of Python objects - -Ready to try Logfire? [Get Started](../index.md)! 🚀 diff --git a/docs/why-logfire/sql.md b/docs/why-logfire/sql.md deleted file mode 100644 index 61f26f946..000000000 --- a/docs/why-logfire/sql.md +++ /dev/null @@ -1,45 +0,0 @@ -# Structured Data and SQL :abacus: {#sql} - -Query your data with pure, canonical PostgreSQL — all the control and (for many) nothing new to learn. We even provide direct access to the underlying Postgres database, which means that you can query Logfire using any Postgres-compatible tools you like. - -This includes BI tools and dashboard-building platforms like - -- Superset -- Grafana -- Google Looker Studio - -As well as data science tools like - -- Pandas -- SQLAlchemy -- `psql` - -Using vanilla PostgreSQL as the querying language throughout the platform ensures a consistent, powerful, and flexible querying experience. - -Another big advantage of using the most widely used SQL databases is that generative AI tools like ChatGPT are excellent at writing SQL for you. - -Just include your Python objects in **Logfire** calls (lists, dict, dataclasses, Pydantic models, DataFrames, and more), -and it'll end up as structured data in our platform ready to be queried. - -For example, using data from a `User` model, we could list users from the USA: - -```sql -SELECT attributes->'result'->>'name' as name, extract(year from (attributes->'result'->>'dob')::date) as "birth year" -FROM records -WHERE attributes->'result'->>'country_code' = 'USA'; -``` - -![Logfire explore query screenshot](../images/index/logfire-screenshot-explore-query.png) - -You can also filter to show only traces related to users in the USA in the live view with - -```sql -attributes->'result'->>'name' = 'Ben' -``` - -![Logfire search query screenshot](../images/index/logfire-screenshot-search-query.png) - - -Structured Data and Direct SQL Access means you can use familiar tools like Pandas, SQLAlchemy, or `psql` -for querying, can integrate seamlessly with BI tools, and can even leverage AI for SQL generation, ensuring your Python -objects and structured data are query-ready. diff --git a/docs/why.md b/docs/why.md new file mode 100644 index 000000000..d73d7f86f --- /dev/null +++ b/docs/why.md @@ -0,0 +1,265 @@ +# Introducing Pydantic Logfire + +From the team behind Pydantic, **Logfire** is an observability platform built on the same belief as our open source library — that the most powerful tools can be easy to use. + +## What sets Logfire apart + +
+ +- :rocket:{ .lg .middle } __Simplicity and Power__ + + --- + + Logfire's dashboard is simple relative to the power it provides, ensuring your entire engineering team will actually use it. Time-to-first-log should be less than 5 minutes. + + [:octicons-arrow-right-24: Read more](#simplicity-and-power-) + +- :snake:{ .lg .middle } __Python-centric Insights__ + + --- + + From rich display of **Python objects**, to **event-loop telemetry**, to **profiling Python code & database queries**, Logfire gives you unparalleled visibility into your Python application's behavior. + + [:octicons-arrow-right-24: Read more](#python-centric-insights-material-snake) + +- :simple-pydantic:{ .lg .middle } __Pydantic Integration__ + + --- + + Understand the data flowing through your Pydantic models and get built-in analytics on validations. + + Pydantic Logfire helps you instrument your applications with less code, less time, and better understanding. + + [:octicons-arrow-right-24: Read more](#pydantic-integration) + +- :telescope:{ .lg .middle } __OpenTelemetry__ + + --- + + Logfire is an opinionated wrapper around OpenTelemetry, allowing you to leverage existing tooling, infrastructure, and instrumentation for many common Python packages, and enabling support for virtually any language. + + [:octicons-arrow-right-24: Read more](#opentelemetry-under-the-hood-) + +- :simple-instructure:{ .lg .middle } __Structured Data__ + + --- + + Include your Python objects in Logfire calls (lists, dict, dataclasses, Pydantic models, DataFrames, and more), and it'll end up as structured data in our platform ready to be queried. + + [:octicons-arrow-right-24: Read more](#structured-data-and-sql--sql) + +- :abacus:{ .lg .middle } __SQL__ + + --- + + Query your data using standard SQL — all the control and (for many) nothing new to learn. Using SQL also means you can query your data with existing BI tools and database querying libraries. + + [:octicons-arrow-right-24: Read more](#structured-data-and-sql--sql) + +
+ + +## Find the needle in a _stack trace_ + +We understand Python and its peculiarities. Pydantic Logfire was crafted by Python developers, for Python developers, addressing the unique challenges and opportunities of the Python environment. It's not just about having data; it's about having the *right* data, presented in ways that make sense for Python applications. + +![Logfire FastAPI screenshot](images/index/logfire-screenshot-fastapi-200.png) + +## Simplicity and Power :rocket: + +Emulating the Pydantic library's philosophy, Pydantic Logfire offers an +intuitive start for beginners while providing the depth experts desire. It's the same balance of ease, sophistication, +and productivity, reimagined for observability. + +Within a few minutes you'll have your first logs: + +![Logfire hello world screenshot](images/index/logfire-screenshot-hello-world-age.png) + + +This might look similar to simple logging, but it's much more powerful — you get: + +- **Structured data** from your logs +- **Nested logs & traces** to contextualize what you're viewing +- **Custom-built platform** to view your data, with no configuration required +- **Pretty display** of Python objects + +Ready to try Logfire? [Get Started](index.md)! 🚀 + +## Python-centric insights :material-snake: + +**Pydantic Logfire** automatically instruments your code for minimal manual effort, provides +exceptional insights into async code, offers detailed performance analytics, and displays Python +objects the same as the interpreter. **Pydantic Logfire** gives you a clearer view into how your +Python is running than any other observability tool. + +### Rich display of Python objects + +![Logfire FastAPI screenshot](images/logfire-screenshot-fastapi-arguments.png) + +In this example, you can see the parameters passed to a FastAPI endpoint formatted as a Python object. + +### Profiling Python code + +![Logfire Auto-tracing screenshot](images/logfire-screenshot-autotracing.png) + +In this simple app example, you can see every interaction the user makes with the web app automatically traced to the Live view using the [Auto-tracing method](guides/onboarding-checklist/add-auto-tracing.md). + +## Pydantic integration + +**Logfire** has an out-of-the-box **Pydantic** integration that lets you understand the data +passing through your Pydantic models and get analytics on validations. For existing Pydantic users, +it delivers unparalleled insights into your usage of Pydantic models. + +We can record Pydantic models directly: + +```py +from datetime import date + +import logfire +from pydantic import BaseModel + +logfire.configure() + +class User(BaseModel): + name: str + country_code: str + dob: date + +user = User(name='Anne', country_code='USA', dob='2000-01-01') +logfire.info('user processed: {user!r}', user=user) # (1)! +``` + +1. This will show `user processed: User(name='Anne', country_code='US', dob=datetime.date(2000, 1, 1))`, but also allow you to see a "pretty" view of the model within the Logfire Platform. + +![Logfire pydantic manual screenshot](images/index/logfire-screenshot-pydantic-manual.png) + +Or we can record information about validations automatically: + +```py +from datetime import date + +import logfire +from pydantic import BaseModel + +logfire.configure() +logfire.instrument_pydantic() # (1)! + +class User(BaseModel): + name: str + country_code: str + dob: date + +User(name='Anne', country_code='USA', dob='2000-01-01') # (2)! +User(name='Ben', country_code='USA', dob='2000-02-02') +User(name='Charlie', country_code='GBR', dob='1990-03-03') +``` + +1. This configuration means details about all Pydantic model validations will be recorded. You can also record details about validation failures only, or just metrics; see the [pydantic plugin docs](integrations/pydantic.md). +2. Since we've enabled the Pydantic Plugin, all Pydantic validations will be recorded in Logfire. + +Learn more about the [Pydantic Plugin here](integrations/pydantic.md). + +![Logfire pydantic plugin screenshot](images/index/logfire-screenshot-pydantic-plugin.png) + +## OpenTelemetry under the hood :telescope: + +Because **Pydantic Logfire** is built on [OpenTelemetry](https://opentelemetry.io/), you can +use a wealth of existing tooling and infrastructure, including +[instrumentation for many common Python packages](https://opentelemetry-python-contrib.readthedocs.io/en/latest/index.html). Logfire also supports cross-language data integration and data export to any OpenTelemetry-compatible backend or proxy. + +For example, we can instrument a simple FastAPI app with just 2 lines of code: + +```py title="main.py" hl_lines="8 9 10" +from datetime import date + +import logfire +from pydantic import BaseModel +from fastapi import FastAPI + +app = FastAPI() + +logfire.configure() +logfire.instrument_fastapi(app) # (1)! +# Here you'd instrument any other library that you use. (2) + + +class User(BaseModel): + name: str + country_code: str + dob: date + + +@app.post('/') +async def add_user(user: User): + # we would store the user here + return {'message': f'{user.name} added'} +``` + +1. In addition to [configuring logfire](reference/configuration.md) this line is all you need to instrument a FastAPI app with Logfire. The same applies to most other popular Python web frameworks. +2. The [integrations](integrations/index.md) page has more information on how to instrument other parts of your app. Run the [inspect](reference/cli.md#inspect-inspect) command for package suggestions. + +We'll need the [FastAPI contrib package](integrations/web-frameworks/fastapi.md), FastAPI itself and uvicorn installed to run this: + +```bash +pip install 'logfire[fastapi]' fastapi uvicorn # (1)! +uvicorn main:app # (2)! +``` + +1. Install the `logfire` package with the `fastapi` extra, FastAPI, and uvicorn. +2. Run the FastAPI app with uvicorn. + +This will give you information on the HTTP request and details of results from successful input validations: + +![Logfire FastAPI 200 response screenshot](images/index/logfire-screenshot-fastapi-200.png) + +And, importantly, details of failed input validations: + +![Logfire FastAPI 422 response screenshot](images/index/logfire-screenshot-fastapi-422.png) + +In the example above, we can see the FastAPI arguments failing (`user` is null when it should always be populated). This demonstrates type-checking from Pydantic used out-of-the-box in FastAPI. + +## Structured Data and SQL :abacus: {#sql} + +Query your data with pure, canonical PostgreSQL — all the control and (for many) nothing new to learn. We even provide direct access to the underlying Postgres database, which means that you can query Logfire using any Postgres-compatible tools you like. + +This includes BI tools and dashboard-building platforms like + +- Superset +- Grafana +- Google Looker Studio + +As well as data science tools like + +- Pandas +- SQLAlchemy +- `psql` + +Using vanilla PostgreSQL as the querying language throughout the platform ensures a consistent, powerful, and flexible querying experience. + +Another big advantage of using the most widely used SQL databases is that generative AI tools like ChatGPT are excellent at writing SQL for you. + +Just include your Python objects in **Logfire** calls (lists, dict, dataclasses, Pydantic models, DataFrames, and more), +and it'll end up as structured data in our platform ready to be queried. + +For example, using data from a `User` model, we could list users from the USA: + +```sql +SELECT attributes->'result'->>'name' as name, extract(year from (attributes->'result'->>'dob')::date) as "birth year" +FROM records +WHERE attributes->'result'->>'country_code' = 'USA'; +``` + +![Logfire explore query screenshot](images/index/logfire-screenshot-explore-query.png) + +You can also filter to show only traces related to users in the USA in the live view with + +```sql +attributes->'result'->>'name' = 'Ben' +``` + +![Logfire search query screenshot](images/index/logfire-screenshot-search-query.png) + + +Structured Data and Direct SQL Access means you can use familiar tools like Pandas, SQLAlchemy, or `psql` +for querying, can integrate seamlessly with BI tools, and can even leverage AI for SQL generation, ensuring your Python +objects and structured data are query-ready. diff --git a/mkdocs.yml b/mkdocs.yml index 9bbb66cdd..5cc0a60a2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,7 +39,7 @@ theme: - content.code.select - navigation.indexes # https://squidfunk.github.io/mkdocs-material/setup/setting-up-navigation/#section-index-pages - navigation.path - - navigation.tabs + - navigation.sections - navigation.instant - navigation.instant.preview - navigation.instant.prefetch @@ -64,17 +64,10 @@ extra_javascript: - "/flarelytics/client.js" nav: - - Why Logfire?: - - Introducing Logfire: why-logfire/index.md - - Simplicity and Power: why-logfire/simplicity.md - - Python-centric Insights: why-logfire/python-centric.md - - Pydantic Integration: why-logfire/pydantic.md - - OpenTelemetry: why-logfire/opentelemetry.md - - Structured Data and SQL: why-logfire/sql.md - - - Get Started: - - Get Started: index.md - - Tracing with Spans: get-started/traces.md + - Logfire: + - Logfire: index.md + - Why Logfire?: why.md + - Concepts: concepts.md - Onboarding Checklist: - Onboarding Checklist: guides/onboarding-checklist/index.md - Integrate Logfire: guides/onboarding-checklist/integrate.md @@ -290,5 +283,12 @@ plugins: "guides/onboarding_checklist/add_metrics.md": "guides/onboarding-checklist/add-metrics.md" "guides/advanced/query_api.md": "guides/advanced/query-api.md" "guides/index.md": "index.md" + "why-logfire/index.md": "why.md" + "why-logfire/pydantic.md": "why.md" + "why-logfire/opentelemetry.md": "why.md" + "why-logfire/simplicity.md": "why.md" + "why-logfire/python-centric.md": "why.md" + "why-logfire/sql.md": "why.md" + "get-started/traces.md": "concepts.md" hooks: - docs/plugins/main.py From 5e2f7ca04c4d9c88d4280729b66e665fcca8dc46 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Fri, 20 Dec 2024 11:12:51 +0100 Subject: [PATCH 08/10] docs: drop navigation hides (#707) --- docs/help.md | 1 - docs/release-notes.md | 5 ----- docs/roadmap.md | 7 ------- 3 files changed, 13 deletions(-) diff --git a/docs/help.md b/docs/help.md index bd3c40d21..7357ef0a3 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1,6 +1,5 @@ --- hide: -- navigation - toc --- diff --git a/docs/release-notes.md b/docs/release-notes.md index c8d13c6f7..786b75d5a 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,6 +1 @@ ---- -hide: -- navigation ---- - --8<-- "CHANGELOG.md" diff --git a/docs/roadmap.md b/docs/roadmap.md index 07a6893bd..93e886bb4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,10 +1,3 @@ ---- -hide: -- navigation ---- - -# Roadmap - Here is the roadmap for **Pydantic Logfire**. This is a living document, and it will be updated as we progress. If you have any questions, or a feature request, **please join our [Slack][slack]**. From 9d1491cb28d2a495e1215419f227b3c76bcff2bd Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Fri, 20 Dec 2024 11:45:44 +0100 Subject: [PATCH 09/10] docs: create how-to-guides section (#708) --- .../onboarding-checklist/add-auto-tracing.md | 2 +- docs/guides/web-ui/live.md | 25 +++++++---- .../alternative-backends.md | 0 .../alternative-clients.md | 2 +- .../create-write-tokens.md} | 2 +- .../environments.md | 7 +-- .../link-to-code-source.md | 4 +- .../advanced => how-to-guides}/query-api.md | 2 +- .../advanced => how-to-guides}/sampling.md | 0 .../advanced => how-to-guides}/scrubbing.md | 4 +- docs/index.md | 7 +-- docs/integrations/event-streams/airflow.md | 2 +- docs/integrations/web-frameworks/index.md | 2 +- .../advanced/backfill.md | 0 .../advanced/generators.md | 0 .../{guides => reference}/advanced/testing.md | 0 docs/reference/examples.md | 2 +- docs/roadmap.md | 2 +- docs/why.md | 10 ++--- mkdocs.yml | 44 ++++++++++++------- 20 files changed, 70 insertions(+), 47 deletions(-) rename docs/{guides/advanced => how-to-guides}/alternative-backends.md (100%) rename docs/{guides/advanced => how-to-guides}/alternative-clients.md (99%) rename docs/{guides/advanced/creating-write-tokens.md => how-to-guides/create-write-tokens.md} (95%) rename docs/{guides/advanced => how-to-guides}/environments.md (90%) rename docs/{guides/advanced => how-to-guides}/link-to-code-source.md (95%) rename docs/{guides/advanced => how-to-guides}/query-api.md (99%) rename docs/{guides/advanced => how-to-guides}/sampling.md (100%) rename docs/{guides/advanced => how-to-guides}/scrubbing.md (94%) rename docs/{guides => reference}/advanced/backfill.md (100%) rename docs/{guides => reference}/advanced/generators.md (100%) rename docs/{guides => reference}/advanced/testing.md (100%) diff --git a/docs/guides/onboarding-checklist/add-auto-tracing.md b/docs/guides/onboarding-checklist/add-auto-tracing.md index fe9bb1e49..ebab8e0ab 100644 --- a/docs/guides/onboarding-checklist/add-auto-tracing.md +++ b/docs/guides/onboarding-checklist/add-auto-tracing.md @@ -22,7 +22,7 @@ main() ``` !!! note - Generator functions will not be traced for reasons explained [here](../advanced/generators.md). + Generator functions will not be traced for reasons explained [here](../../reference/advanced/generators.md). ## Only tracing functions above a minimum duration diff --git a/docs/guides/web-ui/live.md b/docs/guides/web-ui/live.md index a37b94951..c3adac514 100644 --- a/docs/guides/web-ui/live.md +++ b/docs/guides/web-ui/live.md @@ -12,10 +12,14 @@ To search the live view, click `Search your spans` (keyboard shortcut `/`), this ### SQL Search -For confident SQL users, write your queries directly here. For devs who want a bit of help, try the new [PydanticAI](https://ai.pydantic.dev/) feature which generates a SQL query based on your prompt. You can also review the fields available and populate your SQL automatically using the `Reference` list, see more on this below. +For confident SQL users, write your queries directly here. For devs who want a bit of help, +try the new [PydanticAI](https://ai.pydantic.dev/) feature which generates a SQL query based on your prompt. +You can also review the fields available and populate your SQL automatically using the `Reference` list, see more on this below. -**WHERE clause** -As the greyed out `SELECT * FROM RECORDS WHERE` implies, you're searching inside the `WHERE` clause of a SQL query. It has auto-complete & schema hints, so try typing something to get a reminder. To run your query click `Run` or keyboard shortcut `cmd+enter` (or `ctrl+enter` on windows/linux). +**WHERE clause** +As the greyed out `SELECT * FROM RECORDS WHERE` implies, you're searching inside the `WHERE` clause of a SQL query. +It has auto-complete & schema hints, so try typing something to get a reminder. To run your query click `Run` or +keyboard shortcut `cmd+enter` (or `ctrl+enter` on Windows/Linux). Note: you can run more complex queries on the [explore screen](explore.md) @@ -62,7 +66,7 @@ If you're not sure where to start, scroll down to the `Start here` for beginner- ### Ask in Language -> Get SQL -Write your question in your native language, and the model will convert that question to a SQL query. +Write your question in your native language, and the model will convert that question to a SQL query. ![Search box natural language](../../images/guide/live-view-natural-language.png) @@ -74,8 +78,10 @@ Under the hood this feature uses an LLM running with [PydanticAI](https://github Reference: A list of pre-populated query clauses. Clicking any of the clauses will populate the SQL editor, and (where applicable) you can choose a value from the autopopulated dropdown. -This list gives you a powerful way to rapidly generate the query you need, while simultaneously learning more about all the ways you can search your data. Clicking multiple clauses will add them to your query with a SQL `AND` statement. If you'd like something other than an `AND` statement, you can replace this with alternative SQL operators like `OR`, or `NOT`. - +This list gives you a powerful way to rapidly generate the query you need, while simultaneously +learning more about all the ways you can search your data. Clicking multiple clauses will add them +to your query with a SQL `AND` statement. If you'd like something other than an `AND` statement, you +can replace this with alternative SQL operators like `OR`, or `NOT`. ## Details panel closed @@ -83,9 +89,12 @@ This list gives you a powerful way to rapidly generate the query you need, while This is what you'll see when you come to the live view of a project with some data. -1. **Organization and project labels:** In this example, the organization is `christophergs`, and the project is `docs-app`. You can click the organization name to go to the organization overview page; the project name is a link to this page. +1. **Organization and project labels:** In this example, the organization is `christophergs`, and + the project is `docs-app`. You can click the organization name to go to the organization overview page; + the project name is a link to this page. -2. **Environment:** In the above screenshot, this is set to `all envs`. See the [environments docs](../advanced/environments.md) for details. +2. **Environment:** In the above screenshot, this is set to `all envs`. + See the [environments docs](../../how-to-guides/environments.md) for details. 3. **Timeline:** This shows a histogram of the counts of spans matching your query over time. The blue-highlighted section corresponds to the time range currently visible in the scrollable list of traces below. You can click at points on this line to move to viewing logs from that point in time. diff --git a/docs/guides/advanced/alternative-backends.md b/docs/how-to-guides/alternative-backends.md similarity index 100% rename from docs/guides/advanced/alternative-backends.md rename to docs/how-to-guides/alternative-backends.md diff --git a/docs/guides/advanced/alternative-clients.md b/docs/how-to-guides/alternative-clients.md similarity index 99% rename from docs/guides/advanced/alternative-clients.md rename to docs/how-to-guides/alternative-clients.md index ec16c1fe9..eaca1bd6b 100644 --- a/docs/guides/advanced/alternative-clients.md +++ b/docs/how-to-guides/alternative-clients.md @@ -7,7 +7,7 @@ these [environment variables](https://opentelemetry.io/docs/languages/sdk-config - `OTEL_EXPORTER_OTLP_ENDPOINT=https://logfire-api.pydantic.dev` for both traces and metrics, or: - `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://logfire-api.pydantic.dev/v1/traces` for just traces - `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=https://logfire-api.pydantic.dev/v1/metrics` for just metrics -- `OTEL_EXPORTER_OTLP_HEADERS='Authorization=your-write-token'` - see [Creating Write Tokens](./creating-write-tokens.md) +- `OTEL_EXPORTER_OTLP_HEADERS='Authorization=your-write-token'` - see [Create Write Tokens](./create-write-tokens.md) to obtain a write token and replace `your-write-token` with it. - `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` to export in Protobuf format over HTTP (not gRPC). The **Logfire** backend supports both Protobuf and JSON, but only over HTTP for now. Some SDKs (such as Python) already use this value as the default so setting this isn't required, but other SDKs use `grpc` as the default. diff --git a/docs/guides/advanced/creating-write-tokens.md b/docs/how-to-guides/create-write-tokens.md similarity index 95% rename from docs/guides/advanced/creating-write-tokens.md rename to docs/how-to-guides/create-write-tokens.md index 0eb9d0ca9..1a4df7ebf 100644 --- a/docs/guides/advanced/creating-write-tokens.md +++ b/docs/how-to-guides/create-write-tokens.md @@ -1,6 +1,6 @@ To send data to **Logfire**, you need to create a write token. A write token is a unique identifier that allows you to send data to a specific **Logfire** project. -If you set up Logfire according to the [getting started guide](../../index.md), you already have a write token locally tied to the project you created. +If you set up Logfire according to the [getting started guide](../index.md), you already have a write token locally tied to the project you created. But if you want to configure other computers to write to that project, for example in a deployed application, you need to create a new write token. You can create a write token by following these steps: diff --git a/docs/guides/advanced/environments.md b/docs/how-to-guides/environments.md similarity index 90% rename from docs/guides/advanced/environments.md rename to docs/how-to-guides/environments.md index c635884ab..ba0e5aeef 100644 --- a/docs/guides/advanced/environments.md +++ b/docs/how-to-guides/environments.md @@ -23,9 +23,10 @@ If you are using languages other than Python, you can set the environment like t --- Once set, you will see your environment in the Logfire UI `all envs` dropdown, -which is present on the [Live View](../web-ui/live.md), [Dashboards](../web-ui/dashboards.md) and [Explore](../web-ui/explore.md) pages: +which is present on the [Live View](../guides/web-ui/live.md), [Dashboards](../guides/web-ui/dashboards.md) +and [Explore](../guides/web-ui/explore.md) pages: -![Environments](../../images/guide/environments.png) +![Environments](../images/guide/environments.png) !!! info When using an environment for the first time, it may take a **few minutes** for the environment to appear in the UI. @@ -55,5 +56,5 @@ environment name. ## Should I use environments or projects? Environments are more lightweight than projects. Projects give you the ability to assign specific -user groups and permissions levels (see this [organization structure diagram](../../reference/organization-structure.md) +user groups and permissions levels (see this [organization structure diagram](../reference/organization-structure.md) for details). So if you need to allow different team members to view dev vs. prod traces, then projects would be a better fit. diff --git a/docs/guides/advanced/link-to-code-source.md b/docs/how-to-guides/link-to-code-source.md similarity index 95% rename from docs/guides/advanced/link-to-code-source.md rename to docs/how-to-guides/link-to-code-source.md index 49604c557..5affd9117 100644 --- a/docs/guides/advanced/link-to-code-source.md +++ b/docs/how-to-guides/link-to-code-source.md @@ -5,7 +5,7 @@ We support linking to the source code on GitHub, GitLab, and any other VCS provider that uses the same URL format. -![Link to GitHub](../../images/guide/link-to-github.gif) +![Link to GitHub](../images/guide/link-to-github.gif) ## Usage @@ -40,5 +40,5 @@ OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES},vcs.repository.ref.revision OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES},vcs.root.path=. ``` -[help]: ../../help.md +[help]: ../help.md [otel-resource-attributes]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration diff --git a/docs/guides/advanced/query-api.md b/docs/how-to-guides/query-api.md similarity index 99% rename from docs/guides/advanced/query-api.md rename to docs/how-to-guides/query-api.md index 9c7a3d13f..2ab130eb5 100644 --- a/docs/guides/advanced/query-api.md +++ b/docs/how-to-guides/query-api.md @@ -10,7 +10,7 @@ See [here](#additional-configuration) for more details about the available respo ## How to Create a Read Token -If you've set up Logfire following the [getting started guide](../../index.md), you can generate read tokens from +If you've set up Logfire following the [getting started guide](../index.md), you can generate read tokens from the Logfire web interface, for use accessing the Logfire Query API. To create a read token: diff --git a/docs/guides/advanced/sampling.md b/docs/how-to-guides/sampling.md similarity index 100% rename from docs/guides/advanced/sampling.md rename to docs/how-to-guides/sampling.md diff --git a/docs/guides/advanced/scrubbing.md b/docs/how-to-guides/scrubbing.md similarity index 94% rename from docs/guides/advanced/scrubbing.md rename to docs/how-to-guides/scrubbing.md index 312445966..a032ed0b0 100644 --- a/docs/guides/advanced/scrubbing.md +++ b/docs/how-to-guides/scrubbing.md @@ -85,9 +85,9 @@ User details: User(id=123, password='secret') This is necessary so that safe messages such as 'Password is correct' are not redacted completely. Using f-strings (e.g. `logfire.info(f'User details: {user}')`) *is* safe if `inspect_arguments` is enabled (the default in Python 3.11+) and working correctly. -[See here](../onboarding-checklist/add-manual-tracing.md#f-strings) for more information. +[See here](../guides/onboarding-checklist/add-manual-tracing.md#f-strings) for more information. -In short, don't format the message yourself. This is also a good practice in general for [other reasons](../onboarding-checklist/add-manual-tracing.md#messages-and-span-names). +In short, don't format the message yourself. This is also a good practice in general for [other reasons](../guides/onboarding-checklist/add-manual-tracing.md#messages-and-span-names). ### Keep sensitive data out of URLs diff --git a/docs/index.md b/docs/index.md index f5ec4b2eb..3dc8387bd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,7 +52,8 @@ logfire auth ## Instrument your project {#instrument} === ":material-cog-outline: Development" !!! tip "Development setup" - During development, we recommend using the CLI to configure Logfire. You can also use a [write token](guides/advanced/creating-write-tokens.md). + During development, we recommend using the CLI to configure Logfire. + You can also use a [write token](how-to-guides/create-write-tokens.md). 1. Set your project @@ -72,8 +73,8 @@ logfire auth logfire.info('Hello, {name}!', name='world') # (2)! ``` - 1. The `configure()` method should be called once before logging to initialize **Logfire**. - 2. This will log `Hello world!` with `info` level. + 3. The `configure()` method should be called once before logging to initialize **Logfire**. + 4. This will log `Hello world!` with `info` level. !!! info "" Other [log levels][logfire.Logfire] are also available to use, including `trace`, `debug`, `notice`, `warn`, diff --git a/docs/integrations/event-streams/airflow.md b/docs/integrations/event-streams/airflow.md index 35626c858..7c92f0836 100644 --- a/docs/integrations/event-streams/airflow.md +++ b/docs/integrations/event-streams/airflow.md @@ -162,4 +162,4 @@ otel_task_log_event = True [OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/ [OpenTelemetry Collector installation]: https://opentelemetry.io/docs/collector/installation/ [OpenTelemetry Collector Receiver]: https://opentelemetry.io/docs/collector/configuration/#receivers -[write-token]: ../../guides/advanced/creating-write-tokens.md +[write-token]: ../../how-to-guides/create-write-tokens.md diff --git a/docs/integrations/web-frameworks/index.md b/docs/integrations/web-frameworks/index.md index 6e86373ad..80917a143 100644 --- a/docs/integrations/web-frameworks/index.md +++ b/docs/integrations/web-frameworks/index.md @@ -47,7 +47,7 @@ To replace the `Authorization` header value with `[REDACTED]` to avoid leaking u OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS="Authorization" ``` -(although usually it's better to rely on **Logfire**'s [scrubbing](../../guides/advanced/scrubbing.md) feature) +(although usually it's better to rely on **Logfire**'s [scrubbing](../../how-to-guides/scrubbing.md) feature) ## Query HTTP requests duration per percentile diff --git a/docs/guides/advanced/backfill.md b/docs/reference/advanced/backfill.md similarity index 100% rename from docs/guides/advanced/backfill.md rename to docs/reference/advanced/backfill.md diff --git a/docs/guides/advanced/generators.md b/docs/reference/advanced/generators.md similarity index 100% rename from docs/guides/advanced/generators.md rename to docs/reference/advanced/generators.md diff --git a/docs/guides/advanced/testing.md b/docs/reference/advanced/testing.md similarity index 100% rename from docs/guides/advanced/testing.md rename to docs/reference/advanced/testing.md diff --git a/docs/reference/examples.md b/docs/reference/examples.md index 4622e1814..8809a8c30 100644 --- a/docs/reference/examples.md +++ b/docs/reference/examples.md @@ -18,7 +18,7 @@ This example is a simple Python financial calculator app using Flask and SQLAlch ## JavaScript -Currently we only have a Python SDK, but the Logfire backend and UI support data sent by any OpenTelemetry client. See the [alternative clients guide](../guides/advanced/alternative-clients.md) for details on setting up OpenTelemetry in any language. We're working on a JavaScript SDK, but in the meantime here are some examples of using plain OpenTelemetry in JavaScript: +Currently we only have a Python SDK, but the Logfire backend and UI support data sent by any OpenTelemetry client. See the [alternative clients guide](../how-to-guides/alternative-clients.md) for details on setting up OpenTelemetry in any language. We're working on a JavaScript SDK, but in the meantime here are some examples of using plain OpenTelemetry in JavaScript: ### Cloudflare worker example diff --git a/docs/roadmap.md b/docs/roadmap.md index 93e886bb4..f64d1149a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -65,7 +65,7 @@ Logfire is built on top of OpenTelemetry, which means that it supports all the l Still, we are planning to create custom SDKs for JavaScript, TypeScript, and Rust, and make sure that the attributes are displayed in a nice way in the Logfire UI — as they are for Python. -For now, you can check our [Alternative Clients](guides/advanced/alternative-clients.md) section to see how +For now, you can check our [Alternative Clients](how-to-guides/alternative-clients.md) section to see how you can send data to Logfire from other languages. See [this GitHub issue][language-support-gh-issue] for more information. diff --git a/docs/why.md b/docs/why.md index d73d7f86f..009d244c1 100644 --- a/docs/why.md +++ b/docs/why.md @@ -12,7 +12,7 @@ From the team behind Pydantic, **Logfire** is an observability platform built on Logfire's dashboard is simple relative to the power it provides, ensuring your entire engineering team will actually use it. Time-to-first-log should be less than 5 minutes. - [:octicons-arrow-right-24: Read more](#simplicity-and-power-) + [:octicons-arrow-right-24: Read more](#simplicity-and-power) - :snake:{ .lg .middle } __Python-centric Insights__ @@ -20,7 +20,7 @@ From the team behind Pydantic, **Logfire** is an observability platform built on From rich display of **Python objects**, to **event-loop telemetry**, to **profiling Python code & database queries**, Logfire gives you unparalleled visibility into your Python application's behavior. - [:octicons-arrow-right-24: Read more](#python-centric-insights-material-snake) + [:octicons-arrow-right-24: Read more](#python-centric-insights) - :simple-pydantic:{ .lg .middle } __Pydantic Integration__ @@ -38,7 +38,7 @@ From the team behind Pydantic, **Logfire** is an observability platform built on Logfire is an opinionated wrapper around OpenTelemetry, allowing you to leverage existing tooling, infrastructure, and instrumentation for many common Python packages, and enabling support for virtually any language. - [:octicons-arrow-right-24: Read more](#opentelemetry-under-the-hood-) + [:octicons-arrow-right-24: Read more](#opentelemetry-under-the-hood) - :simple-instructure:{ .lg .middle } __Structured Data__ @@ -46,7 +46,7 @@ From the team behind Pydantic, **Logfire** is an observability platform built on Include your Python objects in Logfire calls (lists, dict, dataclasses, Pydantic models, DataFrames, and more), and it'll end up as structured data in our platform ready to be queried. - [:octicons-arrow-right-24: Read more](#structured-data-and-sql--sql) + [:octicons-arrow-right-24: Read more](#sql) - :abacus:{ .lg .middle } __SQL__ @@ -54,7 +54,7 @@ From the team behind Pydantic, **Logfire** is an observability platform built on Query your data using standard SQL — all the control and (for many) nothing new to learn. Using SQL also means you can query your data with existing BI tools and database querying libraries. - [:octicons-arrow-right-24: Read more](#structured-data-and-sql--sql) + [:octicons-arrow-right-24: Read more](#sql) diff --git a/mkdocs.yml b/mkdocs.yml index 5cc0a60a2..7d1d735d8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,18 +79,15 @@ nav: - Dashboards: guides/web-ui/dashboards.md - Alerts (Beta): guides/web-ui/alerts.md - SQL Explorer: guides/web-ui/explore.md - - Advanced User Guide: - - Environments: guides/advanced/environments.md - - Alternative Clients: guides/advanced/alternative-clients.md - - Alternative Backends: guides/advanced/alternative-backends.md - - Sampling: guides/advanced/sampling.md - - Scrubbing: guides/advanced/scrubbing.md - - Generators: guides/advanced/generators.md - - Creating Write Tokens: guides/advanced/creating-write-tokens.md - - Query API: guides/advanced/query-api.md - - Link to Code Source: guides/advanced/link-to-code-source.md - - Testing: guides/advanced/testing.md - - Backfill: guides/advanced/backfill.md + - How To - Guides: + - Create Write Tokens: how-to-guides/create-write-tokens.md + - Use different environments: how-to-guides/environments.md + - Link to Code Source: how-to-guides/link-to-code-source.md + - Use Alternative Clients: how-to-guides/alternative-clients.md + - Use Alternative Backends: how-to-guides/alternative-backends.md + - Implement Sampling Strategies: how-to-guides/sampling.md + - Export your Logfire Data: how-to-guides/query-api.md + - Scrub Sensitive Data: how-to-guides/scrubbing.md - Integrations: - Integrations: integrations/index.md - LLMs: @@ -133,6 +130,10 @@ nav: - Stripe: integrations/stripe.md - AWS Lambda: integrations/aws-lambda.md - Reference: + - Advanced: + - Generators: reference/advanced/generators.md + - Testing: reference/advanced/testing.md + - Backfill: reference/advanced/backfill.md - Examples: reference/examples.md - Configuration: reference/configuration.md - Organization Structure: reference/organization-structure.md @@ -225,9 +226,21 @@ plugins: "guides/web_ui/dashboards.md": "guides/web-ui/dashboards.md" "guides/web_ui/alerts.md": "guides/web-ui/alerts.md" "guides/web_ui/explore.md": "guides/web-ui/explore.md" - "guides/advanced/alternative_backends.md": "guides/advanced/alternative-backends.md" - "guides/advanced/creating_write_tokens.md": "guides/advanced/creating-write-tokens.md" - "guides/advanced/index.md": "guides/advanced/environments.md" + "guides/advanced/generators.md": "reference/advanced/generators.md" + "guides/advanced/testing.md": "reference/advanced/testing.md" + "guides/advanced/backfill.md": "reference/advanced/backfill.md" + "guides/advanced/query_api.md": "how-to-guides/query-api.md" + "guides/advanced/query-api.md": "how-to-guides/query-api.md" + "guides/advanced/scrubbing.md": "how-to-guides/scrubbing.md" + "guides/advanced/sampling.md": "how-to-guides/sampling.md" + "guides/advanced/alternative-clients.md": "how-to-guides/alternative-clients.md" + "guides/advanced/link-to-code-source.md": "how-to-guides/link-to-code-source.md" + "guides/advanced/alternative_backends.md": "how-to-guides/alternative-backends.md" + "guides/advanced/alternative-backends.md": "how-to-guides/alternative-backends.md" + "guides/advanced/creating_write_tokens.md": "how-to-guides/create-write-tokens.md" + "guides/advanced/creating-write-tokens.md": "how-to-guides/create-write-tokens.md" + "guides/advanced/index.md": "how-to-guides/environments.md" + "guides/advanced/environments.md": "how-to-guides/environments.md" "integrations/system_metrics.md": "integrations/system-metrics.md" "integrations/third_party/index.md": "integrations/index.md" "integrations/third-party/index.md": "integrations/index.md" @@ -281,7 +294,6 @@ plugins: "api/integrations/pydantic.md": "reference/api/pydantic.md" "api/integrations/logging.md": "reference/api/logfire.md#logfire.LogfireLoggingHandler" "guides/onboarding_checklist/add_metrics.md": "guides/onboarding-checklist/add-metrics.md" - "guides/advanced/query_api.md": "guides/advanced/query-api.md" "guides/index.md": "index.md" "why-logfire/index.md": "why.md" "why-logfire/pydantic.md": "why.md" From 5ef452bd395fdb37d47a1d202fde4dbc15f4f106 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Fri, 20 Dec 2024 12:10:29 +0100 Subject: [PATCH 10/10] docs: fix highlight on the wrong line (#709) --- docs/integrations/llms/anthropic.md | 2 +- docs/integrations/llms/mirascope.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/llms/anthropic.md b/docs/integrations/llms/anthropic.md index 710ecc764..6bcbcd0c2 100644 --- a/docs/integrations/llms/anthropic.md +++ b/docs/integrations/llms/anthropic.md @@ -6,7 +6,7 @@ integration: logfire **Logfire** supports instrumenting calls to [Anthropic](https://github.com/anthropics/anthropic-sdk-python) with one extra line of code. -```python hl_lines="6" +```python hl_lines="7" import anthropic import logfire diff --git a/docs/integrations/llms/mirascope.md b/docs/integrations/llms/mirascope.md index 86aa0f935..e4d4e5213 100644 --- a/docs/integrations/llms/mirascope.md +++ b/docs/integrations/llms/mirascope.md @@ -40,7 +40,7 @@ Since Mirascope is built on top of [Pydantic][pydantic], you can use the [Pydant This can be particularly useful when [extracting structured information][mirascope-extracting-structured-information] using LLMs: -```py hl_lines="3 5 8 17" +```py hl_lines="3 5 8 18" from typing import Literal, Type import logfire