Add Tracing and rework cli so that the controller command does not st…

…art a FastAPI app (#669)

- Add tracing intialisation and decorators
- In cli.py move some imports within functions to prevent unnecessary
FastAPI apps being created
- Add capability to specify a config file in debug launchers
- Add Environment vars to initialise traceability options and set Jager
export to off by default

---------

Co-authored-by: Callum Forrester <[email protected]>

.devcontainer/devcontainer.json

@@ -5,6 +5,10 @@
         "dockerfile": "../Dockerfile",
         "target": "developer"
     },
+    "containerEnv": {
+        "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf",
+        "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:4318",
+    },
     "remoteEnv": {
         // Allow X11 apps to run inside the container
         "DISPLAY": "${localEnv:DISPLAY}"

.github/pages/make_switcher.py

@@ -52,7 +52,7 @@ def get_versions(ref: str, add: str | None) -> list[str]:
     return versions
 
 
-def write_json(path: Path, repository: str, versions: str):
+def write_json(path: Path, repository: str, versions: list[str]):
     org, repo_name = repository.split("/")
     struct = [
         {"version": version, "url": f"https://{org}.github.io/{repo_name}/{version}/"}

.vscode/launch.json

@@ -25,25 +25,43 @@
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": [
-                "serve"
-            ]
+            "args": "serve",
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
         },
         {
             "name": "Blueapi Server (Custom Config)",
             "type": "debugpy",
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": "--config ${input:config_path} serve"
+            "args": "--config ${input:config_path} serve",
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
         },
         {
             "name": "Blueapi Controller",
             "type": "debugpy",
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": "controller ${input:controller_args}"
+            "args": "controller ${input:controller_args}",
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
+        },
+        {
+            "name": "Blueapi Controller (Custom Config)",
+            "type": "debugpy",
+            "request": "launch",
+            "justMyCode": false,
+            "module": "blueapi",
+            "args": "-c ${input:config_path} controller ${input:controller_args}",
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
         },
     ],
     "inputs": [
@@ -56,7 +74,8 @@
         {
             "id": "config_path",
             "type": "promptString",
-            "description": "Server config file path",
-        }
+            "description": "Path to configuration YAML file",
+            "default": ""
+        },
     ]
 }

dev-requirements.txt

@@ -9,8 +9,11 @@ annotated-types==0.7.0
 anyio==4.6.2.post1
 appdirs==1.4.4
 asciitree==0.3.3
+asgiref==3.8.1
 asttokens==2.4.1
+async-timeout==4.0.3
 attrs==24.2.0
+autodoc_pydantic==2.2.0
 babel==2.16.0
 beautifulsoup4==4.12.3
 bidict==0.23.1
@@ -20,6 +23,7 @@ bluesky-kafka==0.10.0
 bluesky-live==0.0.8
 bluesky-stomp==0.1.2
 boltons==24.0.0
+bump-pydantic==0.8.0
 cachetools==5.5.0
 caproto==1.1.1
 certifi==2024.8.30
@@ -53,11 +57,14 @@ docopt==0.6.2
 doct==1.1.0
 docutils==0.21.2
 dunamai==1.22.0
+email_validator==2.2.0
 entrypoints==0.4
 epicscorelibs==7.0.7.99.1.1
 event-model==1.21.0
+exceptiongroup==1.2.2
 executing==2.1.0
 fastapi==0.115.3
+fastapi-cli==0.0.5
 fasteners==0.19
 filelock==3.16.1
 flexcache==0.3
@@ -68,12 +75,15 @@ fsspec==2024.10.0
 funcy==2.0
 gitdb==4.0.11
 GitPython==3.1.43
+googleapis-common-protos==1.65.0
 graypy==2.1.0
+grpcio==1.66.2
 h11==0.14.0
 h5py==3.12.1
 HeapDict==1.0.1
 historydict==1.2.6
 httpcore==1.0.6
+httptools==0.6.1
 httpx==0.27.2
 humanize==4.11.0
 identify==2.6.1
@@ -86,14 +96,18 @@ iniconfig==2.0.0
 intake==0.6.4
 ipython==8.18.0
 ipywidgets==8.1.5
+itsdangerous==2.2.0
 jedi==0.19.1
 Jinja2==3.1.4
 jinja2-ansible-filters==1.3.2
+jsonpointer==3.0.0
 jsonschema==4.23.0
 jsonschema-specifications==2024.10.1
 jupyterlab_widgets==3.0.13
 kiwisolver==1.4.7
 ldap3==2.9.1
+libcst==1.4.0
+livereload==2.7.0
 locket==1.0.0
 lz4==4.3.3
 markdown-it-py==3.0.0
@@ -118,8 +132,21 @@ nose2==0.15.1
 nslsii==0.10.5
 numcodecs==0.13.1
 numpy==1.26.4
+observability-utils==0.1.2
 opencv-python-headless==4.10.0.84
 opentelemetry-api==1.27.0
+opentelemetry-distro==0.48b0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-exporter-otlp-proto-common==1.27.0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-instrumentation==0.48b0
+opentelemetry-instrumentation-asgi==0.48b0
+opentelemetry-instrumentation-fastapi==0.48b0
+opentelemetry-proto==1.27.0
+opentelemetry-sdk==1.27.0
+opentelemetry-semantic-conventions==0.48b0
+opentelemetry-util-http==0.48b0
 ophyd==1.9.0
 ophyd-async==0.6.0
 orderly-set==5.2.2
@@ -146,6 +173,7 @@ pre_commit==4.0.1
 prettytable==3.11.0
 prompt-toolkit==3.0.36
 propcache==0.2.0
+protobuf==4.25.5
 psutil==6.1.0
 ptyprocess==0.7.0
 pure_eval==0.2.3
@@ -154,6 +182,7 @@ py==1.11.0
 pyasn1==0.6.1
 pycryptodome==3.21.0
 pydantic==2.9.2
+pydantic-extra-types==2.9.0
 pydantic-settings==2.6.0
 pydantic_core==2.23.4
 pydantic_numpy==5.0.2
@@ -170,22 +199,25 @@ pytest-cov==5.0.0
 pytest-random-order==1.1.1
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
+python-multipart==0.0.9
 pytz==2024.2
 PyYAML==6.0.2
+pyyaml-include==2.1
 questionary==2.0.1
 redis==5.2.0
 redis-json-dict==0.2.1
 referencing==0.35.1
 requests==2.32.3
 responses==0.25.3
+rich==13.7.1
 rpds-py==0.20.0
 ruamel.yaml==0.18.6
 ruamel.yaml.clib==0.2.12
 ruff==0.7.1
 scanspec==0.7.6
 semver==3.0.2
-setuptools==75.2.0
 setuptools-dso==2.11
+shellingham==1.5.4
 six==1.16.0
 slicerator==1.1.0
 smmap==5.0.1
@@ -197,6 +229,8 @@ sphinx-autobuild==2024.10.3
 sphinx-autodoc-typehints==2.3.0
 sphinx-click==6.0.0
 sphinx-copybutton==0.5.2
+sphinx-jsonschema==1.19.1
+sphinx-pydantic==0.1.1
 sphinx_design==0.6.1
 sphinx_mdinclude==0.6.2
 sphinxcontrib-applehelp==2.0.0
@@ -216,11 +250,14 @@ suitcase-msgpack==0.3.0
 suitcase-utils==0.5.4
 super-state-machine==2.0.2
 tifffile==2024.9.20
+tomli==2.0.1
 toolz==1.0.0
+tornado==6.4.1
 tox==3.28.0
 tox-direct==0.4
 tqdm==4.66.5
 traitlets==5.14.3
+typer==0.12.4
 types-aiofiles==24.1.0.20240626
 types-mock==5.1.0.20240425
 types-PyYAML==6.0.12.20240917
@@ -230,8 +267,10 @@ typing-inspect==0.9.0
 typing_extensions==4.12.2
 tzdata==2024.2
 tzlocal==5.2
+ujson==5.10.0
 urllib3==2.2.3
 uvicorn==0.32.0
+uvloop==0.19.0
 virtualenv==20.27.0
 watchfiles==0.24.0
 wcwidth==0.2.13

docs/reference/openapi.yaml

@@ -179,6 +179,10 @@ components:
           default: true
           title: Is Pending
           type: boolean
+        request_id:
+          default: ''
+          title: Request Id
+          type: string
         task:
           title: Task
         task_id:

helm/blueapi/values.yaml

@@ -76,13 +76,26 @@ listener:
   resources: {}
 
 # Additional envVars to mount to the pod as a String
-extraEnvVars: []
+extraEnvVars: |
+  - name: OTLP_EXPORT_ENABLED
+    value: {{ .Values.tracing.otlp.export_enabled }}
+  - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL
+    value: {{ .Values.tracing.otlp.protocol }}
+  - name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: "{{ .Values.tracing.otlp.host }}:{{ .Values.tracing.otlp.port }}"
 # - name: RABBITMQ_PASSWORD
 #   valueFrom:
 #     secretKeyRef:
 #       name: rabbitmq-password
 #       key: rabbitmq-password
 
+tracing:
+  otlp:
+    export_enabled: false
+    protocol: http/protobuf
+    host: https://daq-services-jaeger # replace with central instance
+    port: 4318
+
 # Config for the worker goes here, will be mounted into a config file
 worker:
   api:

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
     "pyepics",
     "aioca",
     "pydantic>=2.0",
-    "scanspec>=0.7.2",
+    "scanspec>=0.7.6",
     "pydantic-settings",
     "stomp-py",
     "aiohttp",
@@ -27,12 +27,15 @@ dependencies = [
     "fastapi>=0.112.0",
     "uvicorn",
     "requests",
-    "dls-bluesky-core",     #requires ophyd-async
+    "dls-bluesky-core",                              #requires ophyd-async
     "dls-dodal>=1.31.0",
     "super-state-machine",  # https://github.com/DiamondLightSource/blueapi/issues/553
     "GitPython",
     "bluesky-stomp>=0.1.2",
     "event-model==1.21",    # https://github.com/DiamondLightSource/blueapi/issues/684
+    "opentelemetry-distro>=0.48b0",
+    "opentelemetry-instrumentation-fastapi>=0.48b0",
+    "observability-utils>=0.1.2",
 ]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -80,7 +83,9 @@ write_to = "src/blueapi/_version.py"
 
 [tool.mypy]
 ignore_missing_imports = true # Ignore missing stubs in imported modules
-namespace_packages = false    # rely only on __init__ files to determine fully qualified module names.
+
+# necessary for tracing sdk to work with mypy, set false once migraion to pyright complete
+namespace_packages = true
 
 [tool.pytest.ini_options]
 # Run pytest with all our checkers, and don't spam us with massive tracebacks on error

src/blueapi/cli/cli.py

@@ -9,6 +9,7 @@
 from bluesky.callbacks.best_effort import BestEffortCallback
 from bluesky_stomp.messaging import MessageContext, StompClient
 from bluesky_stomp.models import Broker
+from observability_utils.tracing import setup_tracing
 from pydantic import ValidationError
 from requests.exceptions import ConnectionError
 
@@ -18,14 +19,7 @@
 from blueapi.client.event_bus import AnyEvent, BlueskyStreamingError, EventBusClient
 from blueapi.client.rest import BlueskyRemoteControlError
 from blueapi.config import ApplicationConfig, ConfigLoader
-from blueapi.core import DataEvent
-from blueapi.service.main import start
-from blueapi.service.openapi import (
-    DOCS_SCHEMA_LOCATION,
-    generate_schema,
-    print_schema_as_yaml,
-    write_schema_as_yaml,
-)
+from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent
 from blueapi.worker import ProgressEvent, Task, WorkerEvent
 
 from .scratch import setup_scratch
@@ -72,6 +66,16 @@ def main(ctx: click.Context, config: Path | None | tuple[Path, ...]) -> None:
     help="[Development only] update the schema in the documentation",
 )
 def schema(output: Path | None = None, update: bool = False) -> None:
+    """Only import the service functions when starting the service or generating
+    the schema, not the controller as a new FastAPI app will be started each time.
+    """
+    from blueapi.service.openapi import (
+        DOCS_SCHEMA_LOCATION,
+        generate_schema,
+        print_schema_as_yaml,
+        write_schema_as_yaml,
+    )
+
     """Generate the schema for the REST API"""
     schema = generate_schema()
 
@@ -89,6 +93,16 @@ def start_application(obj: dict):
     """Run a worker that accepts plans to run"""
     config: ApplicationConfig = obj["config"]
 
+    """Only import the service functions when starting the service or generating
+    the schema, not the controller as a new FastAPI app will be started each time.
+    """
+    from blueapi.service.main import start
+
+    """
+    Set up basic automated instrumentation for the FastAPI app, creating the
+    observability context.
+    """
+    setup_tracing("BlueAPI", OTLP_EXPORT_ENABLED)
     start(config)
 
 
@@ -103,6 +117,7 @@ def start_application(obj: dict):
 def controller(ctx: click.Context, output: str) -> None:
     """Client utility for controlling and introspecting the worker"""
 
+    setup_tracing("BlueAPICLI", OTLP_EXPORT_ENABLED)
     if ctx.invoked_subcommand is None:
         print("Please invoke subcommand!")
         return