Add OpenTelemetry instrument for FastMCP

README.md

@@ -440,7 +440,7 @@ class YourSettings(BaseServiceSettings):
     opentelemetry_namespace: str | None = None
     opentelemetry_insecure: bool = True
     opentelemetry_instrumentors: list[OpenTelemetryInstrumentor] = []
-    opentelemetry_exclude_urls: list[str] = []
+    opentelemetry_exclude_urls: list[str] = ["/metrics"]
     opentelemetry_baggage_span_attributes: dict[str, str] = {}
 
     ... # Other settings here
@@ -456,9 +456,9 @@ Parameters description:
 - `opentelemetry_insecure` - is opentelemetry connection secure.
 - `opentelemetry_container_name` - will be passed to the `Resource`.
 - `opentelemetry_instrumentors` - a list of extra instrumentors.
-- `opentelemetry_exclude_urls` - list of ignored urls.
+- `opentelemetry_exclude_urls` - list of url regexes that produce no server spans (`["/metrics"]` by default). For Litestar they are combined with `OTEL_PYTHON_LITESTAR_EXCLUDED_URLS` (or `OTEL_PYTHON_EXCLUDED_URLS`), for FastMCP with `OTEL_PYTHON_STARLETTE_EXCLUDED_URLS` (or `OTEL_PYTHON_EXCLUDED_URLS`).
 - `opentelemetry_log_traces` - traces will be logged to stdout.
-- `opentelemetry_generate_health_check_spans` - generate spans for health check handlers if `True`
+- `opentelemetry_generate_health_check_spans` - generate spans for health check handlers if `True`; if `False`, `health_checks_path` is added to the excluded urls.
 - `opentelemetry_baggage_span_attributes` - maps allowed baggage keys to attributes added to local server and consumer spans.
 
 These settings are subsequently passed to [opentelemetry](https://opentelemetry.io/), finalizing your Opentelemetry integration.
@@ -481,7 +481,7 @@ Only non-`None` baggage values supplied to the scope and present in the mapping
 
 #### FastStream
 
-For FastStream you also should pass `opentelemetry_middleware_cls` - OpenTelemetry middleware for your broker
+To trace broker messages, pass `opentelemetry_middleware_cls` - OpenTelemetry middleware for your broker
 
 ```python
 from microbootstrap import FastStreamSettings, FastStreamTelemetryMiddlewareProtocol
@@ -494,6 +494,68 @@ class YourSettings(FastStreamSettings):
     ...
 ```
 
+HTTP requests to the FastStream ASGI application (health checks, AsyncAPI docs and other `asgi_routes`) are wrapped in
+[`OpenTelemetryMiddleware`](https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/asgi/asgi.html)
+the same way as in Litestar, so each request produces a `SERVER` span named like `GET /health/` with the `http.route` attribute
+and the response status code. This does not require `opentelemetry_middleware_cls`. Requests to unknown paths produce spans
+named after the HTTP method only, without `http.route`. Lifespan events bypass the middleware.
+
+- `opentelemetry_exclude_urls` - urls without spans, `/metrics` by default.
+- `opentelemetry_generate_health_check_spans` - set to `False` to skip spans for `health_checks_path`.
+- The status code attribute name depends on `OTEL_SEMCONV_STABILITY_OPT_IN`: `http.status_code` when unset,
+  `http.response.status_code` for `http`, both for `http/dup`.
+
+To wrap HTTP requests in your own ASGI middleware, use `add_http_middleware` on the bootstrapped application.
+It accepts a factory that receives the current HTTP ASGI app and returns the wrapped one:
+
+```python
+application = FastStreamBootstrapper(settings).bootstrap()
+application.add_http_middleware(lambda app: YourAsgiMiddleware(app))
+```
+
+#### FastMCP
+
+`FastMcpSettings` include all OpenTelemetry settings, so tracing is enabled with `opentelemetry_endpoint` alone:
+
+```python
+from fastmcp import FastMCP
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+
+from microbootstrap import FastMcpSettings
+from microbootstrap.bootstrappers.fastmcp import FastMcpBootstrapper
+from microbootstrap.instruments.opentelemetry_instrument import OpenTelemetryInstrumentor
+
+
+class YourSettings(FastMcpSettings):
+    opentelemetry_endpoint: str | None = "otel-collector:4317"
+    opentelemetry_instrumentors: list[OpenTelemetryInstrumentor] = [OpenTelemetryInstrumentor(HTTPXClientInstrumentor())]
+
+
+application: FastMCP = FastMcpBootstrapper(YourSettings()).bootstrap()
+http_application = application.http_app(path="/mcp")
+```
+
+FastMCP creates its ASGI application only when `http_app()` is called (directly or by `application.run(transport="http")`),
+so `SERVER` spans are added to every application returned by `http_app()`. Each request to it is wrapped in
+[`OpenTelemetryMiddleware`](https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/asgi/asgi.html)
+and produces a span named like `POST /mcp` or `GET /health/` with the `http.route` attribute and the response status code.
+Requests to unknown paths produce spans named after the HTTP method only, without `http.route`.
+
+- `opentelemetry_endpoint` - OTLP endpoint for exported traces.
+- `opentelemetry_instrumentors` - extra instrumentors, e.g. for HTTP clients used by your tools.
+- `opentelemetry_exclude_urls` - urls without spans, `["/metrics"]` by default. Combined with `OTEL_PYTHON_STARLETTE_EXCLUDED_URLS`.
+- `opentelemetry_generate_health_check_spans` - set to `False` to skip spans for `health_checks_path`.
+- The status code attribute name depends on `OTEL_SEMCONV_STABILITY_OPT_IN`: `http.status_code` when unset,
+  `http.response.status_code` for `http`, both for `http/dup`.
+
+Applications already instrumented by `StarletteInstrumentor` are left as is, so the request is never traced twice.
+To post-process every created ASGI application yourself, use `add_http_app_hook` on the bootstrapped application:
+
+```python
+application = FastMcpBootstrapper(settings).bootstrap()
+application.add_http_app_hook(lambda http_application: http_application)
+```
+
 ### [Pyroscope](https://pyroscope.io)
 
 To integrate Pyroscope, specify the `pyroscope_endpoint`.

microbootstrap/bootstrappers/fastmcp.py

@@ -4,12 +4,22 @@
 import prometheus_client
 import typing_extensions
 from fastmcp import FastMCP
+from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
+from opentelemetry.util.http import ExcludeList, get_excluded_urls
+from starlette.applications import Starlette
 from starlette.responses import JSONResponse, Response
+from starlette.routing import Match, Mount, Route
 
 from microbootstrap.bootstrappers.base import ApplicationBootstrapper
 from microbootstrap.config.fastmcp import FastMcpConfig
 from microbootstrap.instruments.health_checks_instrument import HealthChecksInstrument, HealthCheckTypedDict
 from microbootstrap.instruments.logging_instrument import LoggingInstrument
+from microbootstrap.instruments.opentelemetry_instrument import (
+    BaseOpentelemetryInstrument,
+    CombinedExcludeList,
+    OpentelemetryConfig,
+    build_span_name,
+)
 from microbootstrap.instruments.prometheus_instrument import FastMcpPrometheusConfig, PrometheusInstrument
 from microbootstrap.instruments.pyroscope_instrument import PyroscopeInstrument
 from microbootstrap.instruments.sentry_instrument import SentryInstrument
@@ -18,16 +28,44 @@
 
 
 if typing.TYPE_CHECKING:
+    from fastmcp.server.http import StarletteWithLifespan
     from starlette.requests import Request
+    from starlette.types import Scope
+
+
+StarletteT = typing.TypeVar("StarletteT", bound=Starlette)
 
 
 class KwargsFastMCP(FastMCP[typing.Any]):
     def __init__(self, **kwargs: typing.Any) -> None:  # noqa: ANN401
         super().__init__(**kwargs)
+        self.http_app_hooks: list[typing.Callable[[StarletteWithLifespan], StarletteWithLifespan]] = []
+
+    def add_http_app_hook(self, hook: typing.Callable[[StarletteWithLifespan], StarletteWithLifespan]) -> None:
+        self.http_app_hooks.append(hook)
+
+    def http_app(self, *args: typing.Any, **kwargs: typing.Any) -> StarletteWithLifespan:  # noqa: ANN401
+        # ASGI application is created by the user after bootstrap, so instruments subscribe to its creation
+        http_application = super().http_app(*args, **kwargs)
+        for hook in self.http_app_hooks:
+            http_application = hook(http_application)
+        return http_application
+
+
+def build_fastmcp_route_details_from_scope(
+    scope: Scope,
+    routes: typing.Iterable[typing.Any],
+) -> tuple[str, dict[str, str]]:
+    method: typing.Final = str(scope.get("method", "HTTP")).strip()
+    for route in routes:
+        if isinstance(route, (Route, Mount)) and route.matches(scope)[0] == Match.FULL:
+            return build_span_name(method, route.path), {"http.route": route.path}
+    # Unmatched paths get no `http.route` to keep its cardinality low
+    return method, {}
 
 
 class FastMcpBootstrapper(
-    ApplicationBootstrapper[FastMcpSettings, FastMCP[typing.Any], FastMcpConfig],
+    ApplicationBootstrapper[FastMcpSettings, KwargsFastMCP, FastMcpConfig],
 ):
     application_config = FastMcpConfig()
     application_type = KwargsFastMCP
@@ -41,8 +79,8 @@ def bootstrap_before(self: typing_extensions.Self) -> dict[str, typing.Any]:
 
     def bootstrap_before_instruments_after_app_created(
         self,
-        application: FastMCP[typing.Any],
-    ) -> FastMCP[typing.Any]:
+        application: KwargsFastMCP,
+    ) -> KwargsFastMCP:
         self.console_writer.print_bootstrap_table()
         return application
 
@@ -51,6 +89,38 @@ def bootstrap_before_instruments_after_app_created(
 FastMcpBootstrapper.use_instrument()(PyroscopeInstrument)
 
 
+@FastMcpBootstrapper.use_instrument()
+class FastMcpOpentelemetryInstrument(BaseOpentelemetryInstrument[OpentelemetryConfig]):
+    def bootstrap_after(self, application: FastMCP[typing.Any]) -> FastMCP[typing.Any]:  # type: ignore[override]
+        if isinstance(application, KwargsFastMCP):
+            application.add_http_app_hook(self.instrument_http_app)
+        return application
+
+    def instrument_http_app(self, http_application: StarletteT) -> StarletteT:
+        # `StarletteInstrumentor` marks applications the same way, so each application is instrumented once
+        if getattr(http_application, "_is_instrumented_by_opentelemetry", False):
+            return http_application
+
+        def build_route_details(scope: Scope) -> tuple[str, dict[str, str]]:
+            return build_fastmcp_route_details_from_scope(scope, http_application.routes)
+
+        http_application.add_middleware(
+            OpenTelemetryMiddleware,
+            tracer_provider=self.tracer_provider,
+            default_span_details=build_route_details,
+            excluded_urls=CombinedExcludeList(
+                ExcludeList(self.define_exclude_urls()),
+                get_excluded_urls("STARLETTE"),
+            ),
+        )
+        http_application._is_instrumented_by_opentelemetry = True  # type: ignore[attr-defined]  # noqa: SLF001
+        return http_application
+
+    @classmethod
+    def get_config_type(cls) -> type[OpentelemetryConfig]:
+        return OpentelemetryConfig
+
+
 @FastMcpBootstrapper.use_instrument()
 class FastMcpLoggingInstrument(LoggingInstrument):
     def bootstrap_after(self, application: FastMCP[typing.Any]) -> FastMCP[typing.Any]:  # type: ignore[override]

microbootstrap/bootstrappers/faststream.py

@@ -11,7 +11,8 @@
 from faststream.asgi import AsgiFastStream, AsgiResponse
 from faststream.asgi import get as handle_get
 from faststream.specification import AsyncAPI
-from opentelemetry import trace
+from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
+from opentelemetry.util.http import ExcludeList
 
 from microbootstrap.bootstrappers.base import ApplicationBootstrapper
 from microbootstrap.config.faststream import FastStreamConfig
@@ -20,6 +21,7 @@
 from microbootstrap.instruments.opentelemetry_instrument import (
     BaseOpentelemetryInstrument,
     FastStreamOpentelemetryConfig,
+    build_span_name,
 )
 from microbootstrap.instruments.prometheus_instrument import FastStreamPrometheusConfig, PrometheusInstrument
 from microbootstrap.instruments.pyroscope_instrument import PyroscopeInstrument
@@ -28,7 +30,10 @@
 from microbootstrap.settings import FastStreamSettings
 
 
-tracer: typing.Final = trace.get_tracer(__name__)
+if typing.TYPE_CHECKING:
+    from faststream.asgi.types import ASGIApp, Receive, Scope, Send
+
+
 MessageT = typing.TypeVar("MessageT")
 ResponseT = typing.TypeVar("ResponseT")
 
@@ -58,9 +63,32 @@ class KwargsAsgiFastStream(AsgiFastStream):
     def __init__(self, **kwargs: typing.Any) -> None:  # noqa: ANN401
         # `broker` argument is positional-only
         super().__init__(kwargs.pop("broker", None), **kwargs)
+        self.http_app: ASGIApp = super().__call__
+
+    def add_http_middleware(self, build_middleware: typing.Callable[[ASGIApp], ASGIApp]) -> None:
+        self.http_app = build_middleware(self.http_app)
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        # Lifespan and websocket scopes bypass HTTP middlewares
+        if scope["type"] == "http":
+            await self.http_app(scope, receive, send)
+            return
+        await super().__call__(scope, receive, send)
 
 
-class FastStreamBootstrapper(ApplicationBootstrapper[FastStreamSettings, AsgiFastStream, FastStreamConfig]):
+def build_faststream_route_details_from_scope(
+    scope: Scope,
+    routes: typing.Iterable[tuple[str, ASGIApp]],
+) -> tuple[str, dict[str, str]]:
+    method: typing.Final = str(scope.get("method", "HTTP")).strip()
+    path: typing.Final = scope.get("path")
+    # FastStream matches ASGI routes by exact path, unmatched paths get no `http.route` to keep its cardinality low
+    if path is None or all(path != route_path for route_path, _ in routes):
+        return method, {}
+    return build_span_name(method, path), {"http.route": path}
+
+
+class FastStreamBootstrapper(ApplicationBootstrapper[FastStreamSettings, KwargsAsgiFastStream, FastStreamConfig]):
     application_config = FastStreamConfig()
     application_type = KwargsAsgiFastStream
 
@@ -94,9 +122,6 @@ def bootstrap_after(self, application: AsgiFastStream) -> AsgiFastStream:  # typ
 
 @FastStreamBootstrapper.use_instrument()
 class FastStreamOpentelemetryInstrument(BaseOpentelemetryInstrument[FastStreamOpentelemetryConfig]):
-    def is_ready(self) -> bool:
-        return bool(self.instrument_config.opentelemetry_middleware_cls and super().is_ready())
-
     def bootstrap_after(self, application: AsgiFastStream) -> AsgiFastStream:  # type: ignore[override]
         if self.instrument_config.opentelemetry_middleware_cls and application.broker:
             application.broker.add_middleware(
@@ -107,8 +132,23 @@ def bootstrap_after(self, application: AsgiFastStream) -> AsgiFastStream:  # typ
                     baggage_span_attributes=self.instrument_config.opentelemetry_baggage_span_attributes,
                 ),
             )
+        if isinstance(application, KwargsAsgiFastStream):
+            application.add_http_middleware(
+                functools.partial(self.create_open_telemetry_middleware, application=application),
+            )
         return application
 
+    def create_open_telemetry_middleware(self, app: ASGIApp, application: AsgiFastStream) -> ASGIApp:
+        def build_route_details(scope: Scope) -> tuple[str, dict[str, str]]:
+            return build_faststream_route_details_from_scope(scope, application.routes)
+
+        return OpenTelemetryMiddleware(
+            app=app,
+            default_span_details=build_route_details,
+            excluded_urls=ExcludeList(self.define_exclude_urls()),
+            tracer_provider=self.tracer_provider,
+        )
+
     @classmethod
     def get_config_type(cls) -> type[FastStreamOpentelemetryConfig]:
         return FastStreamOpentelemetryConfig
@@ -175,11 +215,6 @@ async def check_health(scope: typing.Any) -> AsgiResponse:  # noqa: ANN401, ARG0
                 else AsgiResponse(b"Service is unhealthy", 500, headers={"content-type": "application/json"})
             )
 
-        if self.instrument_config.opentelemetry_generate_health_check_spans:
-            check_health = tracer.start_as_current_span(f"GET {self.instrument_config.health_checks_path}")(
-                check_health,
-            )
-
         return {"asgi_routes": ((self.instrument_config.health_checks_path, check_health),)}
 
     async def define_health_status(self) -> bool:

microbootstrap/bootstrappers/litestar.py

@@ -14,7 +14,7 @@
 from litestar.types.asgi_types import ASGIApp, Scope
 from litestar_offline_docs import generate_static_files_config
 from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
-from opentelemetry.util.http import get_excluded_urls
+from opentelemetry.util.http import ExcludeList, get_excluded_urls
 from sentry_sdk.integrations.litestar import LitestarIntegration
 
 from microbootstrap.bootstrappers.base import ApplicationBootstrapper
@@ -25,7 +25,11 @@
     HealthCheckTypedDict,
 )
 from microbootstrap.instruments.logging_instrument import LoggingInstrument
-from microbootstrap.instruments.opentelemetry_instrument import OpentelemetryInstrument
+from microbootstrap.instruments.opentelemetry_instrument import (
+    CombinedExcludeList,
+    OpentelemetryInstrument,
+    build_span_name,
+)
 from microbootstrap.instruments.prometheus_instrument import (
     LitestarPrometheusConfig,
     PrometheusInstrument,
@@ -122,12 +126,6 @@ def bootstrap_before(self) -> dict[str, typing.Any]:
 LitestarBootstrapper.use_instrument()(PyroscopeInstrument)
 
 
-def build_span_name(method: str, route: str) -> str:
-    if not route:
-        return method
-    return f"{method} {route}"
-
-
 def build_litestar_route_details_from_scope(
     scope: Scope,
 ) -> tuple[str, dict[str, str]]:
@@ -154,16 +152,20 @@ def build_litestar_route_details_from_scope(
 
 
 class LitestarOpenTelemetryInstrumentationMiddleware(ASGIMiddleware):
-    def __init__(self, config: OpenTelemetryConfig) -> None:
+    def __init__(self, config: OpenTelemetryConfig, exclude_urls: typing.Sequence[str] = ()) -> None:
         self.config = config
+        self.excluded_urls = CombinedExcludeList(
+            ExcludeList(exclude_urls),
+            get_excluded_urls(self.config.exclude_urls_env_key),
+        )
 
     def create_open_telemetry_middleware(self, app: ASGIApp) -> OpenTelemetryMiddleware:
         return OpenTelemetryMiddleware(
             app=app,
             client_request_hook=self.config.client_request_hook_handler,
             client_response_hook=self.config.client_response_hook_handler,
             default_span_details=build_litestar_route_details_from_scope,
-            excluded_urls=get_excluded_urls(self.config.exclude_urls_env_key),
+            excluded_urls=self.excluded_urls,
             meter=self.config.meter,
             meter_provider=self.config.meter_provider,
             server_request_hook=self.config.server_request_hook_handler,
@@ -183,7 +185,8 @@ def bootstrap_before(self) -> dict[str, typing.Any]:
                     LitestarOpentelemetryConfig(
                         tracer_provider=self.tracer_provider,
                         middleware_class=LitestarOpenTelemetryInstrumentationMiddleware,  # type: ignore[arg-type]
-                    )
+                    ),
+                    exclude_urls=self.define_exclude_urls(),
                 )
             ]
         }