Version 0.4.0

Changes:
- put ObservabilityMiddleware before ExceptionHandlerMiddleware to avoid repetative code
- add application startup and last metrics update metrics along with CPU usage metric and threads count
- move host and port to new uvicorn section at config along with new reload and forwarded_allow_ips
- add request_id and remove trace_id/span_id generation if tracing is disabled
- move logging logic from utils to observability
- pass trace_id/span_id in HEX form

{{project_slug}}/__main__.py.jinja

@@ -9,16 +9,7 @@ import click
 import uvicorn
 from dotenv import load_dotenv
 
-from .config import LoggingConfig, {{ProjectName}}Config
-
-LogLevel = tp.Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR"]
-
-
-def _run_uvicorn(configuration: dict[str, tp.Any]) -> tp.NoReturn:
-    uvicorn.run(
-        "{{project_slug}}.fastapi_init:app",
-        **configuration,
-    )
+from .config import {{ProjectName}}Config
 
 
 @click.group()
@@ -42,34 +33,6 @@ def get_config_example(config_path: Path):
 
 
 @cli.command("launch")
-@click.option(
-    "--port",
-    "-p",
-    envvar="PORT",
-    type=int,
-    show_envvar=True,
-    help="Service port number",
-)
-@click.option(
-    "--host",
-    envvar="HOST",
-    show_envvar=True,
-    help="Service HOST address",
-)
-@click.option(
-    "--logger_verbosity",
-    "-v",
-    type=click.Choice(("TRACE", "DEBUG", "INFO", "WARNING", "ERROR")),
-    envvar="LOGGER_VERBOSITY",
-    show_envvar=True,
-    help="Logger verbosity",
-)
-@click.option(
-    "--debug",
-    envvar="DEBUG",
-    is_flag=True,
-    help="Enable debug mode (auto-reload on change, traceback returned to user, etc.)",
-)
 @click.option(
     "--config_path",
     envvar="CONFIG_PATH",
@@ -80,10 +43,6 @@ def get_config_example(config_path: Path):
     help="Path to YAML configuration file",
 )
 def launch(
-    port: int,
-    host: str,
-    logger_verbosity: LogLevel,
-    debug: bool,
     config_path: Path,
 ):
     """
@@ -93,14 +52,10 @@ def launch(
     """
     print(
         "This is a simple method to run the API. You might want to use"
-        "'uvicorn {{project_slug}}.fastapi_init:app' instead to configure more uvicorn options."
+        " 'uvicorn {{project_slug}}.fastapi_init:app' instead to configure more uvicorn options."
     )
 
     config = {{ProjectName}}Config.load(config_path)
-    config.app.host = host or config.app.host
-    config.app.port = port or config.app.port
-    config.app.debug = debug or config.app.debug
-    config.observability.logging = config.observability.logging if logger_verbosity is None else LoggingConfig(level=logger_verbosity)
 
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
         temp_yaml_config_path = temp_file.name
@@ -111,16 +66,18 @@ def launch(
         env_file.write(f"CONFIG_PATH={temp_yaml_config_path}\n")
     try:
         uvicorn_config = {
-            "host": config.app.host,
-            "port": config.app.port,
-            "log_level": config.observability.logging.level.lower(),
+            "host": config.app.uvicorn.host,
+            "port": config.app.uvicorn.port,
+            "forwarded_allow_ips": config.app.uvicorn.forwarded_allow_ips,
+            "log_level": config.observability.logging.root_logger_level.lower(),
             "env_file": temp_envfile_path,
+            "access_log": False,
         }
         if config.app.debug:
             try:
                 _run_uvicorn(uvicorn_config | {"reload": True})
             except:  # pylint: disable=bare-except
-                print("Debug reload is not supported and will be disabled")
+                print("Retrying with Uvicorn reload disabled")
                 _run_uvicorn(uvicorn_config)
         else:
             _run_uvicorn(uvicorn_config)
@@ -131,6 +88,13 @@ def launch(
             os.remove(temp_yaml_config_path)
 
 
+def _run_uvicorn(configuration: dict[str, tp.Any]) -> tp.NoReturn:
+    uvicorn.run(
+        "{{project_slug}}.fastapi_init:app",
+        **configuration,
+    )
+
+
 if __name__ in ("__main__", "{{project_slug}}.__main__"):
     load_dotenv(os.environ.get("ENVFILE", ".env"))
     cli()  # pylint: disable=no-value-for-parameter

{{project_slug}}/config.py.jinja

@@ -4,12 +4,19 @@ from collections import OrderedDict
 from dataclasses import asdict, dataclass, field, fields
 from pathlib import Path
 from types import NoneType, UnionType
-from typing import Any, Literal, TextIO, Type
+from typing import Any, Literal, TextIO, Type, Union, get_origin
 
 import yaml
 
 from {{project_slug}}.db.config import DBConfig, MultipleDBsConfig
-from {{project_slug}}.utils.observability import LoggingConfig, FileLogger, ExporterConfig
+from {{project_slug}}.observability.config import (
+    ExporterConfig,
+    FileLogger,
+    JaegerConfig,
+    LoggingConfig,
+    ObservabilityConfig,
+    PrometheusConfig,
+)
 from {{project_slug}}.utils.secrets import SecretStr, representSecretStrYAML
 
 
@@ -22,32 +29,20 @@ class CORSConfig:
 
 
 @dataclass
-class AppConfig:
+class UvicornConfig:
     host: str
     port: int
+    reload: bool = False
+    forwarded_allow_ips: list[str] = field(default_factory=list)
+
+
+@dataclass
+class AppConfig:
+    uvicorn: UvicornConfig
     debug: bool
     cors: CORSConfig
 
 
-@dataclass
-class PrometheusConfig:
-    host: str
-    port: int
-    urls_mapping: dict[str, str] = field(default_factory=dict)
-
-
-@dataclass
-class JaegerConfig:
-    endpoint: str
-
-
-@dataclass
-class ObservabilityConfig:
-    logging: LoggingConfig
-    prometheus: PrometheusConfig | None = None
-    jaeger: JaegerConfig | None = None
-
-
 @dataclass
 class {{ProjectName}}Config:
     app: AppConfig
@@ -96,7 +91,11 @@ class {{ProjectName}}Config:
         """Generate an example of configuration."""
 
         res = cls(
-            app=AppConfig(host="0.0.0.0", port=8080, debug=False, cors=CORSConfig(["*"], ["*"], ["*"], True)),
+            app=AppConfig(
+                uvicorn=UvicornConfig(host="0.0.0.0", port=8080, reload=True, forwarded_allow_ips=["127.0.0.1"]),
+                debug=True,
+                cors=CORSConfig(["*"], ["*"], ["*"], True),
+            ),
             db=MultipleDBsConfig(
                 master=DBConfig(
                     host="localhost",
@@ -119,12 +118,12 @@ class {{ProjectName}}Config:
             ),
             observability=ObservabilityConfig(
                 logging=LoggingConfig(
-                    level="INFO",
+                    stderr_level="INFO",
                     root_logger_level="INFO",
                     exporter=ExporterConfig(endpoint="http://127.0.0.1:4317", level="INFO", tls_insecure=True),
                     files=[FileLogger(filename="logs/info.log", level="INFO")],
                 ),
-                prometheus=PrometheusConfig(host="0.0.0.0", port=9090, urls_mapping={"/api/debug/.*": "/api/debug/*"}),
+                prometheus=PrometheusConfig(host="0.0.0.0", port=9090),
                 jaeger=JaegerConfig(endpoint="http://127.0.0.1:4318/v1/traces"),
             ),
         )
@@ -152,7 +151,7 @@ class {{ProjectName}}Config:
         """Try to initialize given type field-by-field recursively with data from dictionary substituting {} and None
         if no value provided.
         """
-        if isinstance(t, UnionType):
+        if get_origin(t) is Union or get_origin(t) is UnionType:  # both actually required
             for inner_type in t.__args__:
                 if inner_type is NoneType and data is None:
                     return None

{{project_slug}}/dependencies/auth_dep.py

@@ -1,34 +0,0 @@
-"""Authentication dependency function is defined here."""
-
-from dataclasses import dataclass
-
-import jwt
-from fastapi import Request
-
-from . import logger_dep
-
-
-@dataclass
-class AuthenticationData:
-    api_key: str | None
-    jwt_payload: dict | None
-    jwt_original: str | None
-
-
-def obtain(request: Request) -> AuthenticationData:
-    if hasattr(request.state, "auth_dep"):
-        return request.state.auth_dep
-    auth = AuthenticationData(None, None, None)
-    if (value := request.headers.get("X-API-Key")) is not None:
-        auth.api_key = value
-    if (value := request.headers.get("Authorization")) is not None and value.startswith("Bearer "):
-        value = value[7:]
-        auth.jwt_original = value
-        try:
-            auth.jwt_payload = jwt.decode(value, algorithms=["HS256"], options={"verify_signature": False})
-        except Exception:  # pylint: disable=broad-except
-            logger = logger_dep.obtain(request)
-            logger.warning("failed to parse Authorization header as jwt", value=value)
-            logger.debug("failed to parse Authorization header as jwt", exc_info=True)
-    request.state.auth_dep = auth
-    return auth

{{project_slug}}/dependencies/auth_dep.py.jinja

@@ -0,0 +1,46 @@
+"""Authentication dependency function is defined here."""
+
+from dataclasses import dataclass
+
+import jwt
+from fastapi import Request
+
+from {{project_slug}}.exceptions.auth import NotAuthorizedError
+
+from . import logger_dep
+
+
+@dataclass
+class AuthenticationData:
+    api_key: str | None
+    jwt_payload: dict | None
+
+
+def _from_request(request: Request, required: bool = True) -> AuthenticationData | None:
+    if not hasattr(request.state, "auth_dep"):
+        auth = AuthenticationData(None, None)
+        if (value := request.headers.get("X-API-Key")) is not None:
+            auth.api_key = value
+        if (value := request.headers.get("Authorization")) is not None and value.startswith("Bearer "):
+            value = value[7:]
+            try:
+                auth.jwt_payload = jwt.decode(value, algorithms=["HS256"], options={"verify_signature": False})
+            except Exception:  # pylint: disable=broad-except
+                logger = logger_dep.from_request(request)
+                logger.warning("failed to parse Authorization header as jwt", value=value)
+                logger.debug("failed to parse Authorization header as jwt", exc_info=True)
+        if auth.api_key is not None or auth.jwt_payload is not None:
+            request.state.auth_dep = auth
+        else:
+            request.state.auth_dep = None
+    if required and request.state.auth_dep is None:
+        raise NotAuthorizedError()
+    return request.state.auth_dep
+
+
+def from_request_optional(request: Request) -> AuthenticationData | None:
+    return _from_request(request, required=False)
+
+
+def from_request(request: Request) -> AuthenticationData:
+    return _from_request(request, required=True)

{{project_slug}}/dependencies/connection_manager_dep.py.jinja

@@ -18,10 +18,17 @@ def init_dispencer(app: FastAPI, connection_manager: PostgresConnectionManager)
     app.state.postgres_connection_manager_dep = connection_manager
 
 
-def obtain(app_or_request: FastAPI | Request) -> PostgresConnectionManager:
-    """Get a PostgresConnectionManager from request's app state."""
-    if isinstance(app_or_request, Request):
-        app_or_request = app_or_request.app
-    if not hasattr(app_or_request.state, "postgres_connection_manager_dep"):
+def from_app(app: FastAPI) -> PostgresConnectionManager:
+    """Get a connection_manager from app state."""
+    if not hasattr(app.state, "postgres_connection_manager_dep"):
         raise ValueError("PostgresConnectionManager dispencer was not initialized at app preparation")
-    return app_or_request.state.postgres_connection_manager_dep
+    return app.state.postgres_connection_manager_dep
+
+
+async def from_request(request: Request) -> PostgresConnectionManager:
+    """Get a PostgresConnectionManager from request or app state."""
+    if hasattr(request.state, "postgres_connection_manager_dep"):
+        connection_manager = request.state.postgres_connection_manager_dep
+        if isinstance(connection_manager, PostgresConnectionManager):
+            return connection_manager
+    return from_app(request.app)

{{project_slug}}/dependencies/logger_dep.py

@@ -9,7 +9,7 @@ def init_dispencer(app: FastAPI, logger: BoundLogger) -> None:
     if hasattr(app.state, "logger"):
         if not isinstance(app.state.logger_dep, BoundLogger):
             raise ValueError(
-                "logger attribute of app's state is already set" f"with other value ({app.state.logger_dep})"
+                f"logger attribute of app's state is already set with other value ({app.state.logger_dep})"
             )
         return
 
@@ -24,14 +24,17 @@ def attach_to_request(request: Request, logger: BoundLogger) -> None:
     request.state.logger_dep = logger
 
 
-def obtain(app_or_request: FastAPI | Request) -> BoundLogger:
-    """Get a logger from request or app state."""
-    if isinstance(app_or_request, Request):
-        if hasattr(app_or_request.state, "logger_dep"):
-            logger = app_or_request.state.logger_dep
-            if isinstance(logger, BoundLogger):
-                return logger
-        app_or_request = app_or_request.app
-    if not hasattr(app_or_request.state, "logger_dep"):
+def from_app(app: FastAPI) -> BoundLogger:
+    """Get a logger from app state."""
+    if not hasattr(app.state, "logger_dep"):
         raise ValueError("BoundLogger dispencer was not initialized at app preparation")
-    return app_or_request.state.logger_dep
+    return app.state.logger_dep
+
+
+def from_request(request: Request) -> BoundLogger:
+    """Get a logger from request or app state."""
+    if hasattr(request.state, "logger_dep"):
+        logger = request.state.logger_dep
+        if isinstance(logger, BoundLogger):
+            return logger
+    return from_app(request.app)

{{project_slug}}/dependencies/metrics_dep.py.jinja

@@ -5,22 +5,25 @@ from fastapi import FastAPI, Request
 from {{project_slug}}.observability.metrics import Metrics
 
 
-def init_dispencer(app: FastAPI, connection_manager: Metrics) -> None:
+def init_dispencer(app: FastAPI, metrics: Metrics) -> None:
     """Initialize Metrics dispencer at app's state."""
     if hasattr(app.state, "metrics_dep"):
         if not isinstance(app.state.metrics_dep, Metrics):
             raise ValueError(
-                "metrics_dep attribute of app's state is already set" f"with other value ({app.state.metrics_dep})"
+                f"metrics_dep attribute of app's state is already set with other value ({app.state.metrics_dep})"
             )
         return
 
-    app.state.metrics_dep = connection_manager
+    app.state.metrics_dep = metrics
 
 
-def obtain(app_or_request: FastAPI | Request) -> Metrics:
-    """Get a Metrics from request's app state."""
-    if isinstance(app_or_request, Request):
-        app_or_request = app_or_request.app
-    if not hasattr(app_or_request.state, "metrics_dep"):
+def from_app(app: FastAPI) -> Metrics:
+    """Get a Metrics from app state."""
+    if not hasattr(app.state, "metrics_dep"):
         raise ValueError("Metrics dispencer was not initialized at app preparation")
-    return app_or_request.state.metrics_dep
+    return app.state.metrics_dep
+
+
+async def from_request(request: Request) -> Metrics:
+    """Get a Metrics from request's app state."""
+    return from_app(request.app)

{{project_slug}}/exceptions/auth.py.jinja

@@ -0,0 +1,7 @@
+"""Authentication exceptions are located here."""
+
+from {{project_slug}}.exceptions.base import {{ProjectName}}Error
+
+
+class NotAuthorizedError({{ProjectName}}Error):
+    """Exception to raise when user token is not set, but is required."""

{{project_slug}}/exceptions/mapper.py

@@ -2,23 +2,37 @@
 
 from typing import Callable, Type
 
-from fastapi import HTTPException
 from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException
 
 
 class ExceptionMapper:
+    """Maps exceptions to `JSONResponse` for FastAPI error handling."""
+
     def __init__(self):
         self._known_exceptions: dict[Type, Callable[[Exception], JSONResponse]] = {}
 
-    def register_simple(self, exception_type: Type, status_code: int, detail: str) -> None:
+    def register_simple(self, exception_type: Type, status_code: int, details: str) -> None:
+        """Register simple response handler with setting status_code and details."""
         self._known_exceptions[exception_type] = lambda _: JSONResponse(
-            {"code": status_code, "detail": detail}, status_code=status_code
+            {"error": f"{exception_type.__module__}.{exception_type.__qualname__}", "details": details},
+            status_code=status_code,
         )
 
     def register_func(self, exception_type: Type, func: Callable[[Exception], JSONResponse]) -> None:
+        """Register complex response handler by passing function."""
         self._known_exceptions[exception_type] = func
 
+    def get_status_code(self, exc: Exception) -> int:
+        """Get status code of preparing response."""
+        if isinstance(exc, HTTPException):
+            return exc.status_code
+        if type(exc) in self._known_exceptions:
+            return self._known_exceptions[type(exc)](exc).status_code
+        return 500
+
     def is_known(self, exc: Exception) -> bool:
+
         return type(exc) in self._known_exceptions or isinstance(exc, HTTPException)
 
     def apply(self, exc: Exception) -> JSONResponse:

{{project_slug}}/fastapi_init.py.jinja

@@ -2,8 +2,9 @@
 
 import os
 from contextlib import asynccontextmanager
+from typing import NoReturn
 
-from fastapi import FastAPI
+from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.openapi.docs import get_swagger_ui_html
@@ -14,30 +15,17 @@ from {{project_slug}}.db.connection.manager import PostgresConnectionManager
 from {{project_slug}}.dependencies import connection_manager_dep, logger_dep, metrics_dep
 from {{project_slug}}.exceptions.mapper import ExceptionMapper
 from {{project_slug}}.handlers.debug import DebugException, DebugExceptionWithParams
-from {{project_slug}}.middlewares.exception_handler import ExceptionHandlerMiddleware
+from {{project_slug}}.middlewares.exception_handler import ExceptionHandlerMiddleware, HandlerNotFoundError
 from {{project_slug}}.middlewares.observability import ObservabilityMiddleware
-from {{project_slug}}.observability.metrics import init_metrics
+from {{project_slug}}.observability.logging import configure_logging
+from {{project_slug}}.observability.metrics import setup_metrics
 from {{project_slug}}.observability.otel_agent import OpenTelemetryAgent
-from {{project_slug}}.utils.observability import URLsMapper, configure_logging
+from {{project_slug}}.observability.utils import URLsMapper
 
 from .handlers import list_of_routers
 from .version import LAST_UPDATE, VERSION
 
 
-def _get_exception_mapper(debug: bool) -> ExceptionMapper:
-    mapper = ExceptionMapper()
-    if debug:
-        mapper.register_simple(DebugException, 506, "That's how a debug exception look like")
-        mapper.register_func(
-            DebugExceptionWithParams,
-            lambda exc: JSONResponse(
-                {"error": "That's how a debug exception with params look like", "message": exc.message},
-                status_code=exc.status_code,
-            ),
-        )
-    return mapper
-
-
 def bind_routes(application: FastAPI, prefix: str, debug: bool) -> None:
     """Bind all routes to application."""
     for router in list_of_routers:
@@ -84,6 +72,10 @@ def get_app(prefix: str = "/api") -> FastAPI:
             swagger_css_url="https://unpkg.com/swagger-ui-dist@5.11.7/swagger-ui.css",
         )
 
+    @application.exception_handler(404)
+    async def handle_404(request: Request, exc: Exception) -> NoReturn:
+        raise HandlerNotFoundError() from exc
+
     application.add_middleware(
         CORSMiddleware,
         allow_origins=app_config.app.cors.allow_origins,
@@ -99,30 +91,34 @@ def get_app(prefix: str = "/api") -> FastAPI:
         app_config.observability.logging,
         tracing_enabled=app_config.observability.jaeger is not None,
     )
-    metrics = init_metrics()
-    exception_mapper = _get_exception_mapper(app_config.app.debug)
+    metrics = setup_metrics()
+
+    exception_mapper = ExceptionMapper()
+    _register_exceptions(exception_mapper, debug=app_config.app.debug)
     connection_manager = PostgresConnectionManager(
         master=app_config.db.master,
         replicas=app_config.db.replicas,
         logger=logger,
         application_name=f"{{project_slug}}_{VERSION}",
     )
-    urls_mapper = URLsMapper(app_config.observability.prometheus.urls_mapping)
+    urls_mapper = URLsMapper()
+    urls_mapper.add_routes(application.routes)
 
     connection_manager_dep.init_dispencer(application, connection_manager)
     metrics_dep.init_dispencer(application, metrics)
     logger_dep.init_dispencer(application, logger)
 
     application.add_middleware(
-        ObservabilityMiddleware,
+        ExceptionHandlerMiddleware,
+        debug=app_config.app.debug,
         exception_mapper=exception_mapper,
-        metrics=metrics,
         urls_mapper=urls_mapper,
+        errors_metric=metrics.http.errors,
     )
     application.add_middleware(
-        ExceptionHandlerMiddleware,
-        debug=[app_config.app.debug],
-        exception_mapper=exception_mapper,
+        ObservabilityMiddleware,
+        metrics=metrics,
+        urls_mapper=urls_mapper,
     )
 
     return application
@@ -135,13 +131,9 @@ async def lifespan(application: FastAPI):
     Initializes database connection in pass_services_dependencies middleware.
     """
     app_config: {{ProjectName}}Config = application.state.config
-    logger = logger_dep.obtain(application)
+    logger = logger_dep.from_app(application)
 
-    await logger.ainfo("application is being configured", config=app_config.to_order_dict())
-
-    for middleware in application.user_middleware:
-        if middleware.cls == ExceptionHandlerMiddleware:
-            middleware.kwargs["debug"][0] = app_config.app.debug
+    await logger.ainfo("application is starting", config=app_config.to_order_dict())
 
     otel_agent = OpenTelemetryAgent(
         app_config.observability.prometheus,
@@ -153,4 +145,17 @@ async def lifespan(application: FastAPI):
     otel_agent.shutdown()
 
 
+def _register_exceptions(mapper: ExceptionMapper, debug: bool) -> None:
+    if debug:
+        mapper.register_simple(DebugException, 506, "That's how a debug exception look like")
+        mapper.register_func(
+            DebugExceptionWithParams,
+            lambda exc: JSONResponse(
+                {"error": "That's how a debug exception with params look like", "message": exc.message},
+                status_code=exc.status_code,
+            ),
+        )
+    return mapper
+
+
 app = get_app()

{{project_slug}}/handlers/debug.py.jinja

@@ -4,14 +4,14 @@ import asyncio
 from typing import Literal
 
 import aiohttp
-from fastapi import HTTPException, Request
+from fastapi import Depends, HTTPException
 from fastapi.responses import JSONResponse
 from opentelemetry import trace
 from starlette import status
 
 from {{project_slug}}.dependencies import auth_dep
+from {{project_slug}}.observability.utils import get_tracing_headers
 from {{project_slug}}.schemas import PingResponse
-from {{project_slug}}.utils.observability import get_span_headers
 
 from .routers import debug_errors_router
 
@@ -34,9 +34,7 @@ class DebugExceptionWithParams(Exception):
     status_code=status.HTTP_200_OK,
 )
 async def get_status_code(status_code: int, as_exception: bool = False):
-    """
-    Return given status code. If `as_exception` is set to True, return as HTTPException.
-    """
+    """Return given status code. If `as_exception` is set to True, return as HTTPException."""
     if as_exception:
         raise HTTPException(
             status_code=status_code, detail=f"debugging with status code {status_code} as http_exception"
@@ -50,9 +48,7 @@ async def get_status_code(status_code: int, as_exception: bool = False):
     status_code=status.HTTP_200_OK,
 )
 async def get_exception(error_type: Literal["RuntimeError", "DebugException", "DebugExceptionWithParams"]):
-    """
-    Raise exception: `RuntimeError`, `DebugException` or `DebugExceptionWithParams` depending on given parameter.
-    """
+    """Raise exception: `RuntimeError`, `DebugException` or `DebugExceptionWithParams` depending on given parameter."""
     if error_type == "DebugException":
         raise DebugException()
     if error_type == "DebugExceptionWithParams":
@@ -88,7 +84,7 @@ async def inner_get(host: str = "http://127.0.0.1:8080"):
         return asyncio.create_task(
             session.get(
                 "/api/debug/tracing_check",
-                headers=get_span_headers(),
+                headers=get_tracing_headers(),
             )
         )
 
@@ -115,9 +111,7 @@ async def inner_get(host: str = "http://127.0.0.1:8080"):
     "/authentication_info",
     status_code=status.HTTP_200_OK,
 )
-async def authentication_info(request: Request):
+async def authentication_info(auth: auth_dep.AuthenticationData = Depends(auth_dep.from_request)):
     """Check authentication data from `Authorization` and `X-API-Key` headers."""
 
-    auth = auth_dep.obtain(request)
-
     return JSONResponse({"auth_data": {"x-api-key": auth.api_key, "jwt_payload": auth.jwt_payload}})

{{project_slug}}/handlers/system.py.jinja

@@ -3,6 +3,7 @@
 import fastapi
 from starlette import status
 
+from {{project_slug}}.db.connection.manager import PostgresConnectionManager
 from {{project_slug}}.dependencies import connection_manager_dep
 from {{project_slug}}.logic import system as system_logic
 from {{project_slug}}.schemas import PingResponse
@@ -11,9 +12,9 @@ from .routers import system_router
 
 
 @system_router.get("/", status_code=status.HTTP_307_TEMPORARY_REDIRECT, include_in_schema=False)
-@system_router.get("/api/", status_code=status.HTTP_307_TEMPORARY_REDIRECT, include_in_schema=False)
+@system_router.get("/api", status_code=status.HTTP_307_TEMPORARY_REDIRECT, include_in_schema=False)
 async def redirect_to_swagger_docs():
-    """Redirects to **/api/docs** from **/**"""
+    """Redirects to **/api/docs** from **/** and **/api**."""
     return fastapi.responses.RedirectResponse("/api/docs", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
 
 
@@ -34,10 +35,10 @@ async def ping():
     response_model=PingResponse,
     status_code=status.HTTP_200_OK,
 )
-async def ping_db(request: fastapi.Request, readonly: bool = False):
-    """
-    Check that database connection is valid.
-    """
-    connection_manager = connection_manager_dep.obtain(request)
+async def ping_db(
+    readonly: bool = False,
+    connection_manager: PostgresConnectionManager = fastapi.Depends(connection_manager_dep.from_request),
+):
+    """Check that database connection is valid."""
 
     return await system_logic.ping_db(connection_manager, readonly)

{{project_slug}}/middlewares/exception_handler.py.jinja

@@ -3,13 +3,17 @@
 import itertools
 import traceback
 
-from fastapi import FastAPI, HTTPException, Request
+from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
+from opentelemetry import trace
+from opentelemetry.sdk.metrics import Counter
+from opentelemetry.semconv.attributes import exception_attributes, http_attributes, url_attributes
+from starlette.exceptions import HTTPException
 from starlette.middleware.base import BaseHTTPMiddleware
 
+from {{project_slug}}.dependencies import logger_dep
 from {{project_slug}}.exceptions.mapper import ExceptionMapper
-
-from .observability import ObservableException
+from {{project_slug}}.observability.utils import URLsMapper
 
 
 class ExceptionHandlerMiddleware(BaseHTTPMiddleware):  # pylint: disable=too-few-public-methods
@@ -22,31 +26,67 @@ class ExceptionHandlerMiddleware(BaseHTTPMiddleware):  # pylint: disable=too-few
     def __init__(
         self,
         app: FastAPI,
-        debug: list[bool],
+        *,
+        debug: bool,
         exception_mapper: ExceptionMapper,
-    ):
+        urls_mapper: URLsMapper,
+        errors_metric: Counter,
+    ):  # pylint: disable=too-many-arguments
         """Passing debug as a list with single element is a hack to be able to change the value on the fly."""
         super().__init__(app)
         self._debug = debug
-        self._mapper = exception_mapper
+        self._exception_mapper = exception_mapper
+        self._urls_mapper = urls_mapper
+        self._metric = errors_metric
 
     async def dispatch(self, request: Request, call_next):
         try:
             return await call_next(request)
         except Exception as exc:  # pylint: disable=broad-except
-            additional_headers: dict[str, str] | None = None
-            if isinstance(exc, ObservableException):
-                additional_headers = {"X-Trace-Id": exc.trace_id, "X-Span-Id": str(exc.span_id)}
-                exc = exc.__cause__
             status_code = 500
             detail = "exception occured"
 
+            if isinstance(exc, HandlerNotFoundError):
+                exc = exc.__cause__
+
+            cause = exc
             if isinstance(exc, HTTPException):
                 status_code = exc.status_code  # pylint: disable=no-member
                 detail = exc.detail  # pylint: disable=no-member
+                if exc.__cause__ is not None:
+                    cause = exc.__cause__
 
-            if self._debug[0]:
-                if (res := self._mapper.apply_if_known(exc)) is not None:
+            self._metric.add(
+                1,
+                {
+                    http_attributes.HTTP_REQUEST_METHOD: request.method,
+                    url_attributes.URL_PATH: self._urls_mapper.map(request.method, request.url.path),
+                    "error_type": type(cause).__qualname__,
+                    http_attributes.HTTP_RESPONSE_STATUS_CODE: status_code,
+                },
+            )
+
+            span = trace.get_current_span()
+            logger = logger_dep.from_request(request)
+
+            is_known = self._exception_mapper.is_known(cause)
+            span.record_exception(cause, {"is_known": is_known})
+            if is_known:
+                log_func = logger.aerror
+            else:
+                log_func = logger.aexception
+            await log_func("failed to handle request", error_type=type(cause).__name__)
+            span.set_status(trace.StatusCode.ERROR)
+            span.set_attributes(
+                {
+                    exception_attributes.EXCEPTION_TYPE: type(cause).__name__,
+                    exception_attributes.EXCEPTION_MESSAGE: repr(cause),
+                    http_attributes.HTTP_RESPONSE_STATUS_CODE: status_code,
+                }
+            )
+
+            if self._debug:
+                if (res := self._exception_mapper.apply_if_known(exc)) is not None:
                     response = res
                 else:
                     response = JSONResponse(
@@ -56,18 +96,23 @@ class ExceptionHandlerMiddleware(BaseHTTPMiddleware):  # pylint: disable=too-few
                             "error": str(exc),
                             "error_type": type(exc).__name__,
                             "path": request.url.path,
-                            "params": request.url.query,
+                            "query_params": request.url.query,
                             "tracebacks": _get_tracebacks(exc),
                         },
                         status_code=status_code,
                     )
             else:
-                response = self._mapper.apply(exc)
-            if additional_headers is not None:
-                response.headers.update(additional_headers)
+                response = self._exception_mapper.apply(exc)
             return response
 
 
+class HandlerNotFoundError(Exception):
+    """Exception to raise on FastAPI 404 handler (only for situation when no handler was found for request).
+
+    Guranteed to have `.__cause__` as its parent exception.
+    """
+
+
 def _get_tracebacks(exc: Exception) -> list[list[str]]:
     tracebacks: list[list[str]] = []
     while exc is not None:

{{project_slug}}/middlewares/observability.py.jinja

@@ -1,59 +1,43 @@
 """Observability middleware is defined here."""
 
 import time
-from random import randint
+import uuid
 
-import structlog
-from fastapi import FastAPI, HTTPException, Request, Response
+from fastapi import FastAPI, Request
 from opentelemetry import context as tracing_context
 from opentelemetry import trace
-from opentelemetry.semconv.attributes import exception_attributes, http_attributes, url_attributes
-from opentelemetry.trace import NonRecordingSpan, Span, SpanContext, TraceFlags
+from opentelemetry.semconv.attributes import http_attributes, url_attributes
+from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from {{project_slug}}.dependencies import logger_dep
-from {{project_slug}}.exceptions.mapper import ExceptionMapper
 from {{project_slug}}.observability.metrics import Metrics
-from {{project_slug}}.utils.observability import URLsMapper, get_handler_from_path
+from {{project_slug}}.observability.utils import URLsMapper, get_tracing_headers
 
 _tracer = trace.get_tracer_provider().get_tracer(__name__)
 
 
-class ObservableException(RuntimeError):
-    """Runtime Error with `trace_id` and `span_id` set. Guranteed to have `.__cause__` as its parent exception."""
-
-    def __init__(self, trace_id: str, span_id: int):
-        super().__init__()
-        self.trace_id = trace_id
-        self.span_id = span_id
-
-
 class ObservabilityMiddleware(BaseHTTPMiddleware):  # pylint: disable=too-few-public-methods
     """Middleware for global observability requests.
 
-    - Generate tracing span and adds response header 'X-Trace-Id' and X-Span-Id'
+    - Generate tracing span and adds response headers
+    'X-Trace-Id', 'X-Span-Id' (if tracing is configured) and 'X-Request-Id'
     - Binds trace_id it to logger passing it in request state (`request.state.logger`)
     - Collects metrics for Prometheus
 
-    In case when jaeger is not enabled, trace_id and span_id are generated randomly.
     """
 
-    def __init__(self, app: FastAPI, exception_mapper: ExceptionMapper, metrics: Metrics, urls_mapper: URLsMapper):
+    def __init__(self, app: FastAPI, metrics: Metrics, urls_mapper: URLsMapper):
         super().__init__(app)
-        self._exception_mapper = exception_mapper
         self._http_metrics = metrics.http
         self._urls_mapper = urls_mapper
 
     async def dispatch(self, request: Request, call_next):
-        logger = logger_dep.obtain(request)
+        logger = logger_dep.from_request(request)
         _try_get_parent_span_id(request)
-        with _tracer.start_as_current_span("http-request", record_exception=False) as span:
-            trace_id = hex(span.get_span_context().trace_id or randint(1, 1 << 63))[2:]
-            span_id = span.get_span_context().span_id or randint(1, 1 << 31)
-            if trace_id == 0:
-                trace_id = format(randint(1, 1 << 63), "016x")
-                span_id = format(randint(1, 1 << 31), "032x")
-                logger = logger.bind(trace_id=trace_id, span_id=span_id)
+        with _tracer.start_as_current_span("http request") as span:
+            request_id = str(uuid.uuid4())
+            logger = logger.bind(request_id=request_id)
             logger_dep.attach_to_request(request, logger)
 
             span.set_attributes(
@@ -62,106 +46,46 @@ class ObservabilityMiddleware(BaseHTTPMiddleware):  # pylint: disable=too-few-pu
                     url_attributes.URL_PATH: request.url.path,
                     url_attributes.URL_QUERY: str(request.query_params),
                     "request_client": request.client.host,
+                    "request_id": request_id,
                 }
             )
 
             await logger.ainfo(
-                "handling request",
+                "http begin",
                 client=request.client.host,
                 path_params=request.path_params,
                 method=request.method,
                 url=str(request.url),
             )
 
-            path_for_metric = self._urls_mapper.map(request.url.path)
+            path_for_metric = self._urls_mapper.map(request.method, request.url.path)
             self._http_metrics.requests_started.add(1, {"method": request.method, "path": path_for_metric})
+            self._http_metrics.inflight_requests.add(1)
 
             time_begin = time.monotonic()
-            try:
-                result = await call_next(request)
-                duration_seconds = time.monotonic() - time_begin
+            result = await call_next(request)
+            duration_seconds = time.monotonic() - time_begin
 
-                result.headers.update({"X-Trace-Id": trace_id, "X-Span-Id": str(span_id)})
-                await self._handle_success(
-                    request=request,
-                    result=result,
-                    logger=logger,
-                    span=span,
-                    path_for_metric=path_for_metric,
-                    duration_seconds=duration_seconds,
-                )
-                return result
-            except Exception as exc:
-                duration_seconds = time.monotonic() - time_begin
-                await self._handle_exception(
-                    request=request, exc=exc, logger=logger, span=span, duration_seconds=duration_seconds
-                )
-                raise ObservableException(trace_id=trace_id, span_id=span_id) from exc
-            finally:
-                self._http_metrics.request_processing_duration.record(
-                    duration_seconds, {"method": request.method, "path": path_for_metric}
-                )
+            result.headers.update({"X-Request-Id": request_id} | get_tracing_headers())
 
-    async def _handle_success(  # pylint: disable=too-many-arguments
-        self,
-        *,
-        request: Request,
-        result: Response,
-        logger: structlog.stdlib.BoundLogger,
-        span: Span,
-        path_for_metric: str,
-        duration_seconds: float,
-    ) -> None:
-        await logger.ainfo("request handled successfully", time_consumed=round(duration_seconds, 3))
-        self._http_metrics.requests_finished.add(
-            1, {"method": request.method, "path": path_for_metric, "status_code": result.status_code}
-        )
+            await logger.ainfo("http end", time_consumed=round(duration_seconds, 3), status_code=result.status_code)
+            self._http_metrics.requests_finished.add(
+                1,
+                {
+                    http_attributes.HTTP_REQUEST_METHOD: request.method,
+                    url_attributes.URL_PATH: path_for_metric,
+                    http_attributes.HTTP_RESPONSE_STATUS_CODE: result.status_code,
+                },
+            )
+            self._http_metrics.inflight_requests.add(-1)
 
-        span.set_attribute(http_attributes.HTTP_RESPONSE_STATUS_CODE, result.status_code)
-
-    async def _handle_exception(  # pylint: disable=too-many-arguments
-        self,
-        *,
-        request: Request,
-        exc: Exception,
-        logger: structlog.stdlib.BoundLogger,
-        span: Span,
-        duration_seconds: float,
-    ) -> None:
-
-        cause = exc
-        status_code = 500
-        if isinstance(exc, HTTPException):
-            status_code = getattr(exc, "status_code")
-            if exc.__cause__ is not None:
-                cause = exc.__cause__
-
-        self._http_metrics.errors.add(
-            1,
-            {
-                "method": request.method,
-                "path": get_handler_from_path(request.url.path),
-                "error_type": type(cause).__name__,
-                "status_code": status_code,
-            },
-        )
-
-        span.record_exception(exc)
-        if self._exception_mapper.is_known(exc):
-            log_func = logger.aerror
-        else:
-            log_func = logger.aexception
-        await log_func(
-            "failed to handle request", time_consumed=round(duration_seconds, 3), error_type=type(exc).__name__
-        )
-
-        span.set_attributes(
-            {
-                exception_attributes.EXCEPTION_TYPE: type(exc).__name__,
-                exception_attributes.EXCEPTION_MESSAGE: repr(exc),
-                http_attributes.HTTP_RESPONSE_STATUS_CODE: status_code,
-            }
-        )
+            if result.status_code // 100 == 2:
+                span.set_status(trace.StatusCode.OK)
+            span.set_attribute(http_attributes.HTTP_RESPONSE_STATUS_CODE, result.status_code)
+            self._http_metrics.request_processing_duration.record(
+                duration_seconds, {"method": request.method, "path": path_for_metric}
+            )
+            return result
 
 
 def _try_get_parent_span_id(request: Request) -> None:
@@ -171,11 +95,14 @@ def _try_get_parent_span_id(request: Request) -> None:
     if trace_id_str is None or span_id_str is None:
         return
 
-    if not trace_id_str.isnumeric() or not span_id_str.isnumeric():
+    if not trace_id_str.isalnum() or not span_id_str.isalnum():
         return
 
-    span_context = SpanContext(
-        trace_id=int(trace_id_str), span_id=int(span_id_str), is_remote=True, trace_flags=TraceFlags(0x01)
-    )
+    try:
+        span_context = SpanContext(
+            trace_id=int(trace_id_str, 16), span_id=int(span_id_str, 16), is_remote=True, trace_flags=TraceFlags(0x01)
+        )
+    except Exception:  # pylint: disable=broad-except
+        return
     ctx = trace.set_span_in_context(NonRecordingSpan(span_context))
     tracing_context.attach(ctx)

{{project_slug}}/observability/config.py

@@ -0,0 +1,49 @@
+"""Observability config is defined here."""
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+LoggingLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+
+
+@dataclass
+class ExporterConfig:
+    endpoint: str
+    level: LoggingLevel = "INFO"
+    tls_insecure: bool = False
+
+
+@dataclass
+class FileLogger:
+    filename: str
+    level: LoggingLevel
+
+
+@dataclass
+class LoggingConfig:
+    root_logger_level: LoggingLevel = "INFO"
+    stderr_level: LoggingLevel | None = None
+    exporter: ExporterConfig | None = None
+    files: list[FileLogger] = field(default_factory=list)
+
+    def __post_init__(self):
+        if len(self.files) > 0 and isinstance(self.files[0], dict):
+            self.files = [FileLogger(**f) for f in self.files]
+
+
+@dataclass
+class PrometheusConfig:
+    host: str
+    port: int
+
+
+@dataclass
+class JaegerConfig:
+    endpoint: str
+
+
+@dataclass
+class ObservabilityConfig:
+    logging: LoggingConfig
+    prometheus: PrometheusConfig | None = None
+    jaeger: JaegerConfig | None = None

{{project_slug}}/observability/logging.py.jinja

@@ -0,0 +1,165 @@
+"""Observability helper functions are defined here."""
+
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+import structlog
+from opentelemetry import trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+from opentelemetry.sdk._logs import (
+    LoggerProvider,
+    LoggingHandler,
+    LogRecordProcessor,
+    ReadWriteLogRecord,
+)
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.util.types import Attributes
+
+from {{project_slug}}.observability.otel_agent import get_resource
+
+from .config import ExporterConfig, FileLogger, LoggingConfig, LoggingLevel
+
+
+def configure_logging(
+    config: LoggingConfig,
+    tracing_enabled: bool,
+) -> structlog.stdlib.BoundLogger:
+    processors = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.add_logger_name,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+        structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+    ]
+
+    if tracing_enabled:
+        processors.insert(len(processors) - 1, _add_open_telemetry_spans)
+
+    structlog.configure(
+        processors=processors,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(config.root_logger_level)
+
+    if config.stderr_level is not None:
+        _configure_stderr_logger(root_logger, config.stderr_level)
+
+    if len(config.files) > 0:
+        _configure_file_loggers(root_logger, config.files)
+
+    if config.exporter is not None:
+        _configure_otel_exporter(root_logger, config.exporter)
+
+    logger: structlog.stdlib.BoundLogger = structlog.get_logger("{{project_name}}")
+    logger.setLevel(_level_name_mapping[config.root_logger_level])
+
+    return logger
+
+
+_level_name_mapping: dict[LoggingLevel, int] = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+}
+
+
+def _configure_stderr_logger(root_logger: logging.Logger, level: LoggingLevel) -> None:
+    stderr_handler = logging.StreamHandler(sys.stderr)
+    stderr_handler.setFormatter(
+        structlog.stdlib.ProcessorFormatter(processor=structlog.dev.ConsoleRenderer(colors=True))
+    )
+    stderr_handler.setLevel(_level_name_mapping[level])
+    root_logger.addHandler(stderr_handler)
+
+
+def _configure_file_loggers(root_logger: logging.Logger, config_files: list[FileLogger]) -> None:
+    files = {logger_config.filename: logger_config.level for logger_config in config_files}
+    for filename, level in files.items():
+        try:
+            Path(filename).parent.mkdir(parents=True, exist_ok=True)
+        except Exception as exc:  # pylint: disable=broad-except
+            print(f"Cannot create directory for log file {filename}, application will crash most likely. {exc!r}")
+        file_handler = logging.FileHandler(filename=filename, encoding="utf-8")
+        file_handler.setFormatter(structlog.stdlib.ProcessorFormatter(processor=structlog.processors.JSONRenderer()))
+        file_handler.setLevel(_level_name_mapping[level])
+        root_logger.addHandler(file_handler)
+
+
+def _configure_otel_exporter(root_logger: logging.Logger, config: ExporterConfig) -> None:
+    logger_provider = LoggerProvider(resource=get_resource())
+    set_logger_provider(logger_provider)
+
+    otlp_exporter = OTLPLogExporter(endpoint=config.endpoint, insecure=config.tls_insecure)
+    logger_provider.add_log_record_processor(OtelLogPreparationProcessor())
+    logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_exporter))
+
+    exporter_handler = AttrFilteredLoggingHandler(
+        level=config.level,
+        logger_provider=logger_provider,
+    )
+    exporter_handler.setLevel(_level_name_mapping[config.level])
+    root_logger.addHandler(exporter_handler)
+
+
+def _add_open_telemetry_spans(_, __, event_dict: dict):
+    span = trace.get_current_span()
+    if not span or not span.is_recording():
+        return event_dict
+
+    ctx = span.get_span_context()
+
+    event_dict["span_id"] = format(ctx.span_id, "016x")
+    event_dict["trace_id"] = format(ctx.trace_id, "032x")
+
+    return event_dict
+
+
+class AttrFilteredLoggingHandler(LoggingHandler):
+    DROP_ATTRIBUTES = ["_logger"]
+
+    @staticmethod
+    def _get_attributes(record: logging.LogRecord) -> Attributes:
+        attributes = LoggingHandler._get_attributes(record)
+        for attr in AttrFilteredLoggingHandler.DROP_ATTRIBUTES:
+            if attr in attributes:
+                del attributes[attr]
+        return attributes
+
+
+class OtelLogPreparationProcessor(LogRecordProcessor):
+    """Processor which moves everything except message from log record body to attributes."""
+
+    def on_emit(self, log_record: ReadWriteLogRecord) -> None:
+        if not isinstance(log_record.log_record.body, dict):
+            return
+        for key in log_record.log_record.body:
+            if key == "event":
+                continue
+            save_key = key
+            if key in log_record.log_record.attributes:
+                save_key = f"{key}__body"
+            log_record.log_record.attributes[save_key] = self._format_value(log_record.log_record.body[key])
+        log_record.log_record.body = log_record.log_record.body["event"]
+
+    def _format_value(self, value: Any) -> str:
+        if isinstance(value, (dict, list)):
+            return json.dumps(value)
+        return str(value)
+
+    def force_flush(self, timeout_millis=30000):
+        pass
+
+    def shutdown(self):
+        pass

{{project_slug}}/observability/metrics.py

@@ -1,54 +0,0 @@
-"""Application metrics are defined here."""
-
-from dataclasses import dataclass
-
-from opentelemetry import metrics
-from opentelemetry.sdk.metrics import Counter, Histogram
-
-
-@dataclass
-class HTTPMetrics:
-    request_processing_duration: Histogram
-    """Processing time histogram in seconds by `["method", "path"]`."""
-    requests_started: Counter
-    """Total started requests counter by `["method", "path"]`."""
-    requests_finished: Counter
-    """Total finished requests counter by `["method", "path", "status_code"]`."""
-    errors: Counter
-    """Total errors (exceptions) counter by `["method", "path", "error_type", "status_code"]`."""
-
-
-@dataclass
-class Metrics:
-    http: HTTPMetrics
-
-
-def init_metrics() -> Metrics:
-    meter = metrics.get_meter("{{project_name}}")
-    return Metrics(
-        http=HTTPMetrics(
-            request_processing_duration=meter.create_histogram(
-                "request_processing_duration",
-                "sec",
-                "Request processing duration time in seconds",
-                explicit_bucket_boundaries_advisory=[
-                    0.05,
-                    0.2,
-                    0.3,
-                    0.7,
-                    1.0,
-                    1.5,
-                    2.5,
-                    5.0,
-                    10.0,
-                    20.0,
-                    40.0,
-                    60.0,
-                    120.0,
-                ],
-            ),
-            requests_started=meter.create_counter("requests_started_total", "1", "Total number of started requests"),
-            requests_finished=meter.create_counter("request_finished_total", "1", "Total number of finished requests"),
-            errors=meter.create_counter("request_errors_total", "1", "Total number of errors (exceptions) in requests"),
-        )
-    )

{{project_slug}}/observability/metrics.py.jinja

@@ -0,0 +1,113 @@
+"""Application metrics are defined here."""
+
+import threading
+import time
+from dataclasses import dataclass
+from typing import Callable
+
+import psutil
+from opentelemetry import metrics
+from opentelemetry.metrics import CallbackOptions, Observation
+from opentelemetry.sdk.metrics import Counter, Histogram, UpDownCounter
+
+from {{project_slug}}.version import VERSION
+
+
+@dataclass
+class HTTPMetrics:
+    request_processing_duration: Histogram
+    """Processing time histogram in seconds by `["method", "path"]`."""
+    requests_started: Counter
+    """Total started requests counter by `["method", "path"]`."""
+    requests_finished: Counter
+    """Total finished requests counter by `["method", "path", "status_code"]`."""
+    errors: Counter
+    """Total errors (exceptions) counter by `["method", "path", "error_type", "status_code"]`."""
+    inflight_requests: UpDownCounter
+    """Current number of requests handled simultaniously."""
+
+
+@dataclass
+class Metrics:
+    http: HTTPMetrics
+
+
+def setup_metrics() -> Metrics:
+    meter = metrics.get_meter("{{project_name}}")
+
+    _setup_callback_metrics(meter)
+
+    return Metrics(
+        http=HTTPMetrics(
+            request_processing_duration=meter.create_histogram(
+                "request_processing_duration",
+                "sec",
+                "Request processing duration time in seconds",
+                explicit_bucket_boundaries_advisory=[
+                    0.05,
+                    0.2,
+                    0.3,
+                    0.7,
+                    1.0,
+                    1.5,
+                    2.5,
+                    5.0,
+                    10.0,
+                    20.0,
+                    40.0,
+                    60.0,
+                    120.0,
+                ],
+            ),
+            requests_started=meter.create_counter("requests_started_total", "1", "Total number of started requests"),
+            requests_finished=meter.create_counter("request_finished_total", "1", "Total number of finished requests"),
+            errors=meter.create_counter("request_errors_total", "1", "Total number of errors (exceptions) in requests"),
+            inflight_requests=meter.create_up_down_counter(
+                "inflight_requests", "1", "Current number of requests handled simultaniously"
+            ),
+        )
+    )
+
+
+def _setup_callback_metrics(meter: metrics.Meter) -> None:
+    # Create observable gauge
+    meter.create_observable_gauge(
+        name="system_resource_usage",
+        description="System resource utilization",
+        unit="1",
+        callbacks=[_get_system_metrics_callback()],
+    )
+    meter.create_observable_gauge(
+        name="application_metrics",
+        description="Application-specific metrics",
+        unit="1",
+        callbacks=[_get_application_metrics_callback()],
+    )
+
+
+def _get_system_metrics_callback() -> Callable[[CallbackOptions], None]:
+    def system_metrics_callback(options: CallbackOptions):  # pylint: disable=unused-argument
+        """Callback function to collect system metrics"""
+
+        # Process CPU time, a bit more information than `process_cpu_seconds_total`
+        cpu_times = psutil.Process().cpu_times()
+        yield Observation(cpu_times.user, {"resource": "cpu", "mode": "user"})
+        yield Observation(cpu_times.system, {"resource": "cpu", "mode": "system"})
+
+    return system_metrics_callback
+
+
+def _get_application_metrics_callback() -> Callable[[CallbackOptions], None]:
+    startup_time = time.time()
+
+    def application_metrics_callback(options: CallbackOptions):  # pylint: disable=unused-argument
+        """Callback function to collect application-specific metrics"""
+        # Current timestamp
+        yield Observation(startup_time, {"metric": "startup_time", "version": VERSION})
+        yield Observation(time.time(), {"metric": "last_update_time", "version": VERSION})
+
+        # Active threads
+        active_threads = threading.active_count()
+        yield Observation(active_threads, {"metric": "active_threads"})
+
+    return application_metrics_callback

{{project_slug}}/observability/otel_agent.py.jinja

@@ -1,10 +1,13 @@
 """Open Telemetry agent initialization is defined here"""
 
+import platform
+from functools import cache
+
 from opentelemetry import metrics, trace
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.exporter.prometheus import PrometheusMetricReader
 from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_VERSION, Resource
+from opentelemetry.sdk.resources import SERVICE_INSTANCE_ID, SERVICE_NAME, SERVICE_VERSION, Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
@@ -14,14 +17,16 @@ from {{project_slug}}.version import VERSION as APP_VERSION
 from .metrics_server import PrometheusServer
 
 
+@cache
+def get_resource() -> Resource:
+    return Resource.create(
+        attributes={SERVICE_NAME: "{{project_slug}}", SERVICE_VERSION: APP_VERSION, SERVICE_INSTANCE_ID: platform.node()}
+    )
+
+
 class OpenTelemetryAgent:  # pylint: disable=too-few-public-methods
     def __init__(self, prometheus_config: PrometheusConfig | None, jaeger_config: JaegerConfig | None):
-        self._resource = Resource.create(
-            attributes={
-                SERVICE_NAME: "{{project_name}}",
-                SERVICE_VERSION: APP_VERSION,
-            }
-        )
+        self._resource = get_resource()
         self._prometheus: PrometheusServer | None = None
         self._span_exporter: OTLPSpanExporter | None = None
 

{{project_slug}}/observability/utils.py

@@ -0,0 +1,64 @@
+"""Observability-related utility functions and classes are located here."""
+
+import re
+from collections import defaultdict
+
+import fastapi
+import structlog
+from opentelemetry import trace
+
+
+class URLsMapper:
+    """Helper to change URL from given regex pattern to the given static value.
+
+    For example, with map {"GET": {"/api/debug/.*": "/api/debug/*"}} all GET-requests with URL
+    starting with "/api/debug/" will be placed in path "/api/debug/*" in metrics.
+    """
+
+    def __init__(self, urls_map: dict[str, dict[str, str]] | None = None):
+        self._map: dict[str, dict[re.Pattern, str]] = defaultdict(dict)
+        """[method -> [pattern -> mapped_to]]"""
+
+        if urls_map is not None:
+            for method, patterns in urls_map.items():
+                for pattern, value in patterns.items():
+                    self.add(method, pattern, value)
+
+    def add(self, method: str, pattern: str, mapped_to: str) -> None:
+        """Add entry to the map. If pattern compilation is failed, ValueError is raised."""
+        regexp = re.compile(pattern)
+        self._map[method.upper()][regexp] = mapped_to
+
+    def add_routes(self, routes: list[fastapi.routing.APIRoute]) -> None:
+        """Add full route regexes to the map."""
+        logger: structlog.stdlib.BoundLogger = structlog.get_logger(__name__)
+        for route in routes:
+            if not hasattr(route, "path_regex") or not hasattr(route, "path"):
+                logger.warning("route has no 'path_regex' or 'path' attribute", route=route)
+                continue
+            if "{" not in route.path:  # ignore simple routes
+                continue
+            route_path = route.path
+            while "{" in route_path:
+                lbrace = route_path.index("{")
+                rbrace = route_path.index("}", lbrace + 1)
+                route_path = route_path[:lbrace] + "*" + route_path[rbrace + 1 :]
+            for method in route.methods:
+                self._map[method.upper()][route.path_regex] = route_path
+
+    def map(self, method: str, url: str) -> str:
+        """Check every map entry with `re.match` and return matched value. If not found, return original string."""
+        for regexp, mapped_to in self._map[method.upper()].items():
+            if regexp.match(url) is not None:
+                return mapped_to
+        return url
+
+
+def get_tracing_headers() -> dict[str, str]:
+    ctx = trace.get_current_span().get_span_context()
+    if ctx.trace_id == 0:
+        return {}
+    return {
+        "X-Span-Id": format(ctx.span_id, "016x"),
+        "X-Trace-Id": format(ctx.trace_id, "032x"),
+    }

{{project_slug}}/utils/observability.py

@@ -1,193 +0,0 @@
-"""Observability helper functions are defined here."""
-
-import logging
-import platform
-import re
-import sys
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Literal
-
-import structlog
-from opentelemetry import trace
-from opentelemetry._logs import set_logger_provider
-from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
-from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
-from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
-from opentelemetry.sdk.resources import Resource
-from opentelemetry.util.types import Attributes
-
-LoggingLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
-
-
-@dataclass
-class ExporterConfig:
-    endpoint: str
-    level: LoggingLevel = "INFO"
-    tls_insecure: bool = False
-
-
-@dataclass
-class FileLogger:
-    filename: str
-    level: LoggingLevel
-
-
-@dataclass
-class LoggingConfig:
-    level: LoggingLevel
-    exporter: ExporterConfig | None
-    root_logger_level: LoggingLevel = "INFO"
-    files: list[FileLogger] = field(default_factory=list)
-
-    def __post_init__(self):
-        if len(self.files) > 0 and isinstance(self.files[0], dict):
-            self.files = [FileLogger(**f) for f in self.files]
-
-
-def configure_logging(
-    config: LoggingConfig,
-    tracing_enabled: bool,
-) -> structlog.stdlib.BoundLogger:
-    files = {logger_config.filename: logger_config.level for logger_config in config.files}
-
-    level_name_mapping = {
-        "DEBUG": logging.DEBUG,
-        "INFO": logging.INFO,
-        "WARNING": logging.WARNING,
-        "ERROR": logging.ERROR,
-        "CRITICAL": logging.CRITICAL,
-    }
-
-    log_level = level_name_mapping[config.level]
-
-    processors = [
-        structlog.contextvars.merge_contextvars,
-        structlog.stdlib.add_log_level,
-        structlog.stdlib.add_logger_name,
-        structlog.processors.TimeStamper(fmt="iso"),
-        structlog.processors.StackInfoRenderer(),
-        structlog.processors.format_exc_info,
-        structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
-    ]
-
-    if tracing_enabled:
-
-        def add_open_telemetry_spans(_, __, event_dict: dict):
-            span = trace.get_current_span()
-            if not span or not span.is_recording():
-                return event_dict
-
-            ctx = span.get_span_context()
-
-            event_dict["span_id"] = format(ctx.span_id, "016x")
-            event_dict["trace_id"] = format(ctx.trace_id, "032x")
-
-            return event_dict
-
-        processors.insert(len(processors) - 1, add_open_telemetry_spans)
-
-    structlog.configure(
-        processors=processors,
-        logger_factory=structlog.stdlib.LoggerFactory(),
-        wrapper_class=structlog.stdlib.BoundLogger,
-        cache_logger_on_first_use=True,
-    )
-
-    logger: structlog.stdlib.BoundLogger = structlog.get_logger("main")
-    logger.setLevel(log_level)
-
-    console_handler = logging.StreamHandler(sys.stderr)
-    console_handler.setFormatter(
-        structlog.stdlib.ProcessorFormatter(processor=structlog.dev.ConsoleRenderer(colors=True))
-    )
-
-    root_logger = logging.getLogger()
-    root_logger.addHandler(console_handler)
-
-    for filename, level in files.items():
-        try:
-            Path(filename).parent.mkdir(parents=True, exist_ok=True)
-        except Exception as exc:
-            print(f"Cannot create directory for log file {filename}, application will crash most likely. {exc!r}")
-        file_handler = logging.FileHandler(filename=filename, encoding="utf-8")
-        file_handler.setFormatter(structlog.stdlib.ProcessorFormatter(processor=structlog.processors.JSONRenderer()))
-        file_handler.setLevel(level_name_mapping[level])
-        root_logger.addHandler(file_handler)
-
-    root_logger.setLevel(config.root_logger_level)
-
-    if config.exporter is not None:
-        logger_provider = LoggerProvider(
-            resource=Resource.create(
-                {
-                    "service.name": "{{project_name}}",
-                    "service.instance.id": platform.node(),
-                }
-            ),
-        )
-        set_logger_provider(logger_provider)
-
-        otlp_exporter = OTLPLogExporter(endpoint=config.exporter.endpoint, insecure=config.exporter.tls_insecure)
-        logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_exporter))
-
-        exporter_handler = AttrFilteredLoggingHandler(
-            level=config.exporter.level,
-            logger_provider=logger_provider,
-        )
-        # exporter_handler.setFormatter(structlog.stdlib.ProcessorFormatter(processor=structlog.processors.JSONRenderer()))
-        exporter_handler.setLevel(level_name_mapping[config.exporter.level])
-        logger.addHandler(exporter_handler)
-
-    return logger
-
-
-def get_handler_from_path(path: str) -> str:
-    parts = path.split("/")
-    return "/".join(part if not part.rstrip(".0").isdigit() else "*" for part in parts)
-
-
-class URLsMapper:
-    """Helper to change URL from given regex pattern to the given static value.
-
-    For example, with map {"/api/debug/.*": "/api/debug/*"} all requests with URL starting with "/api/debug/"
-    will be placed in path "/api/debug/*" in metrics.
-    """
-
-    def __init__(self, urls_map: dict[str, str]):
-        self._map: dict[re.Pattern, str] = {}
-
-        for pattern, value in urls_map.items():
-            self.add(pattern, value)
-
-    def add(self, pattern: str, mapped_to: str) -> None:
-        """Add entry to the map. If pattern compilation is failed, ValueError is raised."""
-        regexp = re.compile(pattern)
-        self._map[regexp] = mapped_to
-
-    def map(self, url: str) -> str:
-        """Check every map entry with `re.match` and return matched value. If not found, return original string."""
-        for regexp, mapped_to in self._map.items():
-            if regexp.match(url) is not None:
-                return mapped_to
-        return url
-
-
-def get_span_headers() -> dict[str, str]:
-    ctx = trace.get_current_span().get_span_context()
-    return {
-        "X-Span-Id": str(ctx.span_id),
-        "X-Trace-Id": str(ctx.trace_id),
-    }
-
-
-class AttrFilteredLoggingHandler(LoggingHandler):
-    DROP_ATTRIBUTES = ["_logger"]
-
-    @staticmethod
-    def _get_attributes(record: logging.LogRecord) -> Attributes:
-        attributes = LoggingHandler._get_attributes(record)
-        for attr in AttrFilteredLoggingHandler.DROP_ATTRIBUTES:
-            if attr in attributes:
-                del attributes[attr]
-        return attributes