From 7329a4fb41f41726cf97fc9fcd69d9487c8120d0 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 1 Jun 2026 14:43:02 +0200 Subject: [PATCH 01/13] start --- diracx-core/src/diracx/core/settings.py | 81 ++++++++++++++++++++ diracx-routers/src/diracx/routers/factory.py | 10 +-- docs/admin/reference/env-variables.md | 41 ++++++++++ docs/admin/reference/env-variables.md.j2 | 3 + 4 files changed, 130 insertions(+), 5 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index dd55a88a8..837c78f2c 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -32,6 +32,7 @@ SecretStr, TypeAdapter, UrlConstraints, + create_model, model_validator, ) from pydantic_settings import BaseSettings, SettingsConfigDict @@ -358,3 +359,83 @@ def s3_client(self) -> S3Client: if self._client is None: raise RuntimeError("S3 client accessed before lifetime function") return self._client + + +# def make_factory_settings(model_cls: type[StaticFactorySettings]): +# new_fields = {} +# for var, val in os.environ.items(): +# if var.startswith("XDG_"): +# new_fields[var] = (str, val) +# return create_model("FactorySettings", __base__=model_cls, **new_fields) + + +# FactorySettings = make_factory_settings(StaticFactorySettings) + + +from pydantic import Field + +# Could come from code, YAML, DB, plugin system, etc. +FIELD_SPECS = { + "state_key": { + "type": str, + "default": ..., + "env": "DIRACX_SERVICE_AUTH_STATE_KEY", + "description": "Fernet key used to encrypt/decrypt OAuth2 state.", + }, + "token_issuer": { + "type": str, + "default": ..., + "env": "DIRACX_SERVICE_AUTH_TOKEN_ISSUER", + "description": "JWT issuer value ('iss' claim).", + }, + "access_token_expire_minutes": { + "type": int, + "default": 20, + "env": "DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES", + "description": "Access token lifetime in minutes.", + }, +} + +from .extensions import DiracEntryPoint, select_from_extension + + +def _build_factory_settings_model() -> type[ServiceSettingsBase]: + class _BaseFactorySettings(ServiceSettingsBase): + """Factory settings. + + Settings which do not fit into dedicated classes, + or are dynamically generated + """ + + diracx_legacy_exchange_hashed_api_key: str = "" + enabled_services: dict[str, bool] + + fields: dict[str, tuple[Any, Any]] = {} + + for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): + if "well-known" in entry_point.name: + continue + fields[f"diracx_service_{entry_point.name}_enabled"] = ( + bool, + Field( + default=True, + description=f"Enable the {entry_point.name.upper()} router", + ), + ) + + new_mod = create_model( + "FactorySettings", + __doc__=_BaseFactorySettings.__doc__, + __base__=_BaseFactorySettings, + # __config__=SettingsConfigDict( + # frozen=True, + # extra="ignore", + # use_attribute_docstrings=True, + # ), + **fields, + ) + + return new_mod + + +FactorySettings = _build_factory_settings_model() diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index 342e6d5cd..18e051b94 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -6,7 +6,6 @@ import inspect import logging -import os from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Sequence from functools import partial from http import HTTPStatus @@ -24,14 +23,13 @@ from fastapi.responses import JSONResponse, Response from fastapi.routing import APIRoute from packaging.version import InvalidVersion, parse -from pydantic import TypeAdapter from starlette.middleware.base import BaseHTTPMiddleware from uvicorn.logging import AccessFormatter, DefaultFormatter from diracx.core.config import ConfigSource from diracx.core.exceptions import DiracError, DiracHttpResponseError, NotReadyError from diracx.core.extensions import DiracEntryPoint, select_from_extension -from diracx.core.settings import ServiceSettingsBase +from diracx.core.settings import FactorySettings, ServiceSettingsBase from diracx.core.utils import dotenv_files_from_environment from diracx.db.exceptions import DBUnavailableError from diracx.db.os.utils import BaseOSDB @@ -354,9 +352,11 @@ def create_app() -> DiracFastAPI: # Load all available routers enabled_systems = set() settings_classes = set() + factory_settings = FactorySettings() for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): - env_var = f"DIRACX_SERVICE_{entry_point.name.upper()}_ENABLED" - enabled = TypeAdapter(bool).validate_json(os.environ.get(env_var, "true")) + enabled = getattr( + factory_settings, f"diracx_service_{entry_point.name}_enabled", True + ) logger.debug("Found service %r: enabled=%s", entry_point, enabled) if not enabled: continue diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index ea9a04a49..0d96db09b 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -17,6 +17,47 @@ The variable points to .env files where configuration may be placed. There could Determines whether the jobs service is enabled. +## FactorySettings + +Factory settings. + +``` + Settings which do not fit into dedicated classes, + or are dynamically generated +``` + +### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` + +*Optional*, default value: \`\` + +### `ENABLED_SERVICES` + +**Required** + +### `DIRACX_SERVICE_AUTH_ENABLED` + +*Optional*, default value: `True` + +Enable the AUTH router + +### `DIRACX_SERVICE_CONFIG_ENABLED` + +*Optional*, default value: `True` + +Enable the CONFIG router + +### `DIRACX_SERVICE_HEALTH_ENABLED` + +*Optional*, default value: `True` + +Enable the HEALTH router + +### `DIRACX_SERVICE_JOBS_ENABLED` + +*Optional*, default value: `True` + +Enable the JOBS router + ## AuthSettings Settings for the authentication service. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 29cb4f078..385abca45 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -15,6 +15,9 @@ _X, where X is a number. The files will be loaded in order. ### `DIRACX_SERVICE_JOBS_ENABLED` Determines whether the jobs service is enabled. +{{ render_class('FactorySettings') }} + + {{ render_class('AuthSettings') }} ### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` From f7b489cb3cddff547bc0d042f3a59925ead28349 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 1 Jun 2026 15:57:06 +0200 Subject: [PATCH 02/13] progress --- diracx-core/src/diracx/core/settings.py | 108 +++++++++++++++---- diracx-routers/src/diracx/routers/factory.py | 1 + docs/admin/reference/env-variables.md | 44 ++++++-- docs/admin/reference/env-variables.md.j2 | 7 +- 4 files changed, 134 insertions(+), 26 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 837c78f2c..80efbe316 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -16,7 +16,7 @@ import json from collections.abc import AsyncIterator from pathlib import Path -from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast +from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast, MutableMapping from aiobotocore.session import get_session from botocore.config import Config @@ -400,42 +400,114 @@ def s3_client(self) -> S3Client: def _build_factory_settings_model() -> type[ServiceSettingsBase]: - class _BaseFactorySettings(ServiceSettingsBase): - """Factory settings. - - Settings which do not fit into dedicated classes, - or are dynamically generated - """ - - diracx_legacy_exchange_hashed_api_key: str = "" - enabled_services: dict[str, bool] - fields: dict[str, tuple[Any, Any]] = {} + enabled_services_field: dict[str, tuple[Any, Any]] = {} for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): if "well-known" in entry_point.name: continue - fields[f"diracx_service_{entry_point.name}_enabled"] = ( + enabled_services_field[f"{entry_point.name}"] = ( bool, Field( default=True, + validation_alias=f"DIRACX_SERVICE_{entry_point.name.upper()}_ENABLED", description=f"Enable the {entry_point.name.upper()} router", ), ) + EnabledServices = create_model( + "EnabledServices", + __doc__="Enabled services", + __base__=ServiceSettingsBase, + __config__=SettingsConfigDict( + frozen=True, + use_attribute_docstrings=True, + ), + **enabled_services_field, + ) + + class _BaseFactorySettings(ServiceSettingsBase): + """Factory settings. + + Settings which do not fit into dedicated classes, + or are dynamically generated. + """ + + model_config = SettingsConfigDict(use_attribute_docstrings=True) + + diracx_legacy_exchange_hashed_api_key: str = "" + """Maximum legacy prout. + + Higher values allow more parallel S3 requests (e.g. during bulk sandbox + deletion). + """ + + enabled_services: EnabledServices = Field(default_factory=EnabledServices) + """ + The enabled servicsdsdsds + """ + + fields: dict[str, tuple[Any, Any]] = {} + new_mod = create_model( "FactorySettings", __doc__=_BaseFactorySettings.__doc__, __base__=_BaseFactorySettings, - # __config__=SettingsConfigDict( - # frozen=True, - # extra="ignore", - # use_attribute_docstrings=True, - # ), **fields, ) - + breakpoint() return new_mod FactorySettings = _build_factory_settings_model() + + +# class FactorySettings(ServiceSettingsBase): +# """Factory settings. + +# Settings which do not fit into dedicated classes, +# or are dynamically generated +# """ + +# diracx_legacy_exchange_hashed_api_key: str = "" + +# enabled_services: MutableMapping[str, bool] + + +# @model_validator(mode="before") +# @classmethod +# def check_enabled_services(cls, v): +# for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): + +# if "well-known" in entry_point.name: +# continue +# fields[f"diracx_service_{entry_point.name}_enabled"] = ( +# bool, +# Field( +# default=True, +# description=f"Enable the {entry_point.name.upper()} router", +# ), +# ) + + +class ChildSettings(ServiceSettingsBase): + """The actuall child impl""" + + model_config = SettingsConfigDict(env_prefix="YOYO_", use_attribute_docstrings=True) + + bim: str + """Name of the S3 bucket used for storing job sandboxes. + + This bucket will contain input and output sandbox files for DIRAC jobs. + The bucket must exist or auto_create_bucket must be enabled. + """ + + +class ParentSettings(ServiceSettingsBase): + """It's hard to be a parent""" + + model_config = SettingsConfigDict(env_prefix="TATA_", use_attribute_docstrings=True) + + bucket_name: ChildSettings + """The child. + """ diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index 18e051b94..7d6273618 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -354,6 +354,7 @@ def create_app() -> DiracFastAPI: settings_classes = set() factory_settings = FactorySettings() for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): + factory_settings.enabled_services enabled = getattr( factory_settings, f"diracx_service_{entry_point.name}_enabled", True ) diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index 0d96db09b..de842107b 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -13,7 +13,7 @@ The URL of the configuration backend. The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes \_X, where X is a number. The files will be loaded in order. -### `DIRACX_SERVICE_JOBS_ENABLED` +### `DIRACX_SERVICE_JOBSjinga_ENABLED` Determines whether the jobs service is enabled. @@ -23,36 +23,43 @@ Factory settings. ``` Settings which do not fit into dedicated classes, - or are dynamically generated + or are dynamically generated. ``` ### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` *Optional*, default value: \`\` +Maximum legacy prout. + +Higher values allow more parallel S3 requests (e.g. during bulk sandbox +deletion). + ### `ENABLED_SERVICES` -**Required** +*Optional* -### `DIRACX_SERVICE_AUTH_ENABLED` +The enabled servicsdsdsds + +### `DIRACX_SERVICE_AUTH_ENABLED_PROUT` *Optional*, default value: `True` Enable the AUTH router -### `DIRACX_SERVICE_CONFIG_ENABLED` +### `DIRACX_SERVICE_CONFIG_ENABLED_PROUT` *Optional*, default value: `True` Enable the CONFIG router -### `DIRACX_SERVICE_HEALTH_ENABLED` +### `DIRACX_SERVICE_HEALTH_ENABLED_PROUT` *Optional*, default value: `True` Enable the HEALTH router -### `DIRACX_SERVICE_JOBS_ENABLED` +### `DIRACX_SERVICE_JOBS_ENABLED_PROUT` *Optional*, default value: `True` @@ -302,3 +309,26 @@ Whether to use an insecure gRPC connection for the OpenTelemetry collector. *Optional*, default value: `None` A JSON-encoded dictionary of headers to pass to the OpenTelemetry collector, e.g. {"tenant_id": "lhcbdiracx-cert"}. + +## ParentSettings + +It's hard to be a parent + +### `TATA_BUCKET_NAME` + +**Required** + +The child. + +## ChildSettings + +The actuall child impl + +### `YOYO_BIM` + +**Required** + +Name of the S3 bucket used for storing job sandboxes. + +This bucket will contain input and output sandbox files for DIRAC jobs. +The bucket must exist or auto_create_bucket must be enabled. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 385abca45..c5da98849 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -12,7 +12,7 @@ The URL of the configuration backend. The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes _X, where X is a number. The files will be loaded in order. -### `DIRACX_SERVICE_JOBS_ENABLED` +### `DIRACX_SERVICE_JOBSjinga_ENABLED` Determines whether the jobs service is enabled. {{ render_class('FactorySettings') }} @@ -36,3 +36,8 @@ The URL for the SQL database ``. A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database ``. {{ render_class('OTELSettings') }} + + + +{{ render_class('ParentSettings') }} +{{ render_class('ChildSettings') }} From 3c9f3e1d90d4b49efba734136063b528c1abf438 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 1 Jun 2026 17:15:18 +0200 Subject: [PATCH 03/13] cont --- diracx-core/src/diracx/core/settings.py | 40 +++----------- diracx-routers/src/diracx/routers/factory.py | 7 ++- docs/admin/reference/env-variables.md | 58 +------------------- docs/admin/reference/env-variables.md.j2 | 8 --- 4 files changed, 14 insertions(+), 99 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 80efbe316..6f5a8a0d4 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -16,7 +16,7 @@ import json from collections.abc import AsyncIterator from pathlib import Path -from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast, MutableMapping +from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast from aiobotocore.session import get_session from botocore.config import Config @@ -436,16 +436,14 @@ class _BaseFactorySettings(ServiceSettingsBase): model_config = SettingsConfigDict(use_attribute_docstrings=True) diracx_legacy_exchange_hashed_api_key: str = "" - """Maximum legacy prout. - - Higher values allow more parallel S3 requests (e.g. during bulk sandbox - deletion). + """The hashed API key for the legacy exchange endpoint. """ - enabled_services: EnabledServices = Field(default_factory=EnabledServices) - """ - The enabled servicsdsdsds - """ + enabled_services: EnabledServices = Field( + default_factory=EnabledServices, + # TODO fix the variable doc generation + description=str(EnabledServices.schema_json()), + ) fields: dict[str, tuple[Any, Any]] = {} @@ -455,7 +453,6 @@ class _BaseFactorySettings(ServiceSettingsBase): __base__=_BaseFactorySettings, **fields, ) - breakpoint() return new_mod @@ -488,26 +485,3 @@ class _BaseFactorySettings(ServiceSettingsBase): # description=f"Enable the {entry_point.name.upper()} router", # ), # ) - - -class ChildSettings(ServiceSettingsBase): - """The actuall child impl""" - - model_config = SettingsConfigDict(env_prefix="YOYO_", use_attribute_docstrings=True) - - bim: str - """Name of the S3 bucket used for storing job sandboxes. - - This bucket will contain input and output sandbox files for DIRAC jobs. - The bucket must exist or auto_create_bucket must be enabled. - """ - - -class ParentSettings(ServiceSettingsBase): - """It's hard to be a parent""" - - model_config = SettingsConfigDict(env_prefix="TATA_", use_attribute_docstrings=True) - - bucket_name: ChildSettings - """The child. - """ diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index 7d6273618..d93109721 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -353,11 +353,14 @@ def create_app() -> DiracFastAPI: enabled_systems = set() settings_classes = set() factory_settings = FactorySettings() + print(f"CHRIS {factory_settings}") for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): - factory_settings.enabled_services enabled = getattr( - factory_settings, f"diracx_service_{entry_point.name}_enabled", True + factory_settings.enabled_services, + entry_point.name, + True, ) + print("CHRIS Found service %r: enabled=%s", entry_point, enabled) logger.debug("Found service %r: enabled=%s", entry_point, enabled) if not enabled: continue diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index de842107b..bc486f777 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -30,40 +30,13 @@ Factory settings. *Optional*, default value: \`\` -Maximum legacy prout. - -Higher values allow more parallel S3 requests (e.g. during bulk sandbox -deletion). +The hashed API key for the legacy exchange endpoint. ### `ENABLED_SERVICES` *Optional* -The enabled servicsdsdsds - -### `DIRACX_SERVICE_AUTH_ENABLED_PROUT` - -*Optional*, default value: `True` - -Enable the AUTH router - -### `DIRACX_SERVICE_CONFIG_ENABLED_PROUT` - -*Optional*, default value: `True` - -Enable the CONFIG router - -### `DIRACX_SERVICE_HEALTH_ENABLED_PROUT` - -*Optional*, default value: `True` - -Enable the HEALTH router - -### `DIRACX_SERVICE_JOBS_ENABLED_PROUT` - -*Optional*, default value: `True` - -Enable the JOBS router +{"additionalProperties": false, "description": "Enabled services", "properties": {"DIRACX_SERVICE_AUTH_ENABLED": {"default": true, "description": "Enable the AUTH router", "title": "Diracx Service Auth Enabled", "type": "boolean"}, "DIRACX_SERVICE_CONFIG_ENABLED": {"default": true, "description": "Enable the CONFIG router", "title": "Diracx Service Config Enabled", "type": "boolean"}, "DIRACX_SERVICE_HEALTH_ENABLED": {"default": true, "description": "Enable the HEALTH router", "title": "Diracx Service Health Enabled", "type": "boolean"}, "DIRACX_SERVICE_JOBS_ENABLED": {"default": true, "description": "Enable the JOBS router", "title": "Diracx Service Jobs Enabled", "type": "boolean"}}, "title": "EnabledServices", "type": "object"} ## AuthSettings @@ -186,10 +159,6 @@ Set of security properties available in this DIRAC installation. These properties define various authorization capabilities and are used for access control decisions. Defaults to all available security properties. -### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` - -The hashed API key for the legacy exchange endpoint. - ## SandboxStoreSettings Settings for the sandbox store. @@ -309,26 +278,3 @@ Whether to use an insecure gRPC connection for the OpenTelemetry collector. *Optional*, default value: `None` A JSON-encoded dictionary of headers to pass to the OpenTelemetry collector, e.g. {"tenant_id": "lhcbdiracx-cert"}. - -## ParentSettings - -It's hard to be a parent - -### `TATA_BUCKET_NAME` - -**Required** - -The child. - -## ChildSettings - -The actuall child impl - -### `YOYO_BIM` - -**Required** - -Name of the S3 bucket used for storing job sandboxes. - -This bucket will contain input and output sandbox files for DIRAC jobs. -The bucket must exist or auto_create_bucket must be enabled. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index c5da98849..518fd9db2 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -20,9 +20,6 @@ Determines whether the jobs service is enabled. {{ render_class('AuthSettings') }} -### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` -The hashed API key for the legacy exchange endpoint. - {{ render_class('SandboxStoreSettings') }} ## Databases @@ -36,8 +33,3 @@ The URL for the SQL database ``. A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database ``. {{ render_class('OTELSettings') }} - - - -{{ render_class('ParentSettings') }} -{{ render_class('ChildSettings') }} From 4896a4f6f8d1e8d28764a611133ea369b059f66c Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 1 Jun 2026 17:50:23 +0200 Subject: [PATCH 04/13] fix: tmp --- diracx-core/src/diracx/core/settings.py | 26 +++++++++----------- diracx-routers/src/diracx/routers/factory.py | 2 -- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 6f5a8a0d4..c650fc9a7 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -37,6 +37,7 @@ ) from pydantic_settings import BaseSettings, SettingsConfigDict +from .extensions import DiracEntryPoint, select_from_extension from .properties import SecurityProperty from .s3 import s3_bucket_exists @@ -372,8 +373,6 @@ def s3_client(self) -> S3Client: # FactorySettings = make_factory_settings(StaticFactorySettings) -from pydantic import Field - # Could come from code, YAML, DB, plugin system, etc. FIELD_SPECS = { "state_key": { @@ -396,11 +395,15 @@ def s3_client(self) -> S3Client: }, } -from .extensions import DiracEntryPoint, select_from_extension - def _build_factory_settings_model() -> type[ServiceSettingsBase]: + class _EnabledServicesBase(ServiceSettingsBase): + model_config = SettingsConfigDict( + frozen=True, + use_attribute_docstrings=True, + ) + enabled_services_field: dict[str, tuple[Any, Any]] = {} for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): @@ -415,15 +418,11 @@ def _build_factory_settings_model() -> type[ServiceSettingsBase]: ), ) - EnabledServices = create_model( + EnabledServices = create_model( # noqa: N806 "EnabledServices", __doc__="Enabled services", - __base__=ServiceSettingsBase, - __config__=SettingsConfigDict( - frozen=True, - use_attribute_docstrings=True, - ), - **enabled_services_field, + __base__=_EnabledServicesBase, + **cast(dict[str, Any], enabled_services_field), ) class _BaseFactorySettings(ServiceSettingsBase): @@ -438,11 +437,10 @@ class _BaseFactorySettings(ServiceSettingsBase): diracx_legacy_exchange_hashed_api_key: str = "" """The hashed API key for the legacy exchange endpoint. """ - enabled_services: EnabledServices = Field( default_factory=EnabledServices, # TODO fix the variable doc generation - description=str(EnabledServices.schema_json()), + description=str(cast(BaseSettings, EnabledServices).schema_json()), ) fields: dict[str, tuple[Any, Any]] = {} @@ -451,7 +449,7 @@ class _BaseFactorySettings(ServiceSettingsBase): "FactorySettings", __doc__=_BaseFactorySettings.__doc__, __base__=_BaseFactorySettings, - **fields, + **cast(dict[str, Any], fields), ) return new_mod diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index d93109721..22911bf9c 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -353,14 +353,12 @@ def create_app() -> DiracFastAPI: enabled_systems = set() settings_classes = set() factory_settings = FactorySettings() - print(f"CHRIS {factory_settings}") for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): enabled = getattr( factory_settings.enabled_services, entry_point.name, True, ) - print("CHRIS Found service %r: enabled=%s", entry_point, enabled) logger.debug("Found service %r: enabled=%s", entry_point, enabled) if not enabled: continue From 074862cf363c911aed7b866716662a87b9c0a4d2 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 1 Jun 2026 17:50:59 +0200 Subject: [PATCH 05/13] fix: tmp --- diracx-core/src/diracx/core/settings.py | 69 ++----------------- .../src/diracx/tasks/plumbing/factory.py | 4 +- diracx-tasks/src/diracx/tasks/task_run.py | 4 +- docs/admin/reference/env-variables.md | 10 +-- docs/admin/reference/env-variables.md.j2 | 2 - 5 files changed, 18 insertions(+), 71 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index c650fc9a7..e4daf61fe 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -44,6 +44,8 @@ if TYPE_CHECKING: from types_aiobotocore_s3.client import S3Client + from .config.sources import ConfigSourceUrl + T = TypeVar("T") @@ -362,40 +364,6 @@ def s3_client(self) -> S3Client: return self._client -# def make_factory_settings(model_cls: type[StaticFactorySettings]): -# new_fields = {} -# for var, val in os.environ.items(): -# if var.startswith("XDG_"): -# new_fields[var] = (str, val) -# return create_model("FactorySettings", __base__=model_cls, **new_fields) - - -# FactorySettings = make_factory_settings(StaticFactorySettings) - - -# Could come from code, YAML, DB, plugin system, etc. -FIELD_SPECS = { - "state_key": { - "type": str, - "default": ..., - "env": "DIRACX_SERVICE_AUTH_STATE_KEY", - "description": "Fernet key used to encrypt/decrypt OAuth2 state.", - }, - "token_issuer": { - "type": str, - "default": ..., - "env": "DIRACX_SERVICE_AUTH_TOKEN_ISSUER", - "description": "JWT issuer value ('iss' claim).", - }, - "access_token_expire_minutes": { - "type": int, - "default": 20, - "env": "DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES", - "description": "Access token lifetime in minutes.", - }, -} - - def _build_factory_settings_model() -> type[ServiceSettingsBase]: class _EnabledServicesBase(ServiceSettingsBase): @@ -434,9 +402,14 @@ class _BaseFactorySettings(ServiceSettingsBase): model_config = SettingsConfigDict(use_attribute_docstrings=True) + diracx_config_backend_url: ConfigSourceUrl + """The URL of the configuration backend. + """ + diracx_legacy_exchange_hashed_api_key: str = "" """The hashed API key for the legacy exchange endpoint. """ + enabled_services: EnabledServices = Field( default_factory=EnabledServices, # TODO fix the variable doc generation @@ -455,31 +428,3 @@ class _BaseFactorySettings(ServiceSettingsBase): FactorySettings = _build_factory_settings_model() - - -# class FactorySettings(ServiceSettingsBase): -# """Factory settings. - -# Settings which do not fit into dedicated classes, -# or are dynamically generated -# """ - -# diracx_legacy_exchange_hashed_api_key: str = "" - -# enabled_services: MutableMapping[str, bool] - - -# @model_validator(mode="before") -# @classmethod -# def check_enabled_services(cls, v): -# for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): - -# if "well-known" in entry_point.name: -# continue -# fields[f"diracx_service_{entry_point.name}_enabled"] = ( -# bool, -# Field( -# default=True, -# description=f"Enable the {entry_point.name.upper()} router", -# ), -# ) diff --git a/diracx-tasks/src/diracx/tasks/plumbing/factory.py b/diracx-tasks/src/diracx/tasks/plumbing/factory.py index 17187a056..c27982094 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/factory.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/factory.py @@ -3,7 +3,6 @@ import asyncio import inspect import logging -import os from collections.abc import AsyncIterator, Iterable from contextlib import AsyncExitStack, asynccontextmanager from functools import partial @@ -15,6 +14,7 @@ from fastapi.dependencies.utils import get_dependant from diracx.core.extensions import select_from_extension +from diracx.core.settings import FactorySettings from ._redis_types import LockCoordinator from .base_task import BaseTask @@ -313,7 +313,7 @@ async def setup_dependency_overrides( overrides[os_db_class.session] = partial(_db_context, os_db) # --- Config --- - config_url = os.environ.get("DIRACX_CONFIG_BACKEND_URL") + config_url = FactorySettings().diracx_config_backend_url if config_url: from diracx.core.config import ConfigSource diff --git a/diracx-tasks/src/diracx/tasks/task_run.py b/diracx-tasks/src/diracx/tasks/task_run.py index a9971cfe9..48ab37287 100644 --- a/diracx-tasks/src/diracx/tasks/task_run.py +++ b/diracx-tasks/src/diracx/tasks/task_run.py @@ -22,6 +22,8 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Iterable +from diracx.core.settings import FactorySettings + if TYPE_CHECKING: from .plumbing._redis_types import LockCoordinator @@ -240,7 +242,7 @@ async def start_scheduler(redis_url: str) -> None: task_classes = load_task_registry() config = None - config_url = os.environ.get("DIRACX_CONFIG_BACKEND_URL") + config_url = FactorySettings().diracx_config_backend_url if config_url: from diracx.core.config import ConfigSource diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index bc486f777..af67b28cb 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -4,10 +4,6 @@ ## Core -### `DIRACX_CONFIG_BACKEND_URL` - -The URL of the configuration backend. - ### `DIRACX_SERVICE_DOTENV` The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes @@ -26,6 +22,12 @@ Factory settings. or are dynamically generated. ``` +### `DIRACX_CONFIG_BACKEND_URL` + +**Required** + +The URL of the configuration backend. + ### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` *Optional*, default value: \`\` diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 518fd9db2..957ebeeb8 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -5,8 +5,6 @@ *This page is auto-generated from the settings classes in `diracx.core.settings`.* ## Core -### `DIRACX_CONFIG_BACKEND_URL` -The URL of the configuration backend. ### `DIRACX_SERVICE_DOTENV` The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes From 18d7e6693f154ddc30b8cf51721b5e81c7bb485b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 3 Jun 2026 11:59:03 +0200 Subject: [PATCH 06/13] fix: DOTENV --- diracx-core/src/diracx/core/config/sources.py | 6 ++++-- diracx-core/src/diracx/core/settings.py | 13 ++++++++++++- diracx-routers/src/diracx/routers/factory.py | 7 ------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/diracx-core/src/diracx/core/config/sources.py b/diracx-core/src/diracx/core/config/sources.py index f16fffa82..d0f80dd30 100644 --- a/diracx-core/src/diracx/core/config/sources.py +++ b/diracx-core/src/diracx/core/config/sources.py @@ -7,7 +7,6 @@ import asyncio import logging -import os from abc import ABCMeta, abstractmethod from datetime import datetime, timezone from pathlib import Path @@ -162,7 +161,10 @@ def __init_subclass__(cls) -> None: @classmethod def create(cls): - return cls.create_from_url(backend_url=os.environ["DIRACX_CONFIG_BACKEND_URL"]) + # Avoid circular import + from diracx.core.settings import FactorySettings + + return cls.create_from_url(backend_url=FactorySettings().diracx_config) @classmethod def create_from_url( diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index e4daf61fe..43c0f19f7 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast +import dotenv from aiobotocore.session import get_session from botocore.config import Config from botocore.errorfactory import ClientError @@ -37,14 +38,15 @@ ) from pydantic_settings import BaseSettings, SettingsConfigDict +from .config.sources import ConfigSourceUrl from .extensions import DiracEntryPoint, select_from_extension from .properties import SecurityProperty from .s3 import s3_bucket_exists +from .utils import dotenv_files_from_environment if TYPE_CHECKING: from types_aiobotocore_s3.client import S3Client - from .config.sources import ConfigSourceUrl T = TypeVar("T") @@ -410,6 +412,15 @@ class _BaseFactorySettings(ServiceSettingsBase): """The hashed API key for the legacy exchange endpoint. """ + @model_validator(mode="before") + @classmethod + def load_dotenv_files(cls, data: Any) -> Any: + """Load dotenv files before reading settings from environment.""" + for env_file in dotenv_files_from_environment("DIRACX_SERVICE_DOTENV"): + if not dotenv.load_dotenv(env_file): + raise NotImplementedError(f"Could not load dotenv file {env_file}") + return data + enabled_services: EnabledServices = Field( default_factory=EnabledServices, # TODO fix the variable doc generation diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index 22911bf9c..4761b02c4 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -13,7 +13,6 @@ from logging import Formatter, StreamHandler from typing import Any, TypeVar, cast -import dotenv from cachetools import TTLCache from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status from fastapi.dependencies.models import Dependant @@ -30,7 +29,6 @@ from diracx.core.exceptions import DiracError, DiracHttpResponseError, NotReadyError from diracx.core.extensions import DiracEntryPoint, select_from_extension from diracx.core.settings import FactorySettings, ServiceSettingsBase -from diracx.core.utils import dotenv_files_from_environment from diracx.db.exceptions import DBUnavailableError from diracx.db.os.utils import BaseOSDB from diracx.db.sql.utils import BaseSQLDB @@ -344,11 +342,6 @@ def create_app() -> DiracFastAPI: We attempt to load each setting classes to make sure that the settings are correctly defined. """ - for env_file in dotenv_files_from_environment("DIRACX_SERVICE_DOTENV"): - logger.debug("Loading dotenv file: %s", env_file) - if not dotenv.load_dotenv(env_file): - raise NotImplementedError(f"Could not load dotenv file {env_file}") - # Load all available routers enabled_systems = set() settings_classes = set() From 1e812a38f827c4727cd9baef81031254500c005e Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 3 Jun 2026 14:56:35 +0200 Subject: [PATCH 07/13] feat: checkpoint --- diracx-core/src/diracx/core/settings.py | 1 - docs/admin/reference/env-variables.md | 24 ++++++++-- docs/admin/reference/env-variables.md.j2 | 4 +- docs/templates/_render_class.jinja | 61 ++++++++++++++++++++++++ scripts/generate_settings_docs.py | 1 + 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 43c0f19f7..9e7aa7a92 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -423,7 +423,6 @@ def load_dotenv_files(cls, data: Any) -> Any: enabled_services: EnabledServices = Field( default_factory=EnabledServices, - # TODO fix the variable doc generation description=str(cast(BaseSettings, EnabledServices).schema_json()), ) diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index af67b28cb..c243f0498 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -34,11 +34,29 @@ The URL of the configuration backend. The hashed API key for the legacy exchange endpoint. -### `ENABLED_SERVICES` +### `DIRACX_SERVICE_AUTH_ENABLED` -*Optional* +*Optional*, default value: `True` + +Enable the AUTH router + +### `DIRACX_SERVICE_CONFIG_ENABLED` + +*Optional*, default value: `True` + +Enable the CONFIG router + +### `DIRACX_SERVICE_HEALTH_ENABLED` + +*Optional*, default value: `True` + +Enable the HEALTH router + +### `DIRACX_SERVICE_JOBS_ENABLED` + +*Optional*, default value: `True` -{"additionalProperties": false, "description": "Enabled services", "properties": {"DIRACX_SERVICE_AUTH_ENABLED": {"default": true, "description": "Enable the AUTH router", "title": "Diracx Service Auth Enabled", "type": "boolean"}, "DIRACX_SERVICE_CONFIG_ENABLED": {"default": true, "description": "Enable the CONFIG router", "title": "Diracx Service Config Enabled", "type": "boolean"}, "DIRACX_SERVICE_HEALTH_ENABLED": {"default": true, "description": "Enable the HEALTH router", "title": "Diracx Service Health Enabled", "type": "boolean"}, "DIRACX_SERVICE_JOBS_ENABLED": {"default": true, "description": "Enable the JOBS router", "title": "Diracx Service Jobs Enabled", "type": "boolean"}}, "title": "EnabledServices", "type": "object"} +Enable the JOBS router ## AuthSettings diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 957ebeeb8..580c474fe 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -1,4 +1,4 @@ -{% from '_render_class.jinja' import render_class %} +{% from '_render_class.jinja' import render_class,render_factory_settings %} # List of environment variables @@ -13,7 +13,7 @@ _X, where X is a number. The files will be loaded in order. ### `DIRACX_SERVICE_JOBSjinga_ENABLED` Determines whether the jobs service is enabled. -{{ render_class('FactorySettings') }} +{{ render_factory_settings() }} {{ render_class('AuthSettings') }} diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja index 242d6fb4b..0d5968c2f 100644 --- a/docs/templates/_render_class.jinja +++ b/docs/templates/_render_class.jinja @@ -38,3 +38,64 @@ Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(' {% endif %} {% endmacro %} + +{% macro render_factory_settings() %} +{# Find the FactorySettings class #} +{% set cls = namespace(found=none) %} +{% for c, field_list in classes.items() %} + {% if c.__name__ == 'FactorySettings' %} + {% set cls.found = c %} + {% endif %} +{% endfor %} + +{% if cls.found %} +## {{ cls.found.__name__ }} + +{{ cls.found.__doc__ or "*No description available.*" }} + +{# Build the fields list #} +{% set ns = namespace(fields=[]) %} + +{# Process fields from FactorySettings #} +{% for field_name, field_info in cls.found.model_fields.items() %} + {# Special handling for enabled_services to go down an extra level #} + {% if field_name == 'enabled_services' and field_info.annotation %} + {% set nested_model = field_info.annotation %} + {# Check if the annotation is a Pydantic model with model_fields #} + {% if nested_model.model_fields is defined %} + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} + {# For nested fields, prefer validation_alias if available #} + {% if nested_field_info.validation_alias %} + {% set nested_env_name = nested_field_info.validation_alias %} + {% else %} + {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} + {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} + {% endif %} + {% set _ = ns.fields.append((nested_env_name, nested_field_info)) %} + {% endfor %} + {% endif %} + {% else %} + {# For non-enabled_services fields, add them normally #} + {% set env_prefix = cls.found.model_config.get('env_prefix', '') %} + {% set env_name = (env_prefix ~ field_name).upper() %} + + {# Use validation_alias if available #} + {% if field_info.validation_alias %} + {% set env_name = field_info.validation_alias %} + {% endif %} + + {% set _ = ns.fields.append((env_name, field_info)) %} + {% endif %} +{% endfor %} + +{# Set the fields variable and include the built-in settings_doc template #} +{% set fields = ns.fields %} +{% include '_builtin_markdown.jinja' with context %} +{% else %} +{# Class not found - provide a helpful error #} +**Error: FactorySettings class not found in diracx.core.settings** + +Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(', ') }} +{% endif %} + +{% endmacro %} diff --git a/scripts/generate_settings_docs.py b/scripts/generate_settings_docs.py index 27e34558f..91f6995b2 100644 --- a/scripts/generate_settings_docs.py +++ b/scripts/generate_settings_docs.py @@ -194,6 +194,7 @@ def validate_documentation( # Exclude ServiceSettingsBase as it's the base class undocumented.discard("ServiceSettingsBase") + undocumented.discard("FactorySettings") if undocumented: all_documented = False From 92ec2cc8e659cb3912c949539895fa06aac03d54 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 3 Jun 2026 16:20:51 +0200 Subject: [PATCH 08/13] feat: doc --- diracx-core/src/diracx/core/settings.py | 2 +- docs/admin/reference/env-variables.md | 19 +++++---- docs/templates/_render_class.jinja | 51 ++++++++++++++----------- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 9e7aa7a92..79a763d8d 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -423,7 +423,7 @@ def load_dotenv_files(cls, data: Any) -> Any: enabled_services: EnabledServices = Field( default_factory=EnabledServices, - description=str(cast(BaseSettings, EnabledServices).schema_json()), + description="""The following environment variables dictates which routers are enabled.""", ) fields: dict[str, tuple[Any, Any]] = {} diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index c243f0498..f6caee2c6 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -25,37 +25,36 @@ Factory settings. ### `DIRACX_CONFIG_BACKEND_URL` **Required** - The URL of the configuration backend. ### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` *Optional*, default value: \`\` - The hashed API key for the legacy exchange endpoint. -### `DIRACX_SERVICE_AUTH_ENABLED` +### `ENABLED_SERVICES` -*Optional*, default value: `True` +*Optional* +The following environment variables dictates which routers are enabled. + +#### `DIRACX_SERVICE_AUTH_ENABLED` +*Optional*, default value: `True` Enable the AUTH router -### `DIRACX_SERVICE_CONFIG_ENABLED` +#### `DIRACX_SERVICE_CONFIG_ENABLED` *Optional*, default value: `True` - Enable the CONFIG router -### `DIRACX_SERVICE_HEALTH_ENABLED` +#### `DIRACX_SERVICE_HEALTH_ENABLED` *Optional*, default value: `True` - Enable the HEALTH router -### `DIRACX_SERVICE_JOBS_ENABLED` +#### `DIRACX_SERVICE_JOBS_ENABLED` *Optional*, default value: `True` - Enable the JOBS router ## AuthSettings diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja index 0d5968c2f..39c7e6fed 100644 --- a/docs/templates/_render_class.jinja +++ b/docs/templates/_render_class.jinja @@ -53,44 +53,49 @@ Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(' {{ cls.found.__doc__ or "*No description available.*" }} -{# Build the fields list #} -{% set ns = namespace(fields=[]) %} - -{# Process fields from FactorySettings #} +{# Render all fields manually #} {% for field_name, field_info in cls.found.model_fields.items() %} - {# Special handling for enabled_services to go down an extra level #} + {% set env_prefix = cls.found.model_config.get('env_prefix', '') %} + {% set env_name = (env_prefix ~ field_name).upper() %} + {% if field_info.validation_alias %} + {% set env_name = field_info.validation_alias %} + {% endif %} + + {# Render the field heading #} +{% if not loop.first %} + +{% endif %}### `{{ env_name|upper }}` + +*{% if field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if has_default_value(field_info) %}, default value: `{{ fix_str_enum_value(field_info.default) }}`{% endif %} + +{% if field_info.description %} +{{ field_info.description }} +{% endif %} + + {# Special handling for enabled_services - render nested fields one level deeper #} {% if field_name == 'enabled_services' and field_info.annotation %} {% set nested_model = field_info.annotation %} - {# Check if the annotation is a Pydantic model with model_fields #} {% if nested_model.model_fields is defined %} + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} - {# For nested fields, prefer validation_alias if available #} {% if nested_field_info.validation_alias %} {% set nested_env_name = nested_field_info.validation_alias %} {% else %} {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} {% endif %} - {% set _ = ns.fields.append((nested_env_name, nested_field_info)) %} + +#### `{{ nested_env_name|upper }}` + +*{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} + +{% if nested_field_info.description %} +{{ nested_field_info.description }} +{% endif %} {% endfor %} {% endif %} - {% else %} - {# For non-enabled_services fields, add them normally #} - {% set env_prefix = cls.found.model_config.get('env_prefix', '') %} - {% set env_name = (env_prefix ~ field_name).upper() %} - - {# Use validation_alias if available #} - {% if field_info.validation_alias %} - {% set env_name = field_info.validation_alias %} - {% endif %} - - {% set _ = ns.fields.append((env_name, field_info)) %} {% endif %} {% endfor %} - -{# Set the fields variable and include the built-in settings_doc template #} -{% set fields = ns.fields %} -{% include '_builtin_markdown.jinja' with context %} {% else %} {# Class not found - provide a helpful error #} **Error: FactorySettings class not found in diracx.core.settings** From a97b495bc4bef76778486fe424bfc3d94be08528 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 3 Jun 2026 16:21:27 +0200 Subject: [PATCH 09/13] feat: doc --- docs/admin/reference/env-variables.md | 4 ---- docs/admin/reference/env-variables.md.j2 | 3 --- 2 files changed, 7 deletions(-) diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index f6caee2c6..9aec09e31 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -9,10 +9,6 @@ The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes \_X, where X is a number. The files will be loaded in order. -### `DIRACX_SERVICE_JOBSjinga_ENABLED` - -Determines whether the jobs service is enabled. - ## FactorySettings Factory settings. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 580c474fe..1b0134eaa 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -10,9 +10,6 @@ The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes _X, where X is a number. The files will be loaded in order. -### `DIRACX_SERVICE_JOBSjinga_ENABLED` -Determines whether the jobs service is enabled. - {{ render_factory_settings() }} From 3066f4ced380ddadf74d95b49a880fc13e3b6e8e Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 5 Jun 2026 15:33:47 +0200 Subject: [PATCH 10/13] feat: start os db --- diracx-core/src/diracx/core/settings.py | 32 ++++++++++++++++++++++++ docs/admin/reference/env-variables.md | 14 ++++++++--- docs/admin/reference/env-variables.md.j2 | 4 --- docs/templates/_render_class.jinja | 24 ++++++++++++++++++ 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 79a763d8d..987ad9ceb 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -395,6 +395,33 @@ class _EnabledServicesBase(ServiceSettingsBase): **cast(dict[str, Any], enabled_services_field), ) + class _OpenSearchDBSettingsBase(ServiceSettingsBase): + model_config = SettingsConfigDict( + frozen=True, + use_attribute_docstrings=True, + ) + + opensearch_db_field: dict[str, tuple[Any, Any]] = {} + + for entry_point in select_from_extension(group=DiracEntryPoint.OS_DB): + db_name = entry_point.name + opensearch_db_field[f"{db_name}"] = ( + str, + Field( + default="", + validation_alias=f"DIRACX_OS_DB_{db_name.upper()}", + description="A JSON-encoded dictionary of connection keyword arguments" + f" for the OpenSearch database {db_name}.", + ), + ) + + OpenSearchDBSettings = create_model( # noqa: N806 + "OpenSearchDBSettings", + __doc__="OpenSearch database settings", + __base__=_OpenSearchDBSettingsBase, + **cast(dict[str, Any], opensearch_db_field), + ) + class _BaseFactorySettings(ServiceSettingsBase): """Factory settings. @@ -426,6 +453,11 @@ def load_dotenv_files(cls, data: Any) -> Any: description="""The following environment variables dictates which routers are enabled.""", ) + opensearch_dbs: OpenSearchDBSettings = Field( + default_factory=OpenSearchDBSettings, + description="""The following environment variables configure the OpenSearch database connections.""", + ) + fields: dict[str, tuple[Any, Any]] = {} new_mod = create_model( diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index 9aec09e31..b6baeec59 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -53,6 +53,16 @@ Enable the HEALTH router *Optional*, default value: `True` Enable the JOBS router +### `OPENSEARCH_DBS` + +*Optional* +The following environment variables configure the OpenSearch database connections. + +#### `DIRACX_OS_DB_JOBPARAMETERSDB` + +*Optional*, default value: \`\` +A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database JobParametersDB. + ## AuthSettings Settings for the authentication service. @@ -256,10 +266,6 @@ Controls parallelism of database DELETE operations. The URL for the SQL database ``. -### `DIRACX_OS_DB_` - -A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database ``. - ## OTELSettings Settings for the Open Telemetry Configuration. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 1b0134eaa..22914f49b 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -23,8 +23,4 @@ _X, where X is a number. The files will be loaded in order. The URL for the SQL database ``. -### `DIRACX_OS_DB_` - -A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database ``. - {{ render_class('OTELSettings') }} diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja index 39c7e6fed..2c99321c4 100644 --- a/docs/templates/_render_class.jinja +++ b/docs/templates/_render_class.jinja @@ -89,6 +89,30 @@ Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(' *{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} +{% if nested_field_info.description %} +{{ nested_field_info.description }} +{% endif %} + {% endfor %} + {% endif %} + {% endif %} + + {# Special handling for opensearch_dbs - render nested fields one level deeper #} + {% if field_name == 'opensearch_dbs' and field_info.annotation %} + {% set nested_model = field_info.annotation %} + {% if nested_model.model_fields is defined %} + + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} + {% if nested_field_info.validation_alias %} + {% set nested_env_name = nested_field_info.validation_alias %} + {% else %} + {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} + {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} + {% endif %} + +#### `{{ nested_env_name|upper }}` + +*{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} + {% if nested_field_info.description %} {{ nested_field_info.description }} {% endif %} From 6765367535c00290bd2895077abb7dca9bc9ee6b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 5 Jun 2026 16:21:52 +0200 Subject: [PATCH 11/13] feat: cont os_db --- diracx-db/src/diracx/db/os/utils.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/diracx-db/src/diracx/db/os/utils.py b/diracx-db/src/diracx/db/os/utils.py index 1d80b9958..f58509dca 100644 --- a/diracx-db/src/diracx/db/os/utils.py +++ b/diracx-db/src/diracx/db/os/utils.py @@ -3,7 +3,6 @@ import contextlib import json import logging -import os from abc import ABCMeta, abstractmethod from collections.abc import AsyncIterator from contextvars import ContextVar @@ -14,6 +13,7 @@ from diracx.core.exceptions import InvalidQueryError from diracx.core.extensions import DiracEntryPoint, select_from_extension +from diracx.core.settings import FactorySettings from diracx.db.exceptions import DBUnavailableError logger = logging.getLogger(__name__) @@ -38,7 +38,8 @@ class BaseOSDB(metaclass=ABCMeta): This method returns a dictionary of database names to connection parameters. The available databases are determined by the `diracx.dbs.os` entrypoint in the `pyproject.toml` file and the connection parameters are taken from the - environment variables prefixed with `DIRACX_OS_DB_{DB_NAME}`. + `opensearch_dbs` field in FactorySettings, which reads from environment variables + prefixed with `DIRACX_OS_DB_{DB_NAME}`. If extensions to DiracX are being used, there can be multiple implementations of the same database. To list the available implementations use @@ -104,19 +105,26 @@ def available_implementations(cls, db_name: str) -> list[type[BaseOSDB]]: def available_urls(cls) -> dict[str, dict[str, Any]]: """Return a dict of available OpenSearch database urls. - The list of available URLs is determined by environment variables + The list of available URLs is determined by the opensearch_dbs field + in FactorySettings, which reads from environment variables prefixed with ``DIRACX_OS_DB_{DB_NAME}``. """ + factory_settings = FactorySettings() + opensearch_dbs = factory_settings.opensearch_dbs + conn_kwargs: dict[str, dict[str, Any]] = {} for entry_point in select_from_extension(group=DiracEntryPoint.OS_DB): db_name = entry_point.name - var_name = f"DIRACX_OS_DB_{entry_point.name.upper()}" - if var_name in os.environ: - try: - conn_kwargs[db_name] = json.loads(os.environ[var_name]) - except Exception: - logger.error("Error loading connection parameters for %s", db_name) - raise + # Get the field value from the OpenSearchDBSettings model + if field_value := getattr(opensearch_dbs, db_name, None): + if field_value: + try: + conn_kwargs[db_name] = json.loads(field_value) + except Exception: + logger.error( + "Error loading connection parameters for %s", db_name + ) + raise return conn_kwargs @classmethod From 97858be675a64fa55a8d1597411eae62f4371ff5 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 5 Jun 2026 17:25:06 +0200 Subject: [PATCH 12/13] feat: sql variable --- diracx-core/src/diracx/core/settings.py | 31 +++++++++++ diracx-db/src/diracx/db/sql/utils/base.py | 66 +++++++++++++---------- docs/admin/reference/env-variables.md | 51 +++++++++++++++--- docs/admin/reference/env-variables.md.j2 | 6 --- docs/templates/_render_class.jinja | 24 +++++++++ 5 files changed, 137 insertions(+), 41 deletions(-) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 987ad9ceb..22912f41e 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -422,6 +422,32 @@ class _OpenSearchDBSettingsBase(ServiceSettingsBase): **cast(dict[str, Any], opensearch_db_field), ) + class _SqlDBSettingsBase(ServiceSettingsBase): + model_config = SettingsConfigDict( + frozen=True, + use_attribute_docstrings=True, + ) + + sql_db_field: dict[str, tuple[Any, Any]] = {} + + for entry_point in select_from_extension(group=DiracEntryPoint.SQL_DB): + db_name = entry_point.name + sql_db_field[f"{db_name}"] = ( + str, + Field( + default="", + validation_alias=f"DIRACX_DB_URL_{db_name.upper()}", + description=f"The URL for the SQL database {db_name}.", + ), + ) + + SqlDBSettings = create_model( # noqa: N806 + "SqlDBSettings", + __doc__="SQL database settings", + __base__=_SqlDBSettingsBase, + **cast(dict[str, Any], sql_db_field), + ) + class _BaseFactorySettings(ServiceSettingsBase): """Factory settings. @@ -458,6 +484,11 @@ def load_dotenv_files(cls, data: Any) -> Any: description="""The following environment variables configure the OpenSearch database connections.""", ) + sql_dbs: SqlDBSettings = Field( + default_factory=SqlDBSettings, + description="""The following environment variables configure the SQL database connections.""", + ) + fields: dict[str, tuple[Any, Any]] = {} new_mod = create_model( diff --git a/diracx-db/src/diracx/db/sql/utils/base.py b/diracx-db/src/diracx/db/sql/utils/base.py index 4a8ddfd65..d9141d577 100644 --- a/diracx-db/src/diracx/db/sql/utils/base.py +++ b/diracx-db/src/diracx/db/sql/utils/base.py @@ -2,7 +2,6 @@ import contextlib import logging -import os import re from abc import ABCMeta from collections.abc import AsyncIterator @@ -53,8 +52,9 @@ class BaseSQLDB(metaclass=ABCMeta): The available databases are discovered by calling `BaseSQLDB.available_urls`. This method returns a mapping of database names to connection URLs. The available databases are determined by the `diracx.dbs.sql` entrypoint in the - `pyproject.toml` file and the connection URLs are taken from the environment - variables of the form `DIRACX_DB_URL_`. + `pyproject.toml` file and the connection URLs are taken from the + `sql_dbs` field in FactorySettings, which reads from environment variables + of the form `DIRACX_DB_URL_`. If extensions to DiracX are being used, there can be multiple implementations of the same database. To list the available implementations use @@ -125,37 +125,45 @@ def available_implementations(cls, db_name: str) -> list[type["BaseSQLDB"]]: def available_urls(cls) -> dict[str, str]: """Return a dict of available database urls. - The list of available URLs is determined by environment variables + The list of available URLs is determined by the sql_dbs field + in FactorySettings, which reads from environment variables prefixed with ``DIRACX_DB_URL_{DB_NAME}``. """ + from diracx.core.settings import FactorySettings + + factory_settings = FactorySettings() + sql_dbs = factory_settings.sql_dbs + db_urls: dict[str, str] = {} for entry_point in select_from_extension(group=DiracEntryPoint.SQL_DB): db_name = entry_point.name - var_name = f"DIRACX_DB_URL_{entry_point.name.upper()}" - if var_name in os.environ: - try: - db_url = os.environ[var_name] - if db_url == "sqlite+aiosqlite:///:memory:": - db_urls[db_name] = db_url - # pydantic does not allow for underscore in scheme - # so we do a special case - elif "_" in db_url.split(":")[0]: - # Validate the URL with a fake schema, and then store - # the original one - scheme_id = db_url.find(":") - fake_url = ( - db_url[:scheme_id].replace("_", "-") + db_url[scheme_id:] - ) - TypeAdapter(SqlalchemyDsn).validate_python(fake_url) - db_urls[db_name] = db_url - - else: - db_urls[db_name] = str( - TypeAdapter(SqlalchemyDsn).validate_python(db_url) - ) - except Exception: - logger.error("Error loading URL for %s", db_name) - raise + # Get the field value from the SqlDBSettings model + if hasattr(sql_dbs, db_name): + db_url = getattr(sql_dbs, db_name) + if db_url: + try: + if db_url == "sqlite+aiosqlite:///:memory:": + db_urls[db_name] = db_url + # pydantic does not allow for underscore in scheme + # so we do a special case + elif "_" in db_url.split(":")[0]: + # Validate the URL with a fake schema, and then store + # the original one + scheme_id = db_url.find(":") + fake_url = ( + db_url[:scheme_id].replace("_", "-") + + db_url[scheme_id:] + ) + TypeAdapter(SqlalchemyDsn).validate_python(fake_url) + db_urls[db_name] = db_url + + else: + db_urls[db_name] = str( + TypeAdapter(SqlalchemyDsn).validate_python(db_url) + ) + except Exception: + logger.error("Error loading URL for %s", db_name) + raise return db_urls @classmethod diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index b6baeec59..fc94cee7e 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -63,6 +63,51 @@ The following environment variables configure the OpenSearch database connection *Optional*, default value: \`\` A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database JobParametersDB. +### `SQL_DBS` + +*Optional* +The following environment variables configure the SQL database connections. + +#### `DIRACX_DB_URL_TASKDB` + +*Optional*, default value: \`\` +The URL for the SQL database TaskDB. + +#### `DIRACX_DB_URL_AUTHDB` + +*Optional*, default value: \`\` +The URL for the SQL database AuthDB. + +#### `DIRACX_DB_URL_JOBDB` + +*Optional*, default value: \`\` +The URL for the SQL database JobDB. + +#### `DIRACX_DB_URL_JOBLOGGINGDB` + +*Optional*, default value: \`\` +The URL for the SQL database JobLoggingDB. + +#### `DIRACX_DB_URL_PILOTAGENTSDB` + +*Optional*, default value: \`\` +The URL for the SQL database PilotAgentsDB. + +#### `DIRACX_DB_URL_RESOURCESTATUSDB` + +*Optional*, default value: \`\` +The URL for the SQL database ResourceStatusDB. + +#### `DIRACX_DB_URL_SANDBOXMETADATADB` + +*Optional*, default value: \`\` +The URL for the SQL database SandboxMetadataDB. + +#### `DIRACX_DB_URL_TASKQUEUEDB` + +*Optional*, default value: \`\` +The URL for the SQL database TaskQueueDB. + ## AuthSettings Settings for the authentication service. @@ -260,12 +305,6 @@ Maximum number of concurrent DB delete chunks during cleaning. Controls parallelism of database DELETE operations. -## Databases - -### `DIRACX_DB_URL_` - -The URL for the SQL database ``. - ## OTELSettings Settings for the Open Telemetry Configuration. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 22914f49b..8f17665e1 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -17,10 +17,4 @@ _X, where X is a number. The files will be loaded in order. {{ render_class('SandboxStoreSettings') }} -## Databases - -### `DIRACX_DB_URL_` - -The URL for the SQL database ``. - {{ render_class('OTELSettings') }} diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja index 2c99321c4..c7875d12a 100644 --- a/docs/templates/_render_class.jinja +++ b/docs/templates/_render_class.jinja @@ -113,6 +113,30 @@ Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(' *{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} +{% if nested_field_info.description %} +{{ nested_field_info.description }} +{% endif %} + {% endfor %} + {% endif %} + {% endif %} + + {# Special handling for sql_dbs - render nested fields one level deeper #} + {% if field_name == 'sql_dbs' and field_info.annotation %} + {% set nested_model = field_info.annotation %} + {% if nested_model.model_fields is defined %} + + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} + {% if nested_field_info.validation_alias %} + {% set nested_env_name = nested_field_info.validation_alias %} + {% else %} + {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} + {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} + {% endif %} + +#### `{{ nested_env_name|upper }}` + +*{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} + {% if nested_field_info.description %} {{ nested_field_info.description }} {% endif %} From 9b7ff48b966ef84352054b9e684991417ffdf59d Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 5 Jun 2026 17:25:24 +0200 Subject: [PATCH 13/13] fix: ci fix --- diracx-core/src/diracx/core/config/sources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/diracx-core/src/diracx/core/config/sources.py b/diracx-core/src/diracx/core/config/sources.py index d0f80dd30..fbf3bbed6 100644 --- a/diracx-core/src/diracx/core/config/sources.py +++ b/diracx-core/src/diracx/core/config/sources.py @@ -164,7 +164,9 @@ def create(cls): # Avoid circular import from diracx.core.settings import FactorySettings - return cls.create_from_url(backend_url=FactorySettings().diracx_config) + return cls.create_from_url( + backend_url=FactorySettings().diracx_config_backend_url + ) @classmethod def create_from_url(