diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 4af1ca31ce..a812091bff 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,6 +10,7 @@ v3.0-32 | July XX, 2026 - Added ``GET /api/v3_0/assets//automations`` and ``GET /api/v3_0/assets//automations/`` for listing and inspecting forecast automations, including the sensors an automation reads from and writes to. Asset job entries now include ``created_via`` provenance; automation identity is included only when the caller may read that automation. - Added ``GET /api/v3_0/sources/`` to show the full record of one data source, including the attributes in which data generators store their configuration. - Automation responses now include ``timezone`` and ``scheduling_cursor``. The cursor is an offset-aware UTC scheduling watermark: occurrences at or before it are ineligible for another automatic queueing attempt; it is not a successful-run timestamp. +- Automation list entries now include recent ``job_stats`` counts, collected in one batched cache pass. If Redis is unavailable, the list remains available with empty counts and a ``redis_connection_err`` message. - API endpoints are now rate-limited. A request which exceeds a limit is answered with a ``429 (Too Many Requests)`` status code and a ``Retry-After`` header stating how many seconds to wait. Responses also carry ``X-RateLimit-*`` headers, describing the limit that applied, how much of it is left, and when it resets. A stricter limit applies to ``POST /assets//schedules/trigger``, ``POST /sensors//schedules/trigger`` and ``POST /sensors//forecasts/trigger`` than to other endpoints; the health endpoints are exempt. Per-account overrides are set by assigning the account a plan (a ``Plan`` database row), rather than through an account attribute. - Introduced the ``inflexible-consumption`` and ``inflexible-production`` flex-context fields, which make explicit how the sign of each inflexible device's power data should be read: positive values denote consumption resp. production. Each entry is a sensor reference (``{"sensor": }``), optionally with source filters (``source-types``, ``exclude-source-types``, ``sources``, ``source-account``). Deprecated the ``inflexible-device-sensors`` field (a list of bare sensor IDs, whose sign convention is read from each sensor's ``consumption_is_positive`` attribute); it remains supported, but cannot be combined with the new fields in one flex-context. - Added a ``role`` query parameter to ``GET /api/v3_0/accounts`` for filtering accessible organisations by account role. diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 5dceb8b6a1..aac1c8a0b1 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -36,6 +36,7 @@ New features * Automations can also compute schedules on a recurring basis (``flexmeasures add automation --type schedules``), with the schedule start defaulting to each run's time [see `PR #2293 `_] * Automations can be created, edited and deleted in the UI and through new API endpoints (``[POST|PATCH|DELETE] /assets/(id)/automations``), by organisation admins and consultants, with their recurrence expressed in a selectable IANA timezone, and only involving sensors they can access themselves (read access to the sensors an automation reads, and permission to record data on the sensors it writes to) [see `PR #2294 `_] * Reports can run as background jobs (``flexmeasures add report --as-job``, processed by workers of the new ``reporting`` queue) and be computed on a recurring basis by automations, with a rolling report window expressed as Pandas offsets or defaulting to the last cron period [see `PR #2297 `_] +* Show automation job counts in one batched asset request, load full automation details on demand, and support editing an automation's name, recurrence, timezone and activation state in the UI [see `PR #2299 `_] * ``flexmeasures show data-sources`` now shows which organisation a data source belongs to, and can list the sensors holding data recorded by a given source [see `PR #2401 `_] * New ``inflexible-consumption`` and ``inflexible-production`` flex-context fields make explicit how the sign of each inflexible device's power data should be read (positive values denote consumption resp. production), accepting sensor references with optional source filters; they replace the now-deprecated ``inflexible-device-sensors`` field (bare sensor IDs, sign read from each sensor's ``consumption_is_positive`` attribute), which remains supported [see `PR #2358 `_] * An inflexible (unschedulable) device can be modelled as its own asset by giving its flex-model entry a single ``inflexible-consumption`` or ``inflexible-production`` sensor reference; such a device joins a ``group`` like any other member, so its fixed (measured) load counts towards the group's intermediate power constraint [see `PR #2374 `_] diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index ba36b0dfc3..6ee287a403 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -13,6 +13,7 @@ from flask_sqlalchemy.pagination import SelectPagination from marshmallow import fields, post_load, ValidationError, Schema, validate +from redis.exceptions import RedisError from webargs.flaskparser import use_kwargs, use_args from sqlalchemy import select, func, or_ @@ -59,6 +60,7 @@ create_automation, delete_automation as remove_automation, describe_cronstr, + get_asset_automations_job_stats, get_automation_job_stats, get_automation_sensors, resolve_automation_sensors, @@ -1396,10 +1398,11 @@ def get_automations(self, id: int, asset: GenericAsset): get: summary: Get all automations defined on an asset. description: | - The response will be a list of automations: recurring forecasting or scheduling tasks + The response will be a list of automations: recurring forecasting, scheduling or reporting tasks defined on the asset. Each entry shows the automation's ID, when it was created, - its type, name, activation status, and its recurrence, both as a cron string - and described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted and its persistent scheduling cursor. + its type, name, activation status, recurrence, IANA timezone, persistent scheduling cursor, + and counts of recently created jobs per job status. Jobs in Redis have a limited TTL, + so not all past jobs are counted. security: - ApiKeyAuth: [] parameters: @@ -1429,6 +1432,9 @@ def get_automations(self, id: int, asset: GenericAsset): scheduling_cursor: "2026-07-11T04:00:00+00:00" recurrence_description: "At 06:00" active: true + job_stats: + finished: 3 + redis_connection_err: null 400: description: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS 401: @@ -1440,14 +1446,33 @@ def get_automations(self, id: int, asset: GenericAsset): tags: - Assets """ + redis_connection_err = None + try: + job_stats = get_asset_automations_job_stats(asset) + except NoRedisConfigured as e: + job_stats = {} + redis_connection_err = e.args[0] + except RedisError: + current_app.logger.warning( + "Could not load automation job statistics because Redis is unavailable.", + exc_info=True, + ) + job_stats = {} + redis_connection_err = ( + "Redis is unavailable; job statistics could not be loaded." + ) automations_data = [] for automation in asset.automations: automation_data = automation_schema.dump(automation) automation_data["recurrence_description"] = describe_cronstr( automation.cronstr ) + automation_data["job_stats"] = job_stats.get(automation.id, {}) automations_data.append(automation_data) - return {"automations": automations_data}, 200 + return { + "automations": automations_data, + "redis_connection_err": redis_connection_err, + }, 200 @route("//automations/", methods=["GET"]) @use_kwargs( @@ -1576,6 +1601,15 @@ def get_automation(self, id: int, automation_id: int, asset: GenericAsset): except NoRedisConfigured as e: automation_data["job_stats"] = {} redis_connection_err = e.args[0] + except RedisError: + current_app.logger.warning( + "Could not load automation job statistics because Redis is unavailable.", + exc_info=True, + ) + automation_data["job_stats"] = {} + redis_connection_err = ( + "Redis is unavailable; job statistics could not be loaded." + ) automation_data["redis_connection_err"] = redis_connection_err return automation_data, 200 diff --git a/flexmeasures/api/v3_0/tests/test_automations_api.py b/flexmeasures/api/v3_0/tests/test_automations_api.py index d5483cbfe5..618dae5076 100644 --- a/flexmeasures/api/v3_0/tests/test_automations_api.py +++ b/flexmeasures/api/v3_0/tests/test_automations_api.py @@ -6,6 +6,7 @@ import pytest from flask import url_for +from redis.exceptions import TimeoutError as RedisTimeoutError from sqlalchemy import select from flexmeasures.data.models.automations import Automation @@ -99,12 +100,48 @@ def test_get_automations( assert day_ahead["recurrence_description"] == "At 06:00" assert day_ahead["active"] is True assert day_ahead["created_at"] is not None + assert day_ahead["job_stats"] == {} # this automation has not queued any jobs # generator and parameters are not listed assert "generator_id" not in day_ahead assert "generator" not in day_ahead assert "parameters" not in day_ahead +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_automations_when_redis_times_out( + app, + add_battery_assets_fresh_db, + add_automations, + requesting_user, + monkeypatch, +): + """The automation list remains available when Redis times out.""" + battery = add_battery_assets_fresh_db["Test battery"] + + def raise_redis_timeout(asset): + raise RedisTimeoutError("Redis timed out at private-host.example") + + monkeypatch.setattr( + "flexmeasures.api.v3_0.assets.get_asset_automations_job_stats", + raise_redis_timeout, + ) + + with app.test_client() as client: + response = client.get(url_for("AssetAPI:get_automations", id=battery.id)) + + assert response.status_code == 200 + assert len(response.json["automations"]) == 2 + assert all( + automation["job_stats"] == {} for automation in response.json["automations"] + ) + assert response.json["redis_connection_err"] == ( + "Redis is unavailable; job statistics could not be loaded." + ) + assert "private-host.example" not in response.text + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 9cd1d2a6e5..9d2aaf65ad 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -31,7 +31,10 @@ get_initial_scheduling_cursor, ) from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.queries.generic_assets import asset_is_in_subtree +from flexmeasures.data.queries.generic_assets import ( + asset_is_in_subtree, + descendants_cte, +) from flexmeasures.utils.time_utils import apply_offset_chain, get_timezone, server_now @@ -762,30 +765,30 @@ def _asset_and_ancestor_ids(asset_id: int | None) -> list[int]: return asset_ids -def get_automation_job_stats(automation: Automation) -> dict[str, int]: - """Count the jobs created by this automation, per job status. +def _asset_subtree_sensor_ids(asset_id: int) -> set[int]: + """Return all sensor IDs on an asset and its descendants.""" + tree = descendants_cte(root_asset_id=asset_id, max_depth=None) + return set( + db.session.scalars( + select(Sensor.id).where(Sensor.generic_asset_id.in_(select(tree.c.id))) + ).all() + ) - Note that jobs in Redis have a limited TTL, so this only counts fairly recent jobs. - """ - # Determine the job cache entries to scan. Forecasting and reporting jobs - # are cached under their target/output sensor(s), which may belong to a - # different asset than the automation's own asset. + +def _job_cache_refs( + automation: Automation, schedule_sensor_ids: set[int] | None = None +) -> set[tuple[int, str, str]]: + """Return the job-cache entries in which an automation's jobs may live.""" parameters = automation.parameters or {} if automation.type == "schedules": # Scheduling jobs are cached under the asset (multi-device wrap-up jobs) # and under individual device sensors (per-device jobs), which may belong # to child assets rather than the automation's own (site) asset. - sensor_ids = _relevant_sensor_ids( - automation, - [ - entry.get("sensor") - for entry in parameters.get("flex-model", []) or [] - if isinstance(entry, dict) - ], - ) - cache_refs = [(automation.asset_id, "scheduling", "asset")] + [ - (sensor_id, "scheduling", "sensor") for sensor_id in sensor_ids - ] + if schedule_sensor_ids is None: + schedule_sensor_ids = _asset_subtree_sensor_ids(automation.asset_id) + return {(automation.asset_id, "scheduling", "asset")} | { + (sensor_id, "scheduling", "sensor") for sensor_id in schedule_sensor_ids + } elif automation.type == "reports": sensor_ids = _relevant_sensor_ids( automation, @@ -795,27 +798,60 @@ def get_automation_job_stats(automation: Automation) -> dict[str, int]: if isinstance(output, dict) ], ) - cache_refs = [(sensor_id, "reporting", "sensor") for sensor_id in sensor_ids] + return {(sensor_id, "reporting", "sensor") for sensor_id in sensor_ids} else: sensor_ids = _relevant_sensor_ids( automation, [parameters.get("sensor"), parameters.get("sensor-to-save")], ) - cache_refs = [(sensor_id, "forecasting", "sensor") for sensor_id in sensor_ids] + return {(sensor_id, "forecasting", "sensor") for sensor_id in sensor_ids} - counts: dict[str, int] = {} + +def _count_automation_jobs( + cache_refs: set[tuple[int, str, str]], automation_ids: set[int] +) -> dict[int, dict[str, int]]: + """Count jobs per automation and status in one pass over the cache entries.""" + counts: dict[int, dict[str, int]] = { + automation_id: {} for automation_id in automation_ids + } seen_job_ids: set[str] = set() for entity_id, queue, asset_or_sensor_type in cache_refs: for job in current_app.job_cache.get(entity_id, queue, asset_or_sensor_type): if job.id in seen_job_ids: continue seen_job_ids.add(job.id) - if job.meta.get("trigger", {}).get("automation_id") == automation.id: + automation_id = job.meta.get("trigger", {}).get("automation_id") + if automation_id in counts: status = str(job.get_status().value) - counts[status] = counts.get(status, 0) + 1 + counts[automation_id][status] = counts[automation_id].get(status, 0) + 1 return counts +def get_automation_job_stats(automation: Automation) -> dict[str, int]: + """Count the recent jobs created by this automation, per job status.""" + return _count_automation_jobs(_job_cache_refs(automation), {automation.id})[ + automation.id + ] + + +def get_asset_automations_job_stats(asset) -> dict[int, dict[str, int]]: + """Count recent jobs for all of an asset's automations in one cache pass.""" + automations = asset.automations + if not automations: + return {} + schedule_sensor_ids = ( + _asset_subtree_sensor_ids(asset.id) + if any(automation.type == "schedules" for automation in automations) + else None + ) + cache_refs: set[tuple[int, str, str]] = set() + for automation in automations: + cache_refs |= _job_cache_refs(automation, schedule_sensor_ids) + return _count_automation_jobs( + cache_refs, {automation.id for automation in automations} + ) + + def _prepare_forecast_automation( asset, parameters: dict, generator_class: str | None, config: dict | None, source ) -> tuple[int, list[str]]: diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index 0fd6602672..3ac4b0de2c 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -272,6 +272,17 @@ def test_schedule_automation_stats_include_descendant_jobs_once( app.job_cache.add(root.id, job.id, "scheduling", "asset") app.job_cache.add(child_sensor.id, job.id, "scheduling", "sensor") + child_job = Job.create( + "flexmeasures.utils.time_utils.server_now", connection=queue.connection + ) + child_job.meta["trigger"] = { + "origin": "automation", + "automation_id": schedule_automation.id, + } + child_job.save_meta() + queue.enqueue_job(child_job) + app.job_cache.add(child_sensor.id, child_job.id, "scheduling", "sensor") + other_job = Job.create( "flexmeasures.utils.time_utils.server_now", connection=queue.connection ) @@ -283,7 +294,7 @@ def test_schedule_automation_stats_include_descendant_jobs_once( queue.enqueue_job(other_job) app.job_cache.add(child_sensor.id, other_job.id, "scheduling", "sensor") - assert get_automation_job_stats(schedule_automation) == {"queued": 1} + assert get_automation_job_stats(schedule_automation) == {"queued": 2} def test_automation_has_valid_timezone_and_aware_cursor(automation_with_generator): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index ea960f54dd..bfa41cf7a2 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -3579,7 +3579,7 @@ "/api/v3_0/assets/{id}/automations": { "get": { "summary": "Get all automations defined on an asset.", - "description": "The response will be a list of automations: recurring forecasting or scheduling tasks\ndefined on the asset. Each entry shows the automation's ID, when it was created,\nits type, name, activation status, and its recurrence, both as a cron string\nand described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted and its persistent scheduling cursor.\n", + "description": "The response will be a list of automations: recurring forecasting, scheduling or reporting tasks\ndefined on the asset. Each entry shows the automation's ID, when it was created,\nits type, name, activation status, recurrence, IANA timezone, persistent scheduling cursor,\nand counts of recently created jobs per job status. Jobs in Redis have a limited TTL,\nso not all past jobs are counted.\n", "security": [ { "ApiKeyAuth": [] @@ -3616,9 +3616,13 @@ "timezone": "Europe/Amsterdam", "scheduling_cursor": "2026-07-11T04:00:00+00:00", "recurrence_description": "At 06:00", - "active": true + "active": true, + "job_stats": { + "finished": 3 + } } - ] + ], + "redis_connection_err": null } } } diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 8365a7a13e..6854838956 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -210,6 +210,12 @@
Edit automation
$("#automations_err").removeClass("d-none").text(message); } + function jobStatsText(jobStats) { + return Object.keys(jobStats || {}).length + ? Object.entries(jobStats).map(([status, count]) => `${status}: ${count}`).join(", ") + : "none"; + } + function AutomationRow(automation) { const createdAt = automation.created_at; return { @@ -229,8 +235,8 @@
Edit automation
`${esc(automation.recurrence_description)}` ), timezone: esc(automation.timezone), - jobs: `…`, - details: `Details + jobs: `${esc(jobStatsText(automation.job_stats))}`, + details: `Details