From 4efa6a29171b9520557d2872105c7efaf6478909 Mon Sep 17 00:00:00 2001 From: david Date: Wed, 29 Apr 2026 12:28:56 +0200 Subject: [PATCH 01/18] add MetricSQLite to project --- default.nix | 3 ++- pyproject.toml | 1 + shell.nix | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 26b0725..8a6fb89 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { lib, buildPythonPackage, fetchPypi, setuptools, setuptools-scm, entsoe-apy , fastapi, jinja2, numpy, pandas, pydantic, pymodbus, pyomo, pyyaml, uvicorn -, xsdata, httpx }: +, xsdata, httpx, metricsqlite }: buildPythonPackage { pname = "open-ess"; @@ -11,6 +11,7 @@ buildPythonPackage { nativeBuildInputs = [ setuptools setuptools-scm ]; propagatedBuildInputs = [ + metricsqlite entsoe-apy fastapi jinja2 diff --git a/pyproject.toml b/pyproject.toml index 1b94c86..34e1edf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ version = "0.0.0" authors = [{ name = "David van 't Wout", email = "david@vtwout.com" }] requires-python = ">=3.11" # Because of entsoe-apy dependencies = [ + "metricsqlite", "entsoe-apy", "fastapi", "jinja2", diff --git a/shell.nix b/shell.nix index ac12a0e..50f2d1a 100644 --- a/shell.nix +++ b/shell.nix @@ -3,7 +3,21 @@ # Note: ruff is dynamically linked and the version installed by pip won't work on NixOS. # This can be fixed by adding `programs.nix-ld.enable = true;` to your NixOS config. -let open-ess = pkgs.python3.pkgs.callPackage ./default.nix { }; +let + metricsqlite = pkgs.python3.pkgs.buildPythonPackage { + pname = "metricsqlite"; + version = "0.0.0"; + format = "pyproject"; + src = pkgs.fetchFromGitHub { + owner = "DavidvtWout"; + repo = "MetricSQLite"; + rev = "9758b9f"; + hash = "sha256-k0dsp/Ycaq/tqOEoOalH7d7RYpP+N8n7HI8NvxbOEu0="; + }; + nativeBuildInputs = with pkgs.python3.pkgs; [ setuptools setuptools-scm ]; + propagatedBuildInputs = with pkgs.python3.pkgs; [ pydantic ]; + }; + open-ess = pkgs.python3.pkgs.callPackage ./default.nix { inherit metricsqlite; }; in pkgs.mkShell { packages = with pkgs; [ (python3.withPackages (pp: From 5d2142a06ccbb8c4da80115228e40a9d3c8f23ad Mon Sep 17 00:00:00 2001 From: david Date: Sun, 3 May 2026 22:55:28 +0200 Subject: [PATCH 02/18] timeseries abstraction layer --- default.nix | 5 +- docs/settings.md | 96 +++++++++++ open_ess/config.py | 5 +- open_ess/database/config.py | 2 +- .../database/migrations/002_rename_labels.py | 33 ++++ open_ess/main.py | 8 +- open_ess/timeseries/__init__.py | 45 +++++ open_ess/timeseries/base.py | 92 ++++++++++ open_ess/timeseries/config.py | 13 ++ open_ess/timeseries/metricsqlite/__init__.py | 6 + open_ess/timeseries/metricsqlite/backend.py | 112 ++++++++++++ open_ess/timeseries/metricsqlite/config.py | 14 ++ .../timeseries/victoriametrics/__init__.py | 14 ++ .../timeseries/victoriametrics/backend.py | 163 ++++++++++++++++++ open_ess/timeseries/victoriametrics/client.py | 151 ++++++++++++++++ open_ess/timeseries/victoriametrics/config.py | 16 ++ .../timeseries/victoriametrics/protobuf.py | 118 +++++++++++++ open_ess/victron_modbus/client.py | 146 ++++++---------- open_ess/victron_modbus/service.py | 10 +- pyproject.toml | 13 +- 20 files changed, 965 insertions(+), 97 deletions(-) create mode 100644 docs/settings.md create mode 100644 open_ess/database/migrations/002_rename_labels.py create mode 100644 open_ess/timeseries/__init__.py create mode 100644 open_ess/timeseries/base.py create mode 100644 open_ess/timeseries/config.py create mode 100644 open_ess/timeseries/metricsqlite/__init__.py create mode 100644 open_ess/timeseries/metricsqlite/backend.py create mode 100644 open_ess/timeseries/metricsqlite/config.py create mode 100644 open_ess/timeseries/victoriametrics/__init__.py create mode 100644 open_ess/timeseries/victoriametrics/backend.py create mode 100644 open_ess/timeseries/victoriametrics/client.py create mode 100644 open_ess/timeseries/victoriametrics/config.py create mode 100644 open_ess/timeseries/victoriametrics/protobuf.py diff --git a/default.nix b/default.nix index 8a6fb89..f464bdf 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { lib, buildPythonPackage, fetchPypi, setuptools, setuptools-scm, entsoe-apy , fastapi, jinja2, numpy, pandas, pydantic, pymodbus, pyomo, pyyaml, uvicorn -, xsdata, httpx, metricsqlite }: +, xsdata, httpx, urllib3, python-snappy, metricsqlite }: buildPythonPackage { pname = "open-ess"; @@ -22,6 +22,9 @@ buildPythonPackage { pyomo pyyaml uvicorn + # Victoriametrics client + urllib3 + python-snappy ]; meta = with lib; { diff --git a/docs/settings.md b/docs/settings.md new file mode 100644 index 0000000..ece9120 --- /dev/null +++ b/docs/settings.md @@ -0,0 +1,96 @@ +# Configuration + +OpenESS is configured via a YAML file + +### Example Configuration + +```yaml +database: + path: /var/lib/open-ess/data.db + +prices: + area: NL + entsoe_api_key_file: /var/lib/open-ess/entsoe_api_key + buy_formula: "price" + sell_formula: "price" + +victron_gx: + host: 192.168.1.100 + port: 502 + system_id: 100 + +battery: + control: + type: victron + vebus_id: 228 + capacity_kwh: 10.0 + max_charge_power_kw: 3.0 + max_discharge_power_kw: 3.0 + min_soc: 10 + max_soc: 100 +``` + +### prices + +```yaml +prices: + area: + entsoe_api_key_file: /var/lib/open-ess/entsoe_api_key + buy_formula: "price" + sell_formula: "price" +``` + +The `buy_formula` and `sell_formula` allow you to transform the market price into your actual buy/sell price. Use `price` or `p` as the market price variable (EUR/kWh). + +Allowed operations: `+`, `-`, `*`, `/`, `**`, parentheses + +Examples: +- `"price"` - use market price directly +- `"(price + 0.05) * 1.21"` - add 0.05 EUR/kWh markup and 21% VAT +- `"price * 0.9"` - sell at 90% of market price + +### victron_gx + +Settings for connecting to your Victron GX device via Modbus TCP. + +| Setting | Required | Default | Description | +|---------|----------|---------|-------------| +| `host` | Yes | - | IP address of the GX device | +| `port` | No | `502` | Modbus TCP port | +| `system_id` | Yes | - | Modbus unit ID for system data (usually 100) | +| `grid_id` | No | - | Modbus unit ID for grid meter | +| `pvinverter_id` | No | - | Modbus unit ID for PV inverter | + +To find the Modbus unit IDs, go to your GX device: **Settings > Services > Modbus TCP > Available services** + +### battery + +Configuration for your battery system. Can be a single battery or a list of batteries (multi-battery support is planned). + +| Setting | Required | Default | Description | +|---------|----------|---------|-------------| +| `capacity_kwh` | Yes | - | Total battery capacity in kWh | +| `max_charge_power_kw` | Yes | - | Maximum charge power in kW (AC side) | +| `max_discharge_power_kw` | Yes | - | Maximum discharge power in kW (AC side) | +| `min_soc` | No | `10` | Minimum state of charge (%) | +| `max_soc` | No | `100` | Maximum state of charge (%) | +| `control` | Yes | - | Control configuration (see below) | + +#### battery.control (Victron) + +| Setting | Required | Default | Description | +|---------|----------|---------|-------------| +| `type` | Yes | - | Must be `victron` | +| `vebus_id` | Yes | - | Modbus unit ID of the MultiPlus/Quattro | +| `battery_id` | No | - | Modbus unit ID of the BMS (if available) | +| `monitor_only` | No | `false` | Only collect metrics, don't control the battery | +| `disable_charger_when_idle` | No | `false` | Disable charger when not charging (saves power) | +| `disable_inverter_when_idle` | No | `false` | Disable inverter when not discharging (saves power) | + +#### battery.control (MQTT) - Planned + +| Setting | Required | Default | Description | +|---------|----------|---------|-------------| +| `type` | Yes | - | Must be `mqtt` | +| `topic` | Yes | - | MQTT topic prefix for this battery | +| `monitor_only` | No | `false` | Only collect metrics, don't control the battery | diff --git a/open_ess/config.py b/open_ess/config.py index 66bdc8f..95a050e 100644 --- a/open_ess/config.py +++ b/open_ess/config.py @@ -7,14 +7,17 @@ from open_ess.database import DatabaseConfig from open_ess.frontend import FrontendConfig from open_ess.pricing import PriceConfig +from open_ess.timeseries import TimeseriesConfig +from open_ess.timeseries.metricsqlite.config import MetricSQLiteConfig # TODO: Validate config. If a battery defines mqtt control, require mqtt config. class Config(BaseModel): - database: DatabaseConfig + database: DatabaseConfig = DatabaseConfig() frontend: FrontendConfig prices: PriceConfig + timeseries: TimeseriesConfig = MetricSQLiteConfig() battery_system: BatterySystemConfig | list[BatterySystemConfig] @property diff --git a/open_ess/database/config.py b/open_ess/database/config.py index 678ac5f..8fe81d2 100644 --- a/open_ess/database/config.py +++ b/open_ess/database/config.py @@ -9,5 +9,5 @@ class DatabaseCompressionConfig(BaseModel): class DatabaseConfig(BaseModel): - path: Path + path: Path = Path("./data.db") compression: DatabaseCompressionConfig = DatabaseCompressionConfig() diff --git a/open_ess/database/migrations/002_rename_labels.py b/open_ess/database/migrations/002_rename_labels.py new file mode 100644 index 0000000..80cab7d --- /dev/null +++ b/open_ess/database/migrations/002_rename_labels.py @@ -0,0 +1,33 @@ +"""Rename labels.""" + +from open_ess.database import DatabaseConnection + + +def upgrade(conn: DatabaseConnection) -> None: + renames = [ + ("victron/vebus/228", "victron/c0619ab2f37c"), + ("victron/vebus/228/power/ac_in/l1", "victron/c0619ab2f37c/228/power/ac_in/l1"), + ("victron/vebus/228/power/ac_out/l1", "victron/c0619ab2f37c/228/power/ac_out/l1"), + ("victron/vebus/228/soc", "victron/c0619ab2f37c/228/soc"), + ("victron/vebus/228/power/battery", "victron/c0619ab2f37c/228/power/battery"), + ("victron/vebus/228/energy/ac_in_to_ac_out", "victron/c0619ab2f37c/228/energy/ac_in_to_ac_out"), + ("victron/vebus/228/energy/ac_in_import", "victron/c0619ab2f37c/228/energy/ac_in_import"), + ("victron/vebus/228/energy/ac_out_to_ac_in", "victron/c0619ab2f37c/228/energy/ac_out_to_ac_in"), + ("victron/vebus/228/energy/ac_in_export", "victron/c0619ab2f37c/228/energy/ac_in_export"), + ("victron/vebus/228/energy/ac_out_export", "victron/c0619ab2f37c/228/energy/ac_out_export"), + ("victron/vebus/228/energy/ac_out_import", "victron/c0619ab2f37c/228/energy/ac_out_import"), + ("victron/vebus/228/voltage/battery", "victron/c0619ab2f37c/228/voltage/battery"), + ("victron/battery/225/power/battery", "victron/c0619ab2f37c/225/power/battery"), + ("victron/battery/225/voltage/battery", "victron/c0619ab2f37c/225/voltage/battery"), + ("victron/battery/225/soc", "victron/c0619ab2f37c/225/soc"), + ("victron/pvinverter/31/power/l1", "victron/c0619ab2f37c/31/power/l1"), + ("victron/pvinverter/31/energy/l1", "victron/c0619ab2f37c/31/energy/l1"), + ] + + for old_label, new_label in renames: + conn.execute( + "UPDATE labels SET label = ? WHERE label = ?", + (new_label, old_label), + ) + + conn.commit() diff --git a/open_ess/main.py b/open_ess/main.py index 353beb6..36e7afc 100644 --- a/open_ess/main.py +++ b/open_ess/main.py @@ -10,6 +10,7 @@ from open_ess.optimizer import OptimizerService from open_ess.pricing import EntsoeService from open_ess.service import ServiceManager +from open_ess.timeseries import TimeseriesBackend, create_backend from open_ess.util import EndpointFilter, parse_args, setup_logging from open_ess.victron_modbus import VictronService @@ -24,6 +25,10 @@ def main() -> None: database = Database(config.database) database.run_migrations() + # Create timeseries backend + timeseries: TimeseriesBackend = create_backend(config.timeseries, db_path=config.database.path) + logger.info(f"Using timeseries backend: {config.timeseries.backend}") + # Create services service_manager = ServiceManager() service_manager.register_service(DatabaseService(database)) @@ -31,7 +36,7 @@ def main() -> None: battery_systems: list[BatterySystem] = [] for battery_config in config.battery_systems: if battery_config.is_victron: - victron_service = VictronService(database, battery_config) + victron_service = VictronService(database, battery_config, timeseries) service_manager.register_service(victron_service) battery_system = VictronBatterySystem(battery_config, victron_service.client) battery_systems.append(battery_system) @@ -48,6 +53,7 @@ def main() -> None: def shutdown(signum: int, frame: object) -> None: logger.info("Shutting down...") service_manager.stop() + timeseries.close() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) diff --git a/open_ess/timeseries/__init__.py b/open_ess/timeseries/__init__.py new file mode 100644 index 0000000..007dab4 --- /dev/null +++ b/open_ess/timeseries/__init__.py @@ -0,0 +1,45 @@ +"""Timeseries backend abstraction.""" + +from pathlib import Path + +from .base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend +from .config import TimeseriesConfig +from .metricsqlite.config import MetricSQLiteConfig +from .victoriametrics.config import VictoriaMetricsConfig + + +def create_backend( + config: TimeseriesConfig, + db_path: Path | None = None, +) -> TimeseriesBackend: + """Create a timeseries backend from config. + + Args: + config: Timeseries configuration (MetricSQLiteConfig or VictoriaMetricsConfig). + db_path: Database path for MetricSQLite backend. Required if using MetricSQLite. + + Returns: + Configured backend instance. + """ + if isinstance(config, VictoriaMetricsConfig): + from .victoriametrics.backend import VictoriaMetricsBackend + + return VictoriaMetricsBackend(config) + elif isinstance(config, MetricSQLiteConfig): + from .metricsqlite.backend import MetricSQLiteBackend + + if db_path is None: + raise ValueError("db_path is required for MetricSQLite backend") + return MetricSQLiteBackend(config, db_path) + else: + raise ValueError(f"Unknown timeseries config type: {type(config)}") + + +__all__ = [ + "QueryResult", + "QueryResultSeries", + "Sample", + "TimeseriesBackend", + "TimeseriesConfig", + "create_backend", +] diff --git a/open_ess/timeseries/base.py b/open_ess/timeseries/base.py new file mode 100644 index 0000000..25ed41f --- /dev/null +++ b/open_ess/timeseries/base.py @@ -0,0 +1,92 @@ +"""Base interface for timeseries backends.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime + + +@dataclass +class Sample: + """A single metric sample for writing.""" + + metric: str + value: float + timestamp: datetime + labels: dict[str, str] = field(default_factory=dict) + + +@dataclass +class QueryResultSeries: + """A single series from a query result.""" + + metric: dict[str, str] + values: list[tuple[datetime, float]] # (timestamp, value) pairs + + +@dataclass +class QueryResult: + """Result of a query or query_range call.""" + + series: list[QueryResultSeries] + + def scalar(self) -> float | None: + """Get single scalar value if result has exactly one series with one value.""" + if len(self.series) == 1 and len(self.series[0].values) == 1: + return self.series[0].values[0][1] + return None + + +class TimeseriesBackend(ABC): + """Abstract base class for timeseries backends. + + Provides a unified interface for writing and querying metrics, + supporting both VictoriaMetrics and MetricSQLite backends. + """ + + @abstractmethod + def write(self, samples: list[Sample]) -> None: + """Write a batch of samples. + + Args: + samples: List of samples to write. + """ + ... + + @abstractmethod + def query(self, query: str, time: datetime | None = None) -> QueryResult: + """Execute an instant query. + + Args: + query: MetricsQL/PromQL query string. + time: Evaluation timestamp. Defaults to now. + + Returns: + Query result containing matching series. + """ + ... + + @abstractmethod + def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "1m", + ) -> QueryResult: + """Execute a range query. + + Args: + query: MetricsQL/PromQL query string. + start: Start of time range. + end: End of time range. + step: Query resolution (e.g., "1m", "5m", "1h"). + + Returns: + Query result containing matching series with values at each step. + """ + ... + + @abstractmethod + def close(self) -> None: + """Close any connections.""" + ... diff --git a/open_ess/timeseries/config.py b/open_ess/timeseries/config.py new file mode 100644 index 0000000..caeb115 --- /dev/null +++ b/open_ess/timeseries/config.py @@ -0,0 +1,13 @@ +"""Timeseries backend configuration.""" + +from typing import Annotated + +from pydantic import Field + +from .metricsqlite.config import MetricSQLiteConfig +from .victoriametrics.config import VictoriaMetricsConfig + +TimeseriesConfig = Annotated[ + MetricSQLiteConfig | VictoriaMetricsConfig, + Field(discriminator="backend"), +] diff --git a/open_ess/timeseries/metricsqlite/__init__.py b/open_ess/timeseries/metricsqlite/__init__.py new file mode 100644 index 0000000..2f5d3e5 --- /dev/null +++ b/open_ess/timeseries/metricsqlite/__init__.py @@ -0,0 +1,6 @@ +"""MetricSQLite timeseries backend.""" + +from .backend import MetricSQLiteBackend +from .config import MetricSQLiteConfig + +__all__ = ["MetricSQLiteBackend", "MetricSQLiteConfig"] diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py new file mode 100644 index 0000000..4a3508f --- /dev/null +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -0,0 +1,112 @@ +"""MetricSQLite timeseries backend implementation.""" + +from datetime import datetime +from pathlib import Path + +from metricsqlite import MetricsQLiteClient +from metricsqlite.engine import InstantVector, MatrixResult, RangeVectorResult, ScalarResult + +from ..base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend +from .config import MetricSQLiteConfig + + +class MetricSQLiteBackend(TimeseriesBackend): + """MetricSQLite backend using SQLite for storage.""" + + def __init__(self, config: MetricSQLiteConfig, db_path: Path): + """Initialize MetricSQLite backend. + + Args: + config: MetricSQLite configuration. + db_path: Path to SQLite database file. + """ + self.config = config + self._client = MetricsQLiteClient(db_path, enable_wal=True) + self._client.connect() + self._client.create_tables() + + def write(self, samples: list[Sample]) -> None: + """Write samples as gauge metrics.""" + for sample in samples: + timestamp_ms = int(sample.timestamp.timestamp() * 1000) + self._client.insert_gauge( + name=sample.metric, + value=sample.value, + timestamp=timestamp_ms, + labels=sample.labels if sample.labels else None, + ) + + def query(self, query: str, time: datetime | None = None) -> QueryResult: + """Execute an instant query.""" + eval_time: float | None = None + if time is not None: + eval_time = time.timestamp() * 1000 + + result = self._client.query(query, time=eval_time) + return self._convert_result(result) + + def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "1m", + ) -> QueryResult: + """Execute a range query.""" + start_ms = start.timestamp() * 1000 + end_ms = end.timestamp() * 1000 + + result = self._client.query_range(query, start=start_ms, end=end_ms, step=step) + return self._convert_matrix_result(result) + + def _convert_result( + self, result: InstantVector | RangeVectorResult | ScalarResult + ) -> QueryResult: + """Convert metricsqlite result to QueryResult.""" + if isinstance(result, ScalarResult): + # Scalar result - single value + return QueryResult( + series=[ + QueryResultSeries( + metric={}, + values=[(datetime.fromtimestamp(result.timestamp / 1000), result.value)], + ) + ] + ) + + if isinstance(result, RangeVectorResult): + # Range vector from instant query (e.g., metric[5m]) + series_list = [] + for labels, samples in result.series: + values = [ + (datetime.fromtimestamp(sample.timestamp / 1000), sample.value) + for sample in samples + ] + series_list.append(QueryResultSeries(metric=labels, values=values)) + return QueryResult(series=series_list) + + # InstantVector + series_list = [] + for labels, sample in result.series: + series_list.append( + QueryResultSeries( + metric=labels, + values=[(datetime.fromtimestamp(sample.timestamp / 1000), sample.value)], + ) + ) + return QueryResult(series=series_list) + + def _convert_matrix_result(self, result: MatrixResult) -> QueryResult: + """Convert metricsqlite MatrixResult to QueryResult.""" + series_list = [] + for labels, samples in result.series: + values = [ + (datetime.fromtimestamp(sample.timestamp / 1000), sample.value) + for sample in samples + ] + series_list.append(QueryResultSeries(metric=labels, values=values)) + return QueryResult(series=series_list) + + def close(self) -> None: + """Close the database connection.""" + self._client.close() diff --git a/open_ess/timeseries/metricsqlite/config.py b/open_ess/timeseries/metricsqlite/config.py new file mode 100644 index 0000000..d274705 --- /dev/null +++ b/open_ess/timeseries/metricsqlite/config.py @@ -0,0 +1,14 @@ +"""MetricSQLite timeseries backend configuration.""" + +from typing import Literal + +from pydantic import BaseModel + + +class MetricSQLiteConfig(BaseModel): + """Configuration for MetricSQLite backend. + + Uses the database path from DatabaseConfig. + """ + + backend: Literal["metricsqlite"] = "metricsqlite" diff --git a/open_ess/timeseries/victoriametrics/__init__.py b/open_ess/timeseries/victoriametrics/__init__.py new file mode 100644 index 0000000..e5dcf37 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/__init__.py @@ -0,0 +1,14 @@ +"""VictoriaMetrics timeseries backend.""" + +from .backend import VictoriaMetricsBackend +from .client import RemoteWriteClient, RemoteWriteError, Sample, timestamp_ms +from .config import VictoriaMetricsConfig + +__all__ = [ + "RemoteWriteClient", + "RemoteWriteError", + "Sample", + "VictoriaMetricsBackend", + "VictoriaMetricsConfig", + "timestamp_ms", +] diff --git a/open_ess/timeseries/victoriametrics/backend.py b/open_ess/timeseries/victoriametrics/backend.py new file mode 100644 index 0000000..bd746f7 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/backend.py @@ -0,0 +1,163 @@ +"""VictoriaMetrics timeseries backend implementation.""" + +import json +import logging +from datetime import datetime + +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.util import parse_url + +from ..base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend +from .client import RemoteWriteClient +from .client import Sample as RemoteWriteSample +from .config import VictoriaMetricsConfig + +logger = logging.getLogger(__name__) + + +class VictoriaMetricsBackend(TimeseriesBackend): + """VictoriaMetrics backend using remote write protocol for writes and HTTP API for queries.""" + + def __init__(self, config: VictoriaMetricsConfig): + self.config = config + + # Parse URL to get components + parsed = parse_url(config.url) + self._host = parsed.host + self._port = parsed.port + self._scheme = parsed.scheme or "http" + + # Build base path (strip trailing slash) + self._base_path = (parsed.path or "").rstrip("/") + + # Set up connection pool for queries + pool_cls = HTTPSConnectionPool if self._scheme == "https" else HTTPConnectionPool + pool_kwargs: dict = { + "host": self._host, + "port": self._port, + "timeout": config.timeout, + "maxsize": 2, + "block": True, + } + if config.username and config.password: + import base64 + + credentials = f"{config.username}:{config.password}".encode() + auth = base64.b64encode(credentials).decode("ascii") + pool_kwargs["headers"] = {"Authorization": f"Basic {auth}"} + self._pool = pool_cls(**pool_kwargs) + + # Set up remote write client + write_url = f"{self._scheme}://{self._host}" + if self._port: + write_url += f":{self._port}" + write_url += f"{self._base_path}/api/v1/write" + + self._write_client = RemoteWriteClient( + url=write_url, + username=config.username, + password=config.password, + timeout=config.timeout, + ) + + self._job = config.job + + def write(self, samples: list[Sample]) -> None: + """Write samples using Prometheus remote write protocol.""" + if not samples: + return + + remote_samples = [ + RemoteWriteSample( + metric=s.metric, + value=s.value, + timestamp_ms=int(s.timestamp.timestamp() * 1000), + labels={"job": self._job, **s.labels}, + ) + for s in samples + ] + self._write_client.write(remote_samples) + + def query(self, query: str, time: datetime | None = None) -> QueryResult: + """Execute an instant query.""" + params = {"query": query} + if time is not None: + params["time"] = str(int(time.timestamp())) + + response = self._pool.request( + "GET", + f"{self._base_path}/api/v1/query", + fields=params, + ) + + if response.status != 200: + raise RuntimeError( + f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}" + ) + + data = json.loads(response.data.decode("utf-8")) + return self._parse_response(data) + + def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "1m", + ) -> QueryResult: + """Execute a range query.""" + params = { + "query": query, + "start": str(int(start.timestamp())), + "end": str(int(end.timestamp())), + "step": step, + } + + response = self._pool.request( + "GET", + f"{self._base_path}/api/v1/query_range", + fields=params, + ) + + if response.status != 200: + raise RuntimeError( + f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}" + ) + + data = json.loads(response.data.decode("utf-8")) + return self._parse_response(data) + + def _parse_response(self, data: dict) -> QueryResult: + """Parse VictoriaMetrics/Prometheus API response.""" + if data.get("status") != "success": + error = data.get("error", "Unknown error") + raise RuntimeError(f"Query error: {error}") + + result = data.get("data", {}).get("result", []) + series_list = [] + + for item in result: + metric = item.get("metric", {}) + + # Handle both instant query (value) and range query (values) + if "value" in item: + # Instant query: [timestamp, value] + ts, val = item["value"] + values = [(datetime.fromtimestamp(float(ts)), float(val))] + elif "values" in item: + # Range query: [[timestamp, value], ...] + values = [ + (datetime.fromtimestamp(float(ts)), float(val)) + for ts, val in item["values"] + ] + else: + values = [] + + series_list.append(QueryResultSeries(metric=metric, values=values)) + + return QueryResult(series=series_list) + + def close(self) -> None: + """Close connections.""" + self._write_client.close() + self._pool.close() diff --git a/open_ess/timeseries/victoriametrics/client.py b/open_ess/timeseries/victoriametrics/client.py new file mode 100644 index 0000000..1f966c5 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/client.py @@ -0,0 +1,151 @@ +"""Prometheus remote write client for VictoriaMetrics and Prometheus.""" + +import logging +import time +from dataclasses import dataclass, field + +import snappy +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.util import parse_url + +from .protobuf import encode_timeseries, encode_write_request + +logger = logging.getLogger(__name__) + + +@dataclass +class Sample: + """A single metric sample.""" + + metric: str + value: float + timestamp_ms: int + labels: dict[str, str] = field(default_factory=dict) + + def to_labels(self) -> list[tuple[str, str]]: + """Convert to list of label tuples including __name__.""" + return [("__name__", self.metric), *self.labels.items()] + + +class RemoteWriteError(Exception): + """Error during remote write.""" + + pass + + +class RemoteWriteClient: + """Client for Prometheus remote write protocol. + + Sends metrics to VictoriaMetrics or Prometheus using the remote write + protocol (protobuf + snappy compression). + """ + + def __init__( + self, + url: str, + *, + username: str | None = None, + password: str | None = None, + timeout: float = 30.0, + ): + """Initialize the remote write client. + + Args: + url: Remote write endpoint URL (e.g., "http://localhost:8428/api/v1/write") + username: Optional username for basic auth + password: Optional password for basic auth + timeout: Request timeout in seconds + """ + self.url = url + self.timeout = timeout + + parsed = parse_url(url) + self._path = parsed.path or "/api/v1/write" + self._host = parsed.host + self._port = parsed.port + + # Set up connection pool + pool_cls = HTTPSConnectionPool if parsed.scheme == "https" else HTTPConnectionPool + pool_kwargs: dict = { + "host": self._host, + "port": self._port, + "timeout": timeout, + "maxsize": 1, + "block": True, + } + if username and password: + pool_kwargs["headers"] = { + "Authorization": f"Basic {self._encode_basic_auth(username, password)}" + } + self._pool = pool_cls(**pool_kwargs) + + @staticmethod + def _encode_basic_auth(username: str, password: str) -> str: + """Encode credentials for basic auth header.""" + import base64 + + credentials = f"{username}:{password}".encode() + return base64.b64encode(credentials).decode("ascii") + + def write(self, samples: list[Sample]) -> None: + """Write a batch of samples. + + Args: + samples: List of samples to write. + + Raises: + RemoteWriteError: If the write fails. + """ + if not samples: + return + + # Group samples by metric+labels to create timeseries + timeseries_map: dict[tuple, list[tuple[float, int]]] = {} + for sample in samples: + key = tuple(sorted(sample.to_labels())) + if key not in timeseries_map: + timeseries_map[key] = [] + timeseries_map[key].append((sample.value, sample.timestamp_ms)) + + # Encode timeseries + encoded_timeseries = [] + for labels_tuple, sample_list in timeseries_map.items(): + labels = list(labels_tuple) + # Sort samples by timestamp as required by spec + sample_list.sort(key=lambda x: x[1]) + encoded_timeseries.append(encode_timeseries(labels, sample_list)) + + # Encode write request and compress + write_request = encode_write_request(encoded_timeseries) + compressed = snappy.compress(write_request) + + # Send request + headers = { + "Content-Type": "application/x-protobuf", + "Content-Encoding": "snappy", + "X-Prometheus-Remote-Write-Version": "0.1.0", + "User-Agent": "OpenESS/1.0", + } + + response = self._pool.urlopen( + "POST", + self._path, + body=compressed, + headers=headers, + ) + + if response.status >= 400: + raise RemoteWriteError( + f"Remote write failed: {response.status} {response.data.decode('utf-8', errors='replace')}" + ) + + logger.debug(f"Wrote {len(samples)} samples to {self.url}") + + def close(self) -> None: + """Close the connection pool.""" + self._pool.close() + + +def timestamp_ms() -> int: + """Get current timestamp in milliseconds.""" + return int(time.time() * 1000) diff --git a/open_ess/timeseries/victoriametrics/config.py b/open_ess/timeseries/victoriametrics/config.py new file mode 100644 index 0000000..8f3e161 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/config.py @@ -0,0 +1,16 @@ +"""VictoriaMetrics timeseries backend configuration.""" + +from typing import Literal + +from pydantic import BaseModel + + +class VictoriaMetricsConfig(BaseModel): + """Configuration for VictoriaMetrics backend.""" + + backend: Literal["victoriametrics"] + url: str + job: str = "open_ess" + username: str | None = None + password: str | None = None + timeout: float = 30.0 diff --git a/open_ess/timeseries/victoriametrics/protobuf.py b/open_ess/timeseries/victoriametrics/protobuf.py new file mode 100644 index 0000000..8883840 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/protobuf.py @@ -0,0 +1,118 @@ +"""Minimal protobuf encoder for Prometheus remote write protocol. + +Encodes WriteRequest, TimeSeries, Label, and Sample messages without +requiring protoc or the protobuf library. Only supports the specific +wire format needed for remote write. + +Protobuf wire format reference: +- Varint: variable-length integer encoding +- Wire type 0: varint (int64) +- Wire type 1: 64-bit fixed (double) +- Wire type 2: length-delimited (string, bytes, embedded message) +""" + +import struct + + +def _encode_varint(value: int) -> bytes: + """Encode an unsigned integer as a varint.""" + parts = [] + while value > 127: + parts.append((value & 0x7F) | 0x80) + value >>= 7 + parts.append(value) + return bytes(parts) + + +def _encode_signed_varint(value: int) -> bytes: + """Encode a signed integer as a varint (two's complement for negatives).""" + if value < 0: + value = value + (1 << 64) + return _encode_varint(value) + + +def _encode_field(field_number: int, wire_type: int, data: bytes) -> bytes: + """Encode a field with its tag.""" + tag = (field_number << 3) | wire_type + return _encode_varint(tag) + data + + +def _encode_string(field_number: int, value: str) -> bytes: + """Encode a string field (wire type 2).""" + encoded = value.encode("utf-8") + return _encode_field(field_number, 2, _encode_varint(len(encoded)) + encoded) + + +def _encode_double(field_number: int, value: float) -> bytes: + """Encode a double field (wire type 1).""" + return _encode_field(field_number, 1, struct.pack(" bytes: + """Encode an int64 field (wire type 0).""" + return _encode_field(field_number, 0, _encode_signed_varint(value)) + + +def _encode_message(field_number: int, data: bytes) -> bytes: + """Encode an embedded message field (wire type 2).""" + return _encode_field(field_number, 2, _encode_varint(len(data)) + data) + + +def encode_label(name: str, value: str) -> bytes: + """Encode a Label message. + + message Label { + string name = 1; + string value = 2; + } + """ + return _encode_string(1, name) + _encode_string(2, value) + + +def encode_sample(value: float, timestamp_ms: int) -> bytes: + """Encode a Sample message. + + message Sample { + double value = 1; + int64 timestamp = 2; + } + """ + return _encode_double(1, value) + _encode_int64(2, timestamp_ms) + + +def encode_timeseries(labels: list[tuple[str, str]], samples: list[tuple[float, int]]) -> bytes: + """Encode a TimeSeries message. + + message TimeSeries { + repeated Label labels = 1; + repeated Sample samples = 2; + } + + Args: + labels: List of (name, value) tuples. Must include ("__name__", metric_name). + Will be sorted by name as required by the spec. + samples: List of (value, timestamp_ms) tuples. + """ + data = b"" + # Labels must be sorted by name + for label_name, label_value in sorted(labels, key=lambda x: x[0]): + data += _encode_message(1, encode_label(label_name, label_value)) + for sample_value, timestamp_ms in samples: + data += _encode_message(2, encode_sample(sample_value, timestamp_ms)) + return data + + +def encode_write_request(timeseries: list[bytes]) -> bytes: + """Encode a WriteRequest message. + + message WriteRequest { + repeated TimeSeries timeseries = 1; + } + + Args: + timeseries: List of pre-encoded TimeSeries messages. + """ + data = b"" + for ts in timeseries: + data += _encode_message(1, ts) + return data diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index bfaf303..457badc 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING from open_ess.database import Database, DatabaseConnection +from open_ess.timeseries import Sample, TimeseriesBackend from .config import VictronConfig from .modbus_client import VictronModbusClient @@ -22,13 +23,19 @@ def _get_float(values: dict[Register, float | bytes | None], key: Register) -> f class VictronClient: - def __init__(self, database: Database, config: "BatterySystemConfig"): + def __init__( + self, + database: Database, + config: "BatterySystemConfig", + timeseries: TimeseriesBackend | None = None, + ): if not isinstance(config.control, VictronConfig): raise TypeError(f"VictronClient requires VictronConfig, got {type(config.control).__name__}") self._db = database self._config = config self._control: VictronConfig = config.control self._client = VictronModbusClient(self._control) + self._timeseries = timeseries self._db_conn: DatabaseConnection | None = None self._serial: str | None = None @@ -129,147 +136,106 @@ def write_setpoints(self) -> None: self.write(self.vebus_id, VEBus.ESS_DISABLE_FEEDBACK, 1) def collect_and_store_measurements(self) -> None: - if self._db_conn is None: + if self._db_conn is None and self._timeseries is None: return timestamp = datetime.now(UTC) + samples: list[Sample] = [] + + def add_sample(metric: str, value: float | None) -> None: + if value is not None: + samples.append(Sample(metric, value, timestamp)) # Read System registers system_regs = [System.GRID_L1, System.GRID_L2, System.GRID_L3] system_values = self.read_many(self.system_id, system_regs) - self._db_conn.insert_power("grid/power/l1", timestamp, _get_float(system_values, System.GRID_L1)) - self._db_conn.insert_power("grid/power/l2", timestamp, _get_float(system_values, System.GRID_L2)) - self._db_conn.insert_power("grid/power/l3", timestamp, _get_float(system_values, System.GRID_L3)) + add_sample("grid/power/l1", _get_float(system_values, System.GRID_L1)) + add_sample("grid/power/l2", _get_float(system_values, System.GRID_L2)) + add_sample("grid/power/l3", _get_float(system_values, System.GRID_L3)) if self.grid_id: - # TODO: check if grid meter delivers data per phase or not grid_values = self.read_many( self.grid_id, [GridMeter.ENERGY_TO_NET_TOTAL, GridMeter.ENERGY_FROM_NET_TOTAL], ) - self._db_conn.insert_energy( - "grid/energy/import/total", timestamp, _get_float(grid_values, GridMeter.ENERGY_FROM_NET_TOTAL) - ) - self._db_conn.insert_energy( - "grid/energy/export/total", timestamp, _get_float(grid_values, GridMeter.ENERGY_TO_NET_TOTAL) - ) + add_sample("grid/energy/import/total", _get_float(grid_values, GridMeter.ENERGY_FROM_NET_TOTAL)) + add_sample("grid/energy/export/total", _get_float(grid_values, GridMeter.ENERGY_TO_NET_TOTAL)) if self.pvinverter_id: pvinverter_values = self.read_many( self.pvinverter_id, - [ - SolarInverter.ENERGY_L1, - SolarInverter.POWER_L1, - ], + [SolarInverter.ENERGY_L1, SolarInverter.POWER_L1], ) - self._db_conn.insert_energy( + add_sample( f"victron/pvinverter/{self.pvinverter_id}/energy/l1", - timestamp, _get_float(pvinverter_values, SolarInverter.ENERGY_L1), ) - self._db_conn.insert_power( + add_sample( f"victron/pvinverter/{self.pvinverter_id}/power/l1", - timestamp, _get_float(pvinverter_values, SolarInverter.POWER_L1), ) - # VEBus registers for each device + # VEBus registers vebus_regs = [ VEBus.AC_INPUT_POWER_L1, - # VEBus.AC_INPUT_POWER_L2, - # VEBus.AC_INPUT_POWER_L3, VEBus.AC_OUTPUT_POWER_L1, - # VEBus.AC_OUTPUT_POWER_L2, - # VEBus.AC_OUTPUT_POWER_L3, VEBus.DC_CURRENT, VEBus.DC_VOLTAGE, VEBus.SOC, - # Energy counters VEBus.ENERGY_AC_IN1_TO_AC_OUT, VEBus.ENERGY_AC_IN1_TO_BATTERY, - # VEBus.ENERGY_AC_IN2_TO_AC_OUT, - # VEBus.ENERGY_AC_IN2_TO_BATTERY, VEBus.ENERGY_AC_OUT_TO_AC_IN1, - # VEBus.ENERGY_AC_OUT_TO_AC_IN2, VEBus.ENERGY_BATTERY_TO_AC_IN1, - # VEBus.ENERGY_BATTERY_TO_AC_IN2, VEBus.ENERGY_BATTERY_TO_AC_OUT, VEBus.ENERGY_AC_OUT_TO_BATTERY, ] vebus_prefix = self._control.vebus_prefix - vebus_values = self.read_many(self.vebus_id, vebus_regs) - self._db_conn.insert_power( - f"{vebus_prefix}/power/ac_in/l1", timestamp, _get_float(vebus_values, VEBus.AC_INPUT_POWER_L1) - ) - self._db_conn.insert_power( - f"{vebus_prefix}/power/ac_out/l1", timestamp, _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1) - ) - - soc = _get_float(vebus_values, VEBus.SOC) - if soc is not None: - self._db_conn.insert_soc(f"{vebus_prefix}/soc", timestamp, int(soc)) + add_sample(f"{vebus_prefix}/power/ac_in/l1", _get_float(vebus_values, VEBus.AC_INPUT_POWER_L1)) + add_sample(f"{vebus_prefix}/power/ac_out/l1", _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1)) + add_sample(f"{vebus_prefix}/soc", _get_float(vebus_values, VEBus.SOC)) dc_current = _get_float(vebus_values, VEBus.DC_CURRENT) dc_voltage = _get_float(vebus_values, VEBus.DC_VOLTAGE) - if dc_voltage is not None: - self._db_conn.insert_voltage(f"{vebus_prefix}/voltage/battery", timestamp, dc_voltage) - if dc_current is not None: - dc_power = dc_current * dc_voltage - self._db_conn.insert_power(f"{vebus_prefix}/power/battery", timestamp, dc_power) + add_sample(f"{vebus_prefix}/voltage/battery", dc_voltage) + if dc_voltage is not None and dc_current is not None: + add_sample(f"{vebus_prefix}/power/battery", dc_current * dc_voltage) # Energy flows - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_in_to_ac_out", - timestamp, - _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT), - ) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_in_import", timestamp, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY) - ) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_AC_IN2_TO_AC_OUT)) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_AC_IN2_TO_BATTERY)) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_out_to_ac_in", - timestamp, - _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1), - ) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_AC_OUT_TO_AC_IN2)) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_in_export", timestamp, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_IN1) - ) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_BATTERY_TO_AC_IN2)) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_out_export", timestamp, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT) - ) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_out_import", timestamp, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY) - ) + add_sample(f"{vebus_prefix}/energy/ac_in_to_ac_out", _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT)) + add_sample(f"{vebus_prefix}/energy/ac_in_import", _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY)) + add_sample(f"{vebus_prefix}/energy/ac_out_to_ac_in", _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1)) + add_sample(f"{vebus_prefix}/energy/ac_in_export", _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_IN1)) + add_sample(f"{vebus_prefix}/energy/ac_out_export", _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT)) + add_sample(f"{vebus_prefix}/energy/ac_out_import", _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY)) if self.battery_id is not None: bms_prefix = self._control.battery_prefix - bms_values = self.read_many( self.battery_id, - [ - Battery.DC_VOLTAGE, - Battery.DC_POWER, - Battery.SOC, - # Battery.CHARGED_ENERGY, - # Battery.DISCHARGED_ENERGY, - ], - ) - - self._db_conn.insert_power( - f"{bms_prefix}/power/battery", timestamp, _get_float(bms_values, Battery.DC_POWER) - ) - self._db_conn.insert_voltage( - f"{bms_prefix}/voltage/battery", timestamp, _get_float(bms_values, Battery.DC_VOLTAGE) + [Battery.DC_VOLTAGE, Battery.DC_POWER, Battery.SOC], ) - bms_soc = _get_float(bms_values, Battery.SOC) - if bms_soc is not None: - self._db_conn.insert_soc(f"{bms_prefix}/soc", timestamp, round(bms_soc)) + add_sample(f"{bms_prefix}/power/battery", _get_float(bms_values, Battery.DC_POWER)) + add_sample(f"{bms_prefix}/voltage/battery", _get_float(bms_values, Battery.DC_VOLTAGE)) + add_sample(f"{bms_prefix}/soc", _get_float(bms_values, Battery.SOC)) + + # Write to timeseries backend + if self._timeseries is not None and samples: + try: + self._timeseries.write(samples) + except Exception: + logger.exception("Failed to write samples to timeseries backend") + + # Legacy: also write to database + if self._db_conn is not None: + for sample in samples: + if "power" in sample.metric or "voltage" in sample.metric: + self._db_conn.insert_power(sample.metric, sample.timestamp, sample.value) + elif "energy" in sample.metric: + self._db_conn.insert_energy(sample.metric, sample.timestamp, sample.value) + elif "soc" in sample.metric: + self._db_conn.insert_soc(sample.metric, sample.timestamp, round(sample.value)) # --------------------------------# # VictronModbusClient bindings # diff --git a/open_ess/victron_modbus/service.py b/open_ess/victron_modbus/service.py index 2dd48c0..7ad304a 100644 --- a/open_ess/victron_modbus/service.py +++ b/open_ess/victron_modbus/service.py @@ -4,6 +4,7 @@ from open_ess.database import Database from open_ess.service import Service +from open_ess.timeseries import TimeseriesBackend from .client import VictronClient @@ -16,10 +17,15 @@ class VictronService(Service): """Collects measurements from Victron GX every second.""" - def __init__(self, db: Database, config: "BatterySystemConfig"): + def __init__( + self, + db: Database, + config: "BatterySystemConfig", + timeseries: TimeseriesBackend | None = None, + ): super().__init__("VictronService") self._config = config - self._client = VictronClient(db, config) + self._client = VictronClient(db, config, timeseries) @property def client(self) -> VictronClient: diff --git a/pyproject.toml b/pyproject.toml index 34e1edf..dea5a4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,15 @@ warn_unused_ignores = true disallow_untyped_defs = true [[tool.mypy.overrides]] -module = ["entsoe", "entsoe.*", "pyomo", "pyomo.*"] +module = [ + "entsoe", "entsoe.*", + "metricsqlite", "metricsqlite.*", + "pandas", "pandas.*", + "pyomo", "pyomo.*", + "snappy", + "urllib3", "urllib3.*", + "yaml", +] ignore_missing_imports = true [[tool.mypy.overrides]] @@ -72,8 +80,11 @@ dependencies = [ "pydantic", "pymodbus", "pyomo", + "python-snappy", "pyyaml", "uvicorn", + "urllib3", # victoriametrics client + "python-snappy", # victoriametrics client ] [project.optional-dependencies] From 69fc329b0899f8cf365bb4880f8b45fee1b71aaa Mon Sep 17 00:00:00 2001 From: david Date: Mon, 4 May 2026 13:47:07 +0200 Subject: [PATCH 03/18] rename timeseries --- open_ess/timeseries/metricsqlite/backend.py | 14 +- .../timeseries/victoriametrics/backend.py | 17 +-- open_ess/timeseries/victoriametrics/client.py | 4 +- open_ess/timeseries/victoriametrics/config.py | 1 - open_ess/victron_modbus/client.py | 137 ++++++++++-------- open_ess/victron_modbus/registers.py | 4 +- 6 files changed, 88 insertions(+), 89 deletions(-) diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py index 4a3508f..a73272d 100644 --- a/open_ess/timeseries/metricsqlite/backend.py +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -59,9 +59,7 @@ def query_range( result = self._client.query_range(query, start=start_ms, end=end_ms, step=step) return self._convert_matrix_result(result) - def _convert_result( - self, result: InstantVector | RangeVectorResult | ScalarResult - ) -> QueryResult: + def _convert_result(self, result: InstantVector | RangeVectorResult | ScalarResult) -> QueryResult: """Convert metricsqlite result to QueryResult.""" if isinstance(result, ScalarResult): # Scalar result - single value @@ -78,10 +76,7 @@ def _convert_result( # Range vector from instant query (e.g., metric[5m]) series_list = [] for labels, samples in result.series: - values = [ - (datetime.fromtimestamp(sample.timestamp / 1000), sample.value) - for sample in samples - ] + values = [(datetime.fromtimestamp(sample.timestamp / 1000), sample.value) for sample in samples] series_list.append(QueryResultSeries(metric=labels, values=values)) return QueryResult(series=series_list) @@ -100,10 +95,7 @@ def _convert_matrix_result(self, result: MatrixResult) -> QueryResult: """Convert metricsqlite MatrixResult to QueryResult.""" series_list = [] for labels, samples in result.series: - values = [ - (datetime.fromtimestamp(sample.timestamp / 1000), sample.value) - for sample in samples - ] + values = [(datetime.fromtimestamp(sample.timestamp / 1000), sample.value) for sample in samples] series_list.append(QueryResultSeries(metric=labels, values=values)) return QueryResult(series=series_list) diff --git a/open_ess/timeseries/victoriametrics/backend.py b/open_ess/timeseries/victoriametrics/backend.py index bd746f7..c5003b1 100644 --- a/open_ess/timeseries/victoriametrics/backend.py +++ b/open_ess/timeseries/victoriametrics/backend.py @@ -60,8 +60,6 @@ def __init__(self, config: VictoriaMetricsConfig): timeout=config.timeout, ) - self._job = config.job - def write(self, samples: list[Sample]) -> None: """Write samples using Prometheus remote write protocol.""" if not samples: @@ -72,7 +70,7 @@ def write(self, samples: list[Sample]) -> None: metric=s.metric, value=s.value, timestamp_ms=int(s.timestamp.timestamp() * 1000), - labels={"job": self._job, **s.labels}, + labels=s.labels, ) for s in samples ] @@ -91,9 +89,7 @@ def query(self, query: str, time: datetime | None = None) -> QueryResult: ) if response.status != 200: - raise RuntimeError( - f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}" - ) + raise RuntimeError(f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}") data = json.loads(response.data.decode("utf-8")) return self._parse_response(data) @@ -120,9 +116,7 @@ def query_range( ) if response.status != 200: - raise RuntimeError( - f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}" - ) + raise RuntimeError(f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}") data = json.loads(response.data.decode("utf-8")) return self._parse_response(data) @@ -146,10 +140,7 @@ def _parse_response(self, data: dict) -> QueryResult: values = [(datetime.fromtimestamp(float(ts)), float(val))] elif "values" in item: # Range query: [[timestamp, value], ...] - values = [ - (datetime.fromtimestamp(float(ts)), float(val)) - for ts, val in item["values"] - ] + values = [(datetime.fromtimestamp(float(ts)), float(val)) for ts, val in item["values"]] else: values = [] diff --git a/open_ess/timeseries/victoriametrics/client.py b/open_ess/timeseries/victoriametrics/client.py index 1f966c5..561ae28 100644 --- a/open_ess/timeseries/victoriametrics/client.py +++ b/open_ess/timeseries/victoriametrics/client.py @@ -74,9 +74,7 @@ def __init__( "block": True, } if username and password: - pool_kwargs["headers"] = { - "Authorization": f"Basic {self._encode_basic_auth(username, password)}" - } + pool_kwargs["headers"] = {"Authorization": f"Basic {self._encode_basic_auth(username, password)}"} self._pool = pool_cls(**pool_kwargs) @staticmethod diff --git a/open_ess/timeseries/victoriametrics/config.py b/open_ess/timeseries/victoriametrics/config.py index 8f3e161..5ab2543 100644 --- a/open_ess/timeseries/victoriametrics/config.py +++ b/open_ess/timeseries/victoriametrics/config.py @@ -10,7 +10,6 @@ class VictoriaMetricsConfig(BaseModel): backend: Literal["victoriametrics"] url: str - job: str = "open_ess" username: str | None = None password: str | None = None timeout: float = 30.0 diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index 457badc..6ce02a5 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -15,6 +15,11 @@ logger = logging.getLogger(__name__) +POWER_METRIC = "openess_power_watts" +ENERGY_METRIC = "openess_energy_kwh" +SOC_METRIC = "openess_soc_ratio" +VOLTAGE_METRIC = "openess_voltage_volts" + def _get_float(values: dict[Register, float | bytes | None], key: Register) -> float | None: """Extract a float value from read_many results, filtering out bytes and None.""" @@ -136,45 +141,48 @@ def write_setpoints(self) -> None: self.write(self.vebus_id, VEBus.ESS_DISABLE_FEEDBACK, 1) def collect_and_store_measurements(self) -> None: - if self._db_conn is None and self._timeseries is None: + if self._timeseries is None: return timestamp = datetime.now(UTC) samples: list[Sample] = [] - def add_sample(metric: str, value: float | None) -> None: + def add(metric: str, value: float | None, labels: dict[str, str]) -> None: + labels["device"] = self.serial if value is not None: - samples.append(Sample(metric, value, timestamp)) + samples.append(Sample(metric, value, timestamp, labels)) - # Read System registers + # Grid power system_regs = [System.GRID_L1, System.GRID_L2, System.GRID_L3] system_values = self.read_many(self.system_id, system_regs) - add_sample("grid/power/l1", _get_float(system_values, System.GRID_L1)) - add_sample("grid/power/l2", _get_float(system_values, System.GRID_L2)) - add_sample("grid/power/l3", _get_float(system_values, System.GRID_L3)) + add(POWER_METRIC, _get_float(system_values, System.GRID_L1), {"from": "grid", "phase": "L1"}) + add(POWER_METRIC, _get_float(system_values, System.GRID_L2), {"from": "grid", "phase": "L2"}) + add(POWER_METRIC, _get_float(system_values, System.GRID_L3), {"from": "grid", "phase": "L3"}) + # Grid energy if self.grid_id: grid_values = self.read_many( - self.grid_id, - [GridMeter.ENERGY_TO_NET_TOTAL, GridMeter.ENERGY_FROM_NET_TOTAL], + self.grid_id, [GridMeter.ENERGY_TO_GRID_TOTAL, GridMeter.ENERGY_FROM_GRID_TOTAL] + ) + add( + ENERGY_METRIC, + _get_float(grid_values, GridMeter.ENERGY_FROM_GRID_TOTAL), + {"from": "grid", "phase": "total"}, + ) + add( + ENERGY_METRIC, _get_float(grid_values, GridMeter.ENERGY_TO_GRID_TOTAL), {"to": "grid", "phase": "total"} ) - add_sample("grid/energy/import/total", _get_float(grid_values, GridMeter.ENERGY_FROM_NET_TOTAL)) - add_sample("grid/energy/export/total", _get_float(grid_values, GridMeter.ENERGY_TO_NET_TOTAL)) + # PV inverter if self.pvinverter_id: - pvinverter_values = self.read_many( + pv_values = self.read_many( self.pvinverter_id, [SolarInverter.ENERGY_L1, SolarInverter.POWER_L1], ) - add_sample( - f"victron/pvinverter/{self.pvinverter_id}/energy/l1", - _get_float(pvinverter_values, SolarInverter.ENERGY_L1), - ) - add_sample( - f"victron/pvinverter/{self.pvinverter_id}/power/l1", - _get_float(pvinverter_values, SolarInverter.POWER_L1), - ) + pv_labels = {"from": "pvinverter", "unit_id": str(self.pvinverter_id), "phase": "L1"} + add(POWER_METRIC, _get_float(pv_values, SolarInverter.POWER_L1), pv_labels) + add(ENERGY_METRIC, _get_float(pv_values, SolarInverter.ENERGY_L1), pv_labels) - # VEBus registers + # VEBus (inverter/charger) vebus_regs = [ VEBus.AC_INPUT_POWER_L1, VEBus.AC_OUTPUT_POWER_L1, @@ -188,54 +196,65 @@ def add_sample(metric: str, value: float | None) -> None: VEBus.ENERGY_BATTERY_TO_AC_OUT, VEBus.ENERGY_AC_OUT_TO_BATTERY, ] - - vebus_prefix = self._control.vebus_prefix vebus_values = self.read_many(self.vebus_id, vebus_regs) - add_sample(f"{vebus_prefix}/power/ac_in/l1", _get_float(vebus_values, VEBus.AC_INPUT_POWER_L1)) - add_sample(f"{vebus_prefix}/power/ac_out/l1", _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1)) - add_sample(f"{vebus_prefix}/soc", _get_float(vebus_values, VEBus.SOC)) - - dc_current = _get_float(vebus_values, VEBus.DC_CURRENT) - dc_voltage = _get_float(vebus_values, VEBus.DC_VOLTAGE) - add_sample(f"{vebus_prefix}/voltage/battery", dc_voltage) - if dc_voltage is not None and dc_current is not None: - add_sample(f"{vebus_prefix}/power/battery", dc_current * dc_voltage) - - # Energy flows - add_sample(f"{vebus_prefix}/energy/ac_in_to_ac_out", _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT)) - add_sample(f"{vebus_prefix}/energy/ac_in_import", _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY)) - add_sample(f"{vebus_prefix}/energy/ac_out_to_ac_in", _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1)) - add_sample(f"{vebus_prefix}/energy/ac_in_export", _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_IN1)) - add_sample(f"{vebus_prefix}/energy/ac_out_export", _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT)) - add_sample(f"{vebus_prefix}/energy/ac_out_import", _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY)) - + # VEBus power + add( + POWER_METRIC, + _get_float(vebus_values, VEBus.AC_INPUT_POWER_L1), + {"from": "ac_in", "to": "system", "phase": "L1"}, + ) + add( + POWER_METRIC, + _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1), + {"from": "ac_out", "to": "system", "phase": "L1"}, + ) + + # VEBus battery + vebus_dc_current = _get_float(vebus_values, VEBus.DC_CURRENT) + vebus_dc_voltage = _get_float(vebus_values, VEBus.DC_VOLTAGE) + vebus_battery_power = ( + vebus_dc_current * vebus_dc_voltage + if vebus_dc_current is not None and vebus_dc_voltage is not None + else None + ) + add(POWER_METRIC, vebus_battery_power, {"from": "system", "to": "battery", "unit": "vebus"}) + add(VOLTAGE_METRIC, vebus_dc_voltage, {"node": "battery", "unit": "vebus"}) + + # VEBus SOC + vebus_soc = _get_float(vebus_values, VEBus.SOC) + if vebus_soc is not None: + add(SOC_METRIC, vebus_soc / 100, {"node": "battery", "unit": "vebus"}) + + # VEBus energy flows + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT), {"from": "ac_in", "to": "ac_out"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY), {"from": "ac_in", "to": "system"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1), {"from": "ac_out", "to": "ac_in"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_IN1), {"from": "system", "to": "ac_in"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT), {"from": "system", "to": "ac_out"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY), {"from": "ac_out", "to": "system"}) + + # BMS (direct battery measurements) if self.battery_id is not None: - bms_prefix = self._control.battery_prefix bms_values = self.read_many( self.battery_id, [Battery.DC_VOLTAGE, Battery.DC_POWER, Battery.SOC], ) - add_sample(f"{bms_prefix}/power/battery", _get_float(bms_values, Battery.DC_POWER)) - add_sample(f"{bms_prefix}/voltage/battery", _get_float(bms_values, Battery.DC_VOLTAGE)) - add_sample(f"{bms_prefix}/soc", _get_float(bms_values, Battery.SOC)) + add( + POWER_METRIC, + _get_float(bms_values, Battery.DC_POWER), + {"from": "system", "to": "battery", "unit": "battery"}, + ) + add(VOLTAGE_METRIC, _get_float(bms_values, Battery.DC_VOLTAGE), {"node": "battery", "unit": "battery"}) + bms_soc = _get_float(bms_values, Battery.SOC) + if bms_soc is not None: + add(SOC_METRIC, bms_soc / 100, {"node": "battery", "unit": "battery"}) - # Write to timeseries backend - if self._timeseries is not None and samples: + if samples: try: self._timeseries.write(samples) - except Exception: - logger.exception("Failed to write samples to timeseries backend") - - # Legacy: also write to database - if self._db_conn is not None: - for sample in samples: - if "power" in sample.metric or "voltage" in sample.metric: - self._db_conn.insert_power(sample.metric, sample.timestamp, sample.value) - elif "energy" in sample.metric: - self._db_conn.insert_energy(sample.metric, sample.timestamp, sample.value) - elif "soc" in sample.metric: - self._db_conn.insert_soc(sample.metric, sample.timestamp, round(sample.value)) + except Exception as e: + logger.exception(f"Failed to write samples to timeseries backend: {e}") # --------------------------------# # VictronModbusClient bindings # diff --git a/open_ess/victron_modbus/registers.py b/open_ess/victron_modbus/registers.py index e9c2d5e..0be6801 100644 --- a/open_ess/victron_modbus/registers.py +++ b/open_ess/victron_modbus/registers.py @@ -272,5 +272,5 @@ class GridMeter: ENERGY_FROM_NET_L3 = Register("Energy from net L3", 2630, DataType.UINT32, scale=100) ENERGY_TO_NET_L3 = Register("Energy to net L3", 2632, DataType.UINT32, scale=100) - ENERGY_FROM_NET_TOTAL = Register("Energy from net", 2634, DataType.UINT32, scale=100) - ENERGY_TO_NET_TOTAL = Register("Energy to net", 2636, DataType.UINT32, scale=100) + ENERGY_FROM_GRID_TOTAL = Register("Energy from net", 2634, DataType.UINT32, scale=100) + ENERGY_TO_GRID_TOTAL = Register("Energy to net", 2636, DataType.UINT32, scale=100) From a1cfd4b7ec6668a315ea60e229561b6d25ecac86 Mon Sep 17 00:00:00 2001 From: david Date: Mon, 4 May 2026 21:42:59 +0200 Subject: [PATCH 04/18] frontend: use MetricsQL --- open_ess/battery_system/__init__.py | 4 +- open_ess/battery_system/battery_system.py | 4 +- open_ess/battery_system/config.py | 87 +-- open_ess/frontend/app.py | 15 +- open_ess/frontend/dependencies.py | 6 + open_ess/frontend/routes/__init__.py | 3 +- open_ess/frontend/routes/api.py | 706 ++++++++++++++++++---- open_ess/frontend/routes/timeseries.py | 85 +++ open_ess/frontend/routes/util.py | 25 + open_ess/frontend/static/metrics.js | 77 ++- open_ess/frontend/static/timeseries.js | 201 ++++++ open_ess/frontend/templates/base.html | 1 + open_ess/victron_modbus/config.py | 8 - 13 files changed, 1012 insertions(+), 210 deletions(-) create mode 100644 open_ess/frontend/routes/timeseries.py create mode 100644 open_ess/frontend/static/timeseries.js diff --git a/open_ess/battery_system/__init__.py b/open_ess/battery_system/__init__.py index 48377e4..4132a0f 100644 --- a/open_ess/battery_system/__init__.py +++ b/open_ess/battery_system/__init__.py @@ -1,4 +1,4 @@ from .battery_system import BatterySystem, VictronBatterySystem -from .config import BatterySystemConfig +from .config import BatterySystemConfig, QueriesConfig -__all__ = ["BatterySystem", "BatterySystemConfig", "VictronBatterySystem"] +__all__ = ["BatterySystem", "BatterySystemConfig", "QueriesConfig", "VictronBatterySystem"] diff --git a/open_ess/battery_system/battery_system.py b/open_ess/battery_system/battery_system.py index 4bfc80c..b545c65 100644 --- a/open_ess/battery_system/battery_system.py +++ b/open_ess/battery_system/battery_system.py @@ -37,9 +37,7 @@ def __init__(self, config: BatterySystemConfig, control: VictronClient): @property def id(self) -> str | None: - if self._victron_client.serial is None: - return None - return f"victron/{self._victron_client.serial}" + return self._victron_client.serial def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: if until is None: diff --git a/open_ess/battery_system/config.py b/open_ess/battery_system/config.py index 144efa0..ea7e7be 100644 --- a/open_ess/battery_system/config.py +++ b/open_ess/battery_system/config.py @@ -1,6 +1,6 @@ from typing import Annotated, Literal -from pydantic import BaseModel, Field, computed_field, model_validator +from pydantic import BaseModel, Field, model_validator from open_ess.victron_modbus import VictronConfig @@ -9,10 +9,6 @@ class MqttControl(BaseModel): type: Literal["mqtt"] = "mqtt" topic: str - @property - def metrics_prefix(self) -> str: - return f"mqtt/{self.topic}" - class MetricsConfig(BaseModel): battery_soc: str | list[str] | None = None @@ -25,6 +21,25 @@ class MetricsConfig(BaseModel): energy_from_battery: str | list[str] | None = None +class QueriesConfig(BaseModel): + # Battery state + soc: str = 'openess_soc_ratio{node="battery", device="$device"} * 100' + voltage: str = 'openess_voltage_volts{node="battery", device="$device"}' + + # Power + power_grid: str = 'openess_power_watts{from="grid", device="$device"}' + power_pv: str = 'openess_power_watts{from="pvinverter", device="$device"}' + power_battery: str = 'openess_power_watts{from="system", to="battery", device="$device"}' + power_ac_in: str = 'openess_power_watts{from="ac_in", device="$device"}' + power_ac_out: str = 'openess_power_watts{from="ac_out", device="$device"}' + + # Energy + energy_grid_import: str = 'openess_energy_kwh{from="grid", device="$device"}' + energy_grid_export: str = 'openess_energy_kwh{to="grid", device="$device"}' + energy_to_battery: str = 'openess_energy_kwh{to="system", device="$device"}' + energy_from_battery: str = 'openess_energy_kwh{from="system", device="$device"}' + + class BatterySystemConfig(BaseModel): name: str | None = None # Is set to self.id if not provided. monitor_only: bool = False @@ -38,14 +53,7 @@ class BatterySystemConfig(BaseModel): control: Annotated[VictronConfig | MqttControl, Field(discriminator="type")] metrics: MetricsConfig = MetricsConfig() - - @computed_field # type: ignore[prop-decorator] - @property - def id(self) -> str: - if isinstance(self.control, VictronConfig): - return f"victron/vebus/{self.control.vebus_id}" - else: - return f"mqtt/{self.control.topic}" + queries: QueriesConfig = QueriesConfig() @property def is_victron(self) -> bool: @@ -63,56 +71,3 @@ def check_power_limits(self) -> "BatterySystemConfig": "max_invert_power_kw is not configured. Either set a value or set monitor_only to True." ) return self - - @model_validator(mode="after") - def set_defaults(self) -> "BatterySystemConfig": - if self.name is None: - self.name = self.id - - if isinstance(self.control, VictronConfig): - vebus_prefix = self.control.vebus_prefix - bms_prefix = self.control.battery_prefix - - if self.metrics.battery_soc is None: - if bms_prefix: - self.metrics.battery_soc = [f"{bms_prefix}/soc", f"{vebus_prefix}/soc"] - else: - self.metrics.battery_soc = f"{vebus_prefix}/soc" - if self.metrics.battery_voltage is None: - if bms_prefix: - self.metrics.battery_voltage = [f"{bms_prefix}/voltage/battery", f"{vebus_prefix}/voltage/battery"] - else: - self.metrics.battery_voltage = f"{vebus_prefix}/voltage/battery" - if self.metrics.power_to_system is None: - self.metrics.power_to_system = f"{vebus_prefix}/power/ac_in/l1" - if self.metrics.power_to_battery is None: - if bms_prefix: - self.metrics.power_to_battery = [f"{bms_prefix}/power/battery", f"{vebus_prefix}/power/battery"] - else: - self.metrics.power_to_battery = f"{vebus_prefix}/power/battery" - if self.metrics.energy_to_system is None: - self.metrics.energy_to_system = f"{vebus_prefix}/energy/ac_in_import" # TODO + ac_out_import - if self.metrics.energy_from_system is None: - self.metrics.energy_from_system = f"{vebus_prefix}/energy/ac_in_export" # TODO + ac_out_export - if self.metrics.energy_to_battery is None: - if bms_prefix: - self.metrics.energy_to_battery = [ - f"{bms_prefix}/energy/charged_energy", - f"{bms_prefix}/power/battery", # integrate power to obtain energy - f"{vebus_prefix}/power/battery", # integrate power to obtain energy - ] - else: - self.metrics.energy_to_battery = f"{vebus_prefix}/power/battery" - if self.metrics.energy_from_battery is None: - if bms_prefix: - self.metrics.energy_from_battery = [ - f"{bms_prefix}/energy/discharged_energy", - f"-{bms_prefix}/power/battery", # integrate power to obtain energy - f"-{vebus_prefix}/power/battery", # integrate power to obtain energy - ] - else: - self.metrics.energy_from_battery = f"-{vebus_prefix}/power/battery" - else: - pass # TODO - - return self diff --git a/open_ess/frontend/app.py b/open_ess/frontend/app.py index 64aacef..bf7fc2b 100644 --- a/open_ess/frontend/app.py +++ b/open_ess/frontend/app.py @@ -8,8 +8,10 @@ from open_ess.battery_system import BatterySystem from open_ess.database import Database +from open_ess.timeseries import TimeseriesBackend +from open_ess.timeseries.metricsqlite.backend import MetricSQLiteBackend -from .routes import api_router, pages_router +from .routes import api_router, pages_router, timeseries_router if TYPE_CHECKING: from open_ess.config import Config @@ -21,12 +23,14 @@ def create_app( database: Database, config: "Config", battery_systems: list[BatterySystem], + timeseries: TimeseriesBackend | None = None, ) -> FastAPI: @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: _app.state.database = database.connect() _app.state.price_config = config.prices _app.state.battery_systems = battery_systems + _app.state.timeseries = timeseries yield _app.state.database.close() @@ -39,4 +43,13 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: app.include_router(pages_router) app.include_router(api_router, prefix="/api") + # Mount timeseries query endpoints + if timeseries is not None: + if isinstance(timeseries, MetricSQLiteBackend): + from metricsqlite.fastapi import create_router as create_metricsqlite_router + + app.include_router(create_metricsqlite_router(timeseries._client), prefix="/api/v1") + else: + app.include_router(timeseries_router, prefix="/api/v1") + return app diff --git a/open_ess/frontend/dependencies.py b/open_ess/frontend/dependencies.py index 8b870c9..72678e2 100644 --- a/open_ess/frontend/dependencies.py +++ b/open_ess/frontend/dependencies.py @@ -5,6 +5,7 @@ from open_ess.battery_system import BatterySystem from open_ess.database import DatabaseConnection from open_ess.pricing import PriceConfig +from open_ess.timeseries import TimeseriesBackend def get_database(request: Request) -> DatabaseConnection: @@ -19,7 +20,12 @@ def get_battery_systems(request: Request) -> list[BatterySystem]: return request.app.state.battery_systems # type: ignore[no-any-return] +def get_timeseries(request: Request) -> TimeseriesBackend | None: + return request.app.state.timeseries # type: ignore[no-any-return] + + # Type aliases for cleaner route signatures Database = Annotated[DatabaseConnection, Depends(get_database)] PriceConfigDep = Annotated[PriceConfig, Depends(get_price_config)] BatterySystemsDep = Annotated[list[BatterySystem], Depends(get_battery_systems)] +TimeseriesDep = Annotated[TimeseriesBackend | None, Depends(get_timeseries)] diff --git a/open_ess/frontend/routes/__init__.py b/open_ess/frontend/routes/__init__.py index df385c6..2a8a778 100644 --- a/open_ess/frontend/routes/__init__.py +++ b/open_ess/frontend/routes/__init__.py @@ -1,4 +1,5 @@ from .api import router as api_router from .pages import router as pages_router +from .timeseries import router as timeseries_router -__all__ = ["api_router", "pages_router"] +__all__ = ["api_router", "pages_router", "timeseries_router"] diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index ec31e11..249f2f6 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,13 +1,18 @@ import logging from datetime import UTC, datetime, timedelta from enum import StrEnum +from typing import TYPE_CHECKING from fastapi import APIRouter, HTTPException, Query from pydantic import BaseModel -from open_ess.frontend.dependencies import BatterySystemsDep, Database, PriceConfigDep +from open_ess.frontend.dependencies import BatterySystemsDep, Database, PriceConfigDep, TimeseriesDep -from .util import TimeSeries, data_to_timeseries, find_full_battery_cycles +from .util import TimeSeries, data_to_timeseries, find_full_battery_cycles, query_result_to_timeseries + +if TYPE_CHECKING: + from open_ess.database import DatabaseConnection + from open_ess.timeseries import QueryResult, TimeseriesBackend logger = logging.getLogger(__name__) @@ -81,11 +86,83 @@ class PowerFlowData(BaseModel): batteries: dict[str, BatteryPowerValues] +def _get_instant_value(result: "QueryResult") -> float | None: + """Extract the latest value from an instant query result.""" + if result.series and result.series[0].values: + return result.series[0].values[-1][1] + return None + + @router.get("/power-flow", response_model=PowerFlowData) -async def get_power_flow(db: Database, battery_systems: BatterySystemsDep) -> PowerFlowData: +async def get_power_flow( + db: Database, + timeseries: TimeseriesDep, + battery_systems: BatterySystemsDep, +) -> PowerFlowData: + if timeseries is not None: + return await _get_power_flow_timeseries(timeseries, battery_systems) + return await _get_power_flow_legacy(db, battery_systems) + + +async def _get_power_flow_timeseries( + timeseries: "TimeseriesBackend", + battery_systems: list, +) -> PowerFlowData: + """Get power flow data from timeseries backend.""" + now = datetime.now(UTC) + + # Grid power per phase + grid_power: dict[str, float | None] = {} + for phase in ("L1", "L2", "L3"): + query = f'openess_power_watts{{from="grid", phase="{phase}"}}' + result = timeseries.query(query, now) + grid_power[phase] = _get_instant_value(result) + + # Solar power + solar_query = 'openess_power_watts{from="pvinverter"}' + solar_result = timeseries.query(solar_query, now) + solar_power = _get_instant_value(solar_result) + + # Battery power for each system + batteries: dict[str, BatteryPowerValues] = {} + for battery_system in battery_systems: + device = battery_system.id + + # AC power (charger/inverter) + ac_in_query = battery_system.config.queries.power_ac_in.replace("$device", device) + ac_in_result = timeseries.query(ac_in_query, now) + system = _get_instant_value(ac_in_result) or 0 + + charger = -system if system < 0 else 0 + inverter = system if system > 0 else 0 + + # DC battery power + battery_query = battery_system.config.queries.power_battery.replace("$device", device) + battery_result = timeseries.query(battery_query, now) + battery = _get_instant_value(battery_result) or 0 + + losses = battery - system + + batteries[battery_system.id] = BatteryPowerValues( + charger=charger, + inverter=inverter, + battery=battery, + losses=losses, + ) + + return PowerFlowData( + grid=grid_power, + solar=solar_power, + consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, + batteries=batteries, + ) + + +async def _get_power_flow_legacy(db: "DatabaseConnection", battery_systems: list) -> PowerFlowData: + """Get power flow data from legacy database.""" start = datetime.now(UTC) - timedelta(seconds=10) - grid_power = {} + grid_power: dict[str, float | None] = {} for i in (1, 2, 3): power = None result = db.get_power(f"grid/power/l{i}", start=start, bucket_seconds=None) @@ -98,7 +175,7 @@ async def get_power_flow(db: Database, battery_systems: BatterySystemsDep) -> Po if result: _, solar_power = result[-1] - batteries = {} + batteries: dict[str, BatteryPowerValues] = {} for battery_system in battery_systems: charger = 0 inverter = 0 @@ -211,6 +288,7 @@ class EnergyGraphResponse(BaseModel): @router.get("/energy-graph", response_model=EnergyGraphResponse) async def get_energy_flow_endpoint( db: Database, + timeseries: TimeseriesDep, battery_systems: BatterySystemsDep, battery_id: str | None = Query(default=None), start: datetime | None = Query(default=None), @@ -239,69 +317,163 @@ async def get_energy_flow_endpoint( if end is None: end = now - series = { - "grid_import": db.get_energy_aggregated( - "grid/energy/import/total", bucket_minutes * 60, start, end, center_buckets=True - ), - "grid_export": db.get_energy_aggregated( - "grid/energy/export/total", bucket_minutes * 60, start, end, center_buckets=True - ), - "vebus_228_import": db.get_energy_aggregated( - battery_system.config.metrics.energy_to_system, bucket_minutes * 60, start, end, center_buckets=True - ), - "vebus_228_export": db.get_energy_aggregated( - battery_system.config.metrics.energy_from_system, bucket_minutes * 60, start, end, center_buckets=True - ), - } - - timestamps = set() - series_as_dict: dict[str, dict[datetime, float]] = {name: {} for name in series} - for name, s in series.items(): - for ts, v in s: - timestamps.add(ts) - series_as_dict[name][ts] = v - timestamps = list(sorted(timestamps)) + if timeseries is not None: + return await _get_energy_graph_timeseries(timeseries, battery_system, start, end, bucket_minutes) - grid_exports = { - "From MP": [], - } - grid_imports = { - "Consumption": [], - "To MP": [], - } - battery_stats = BatteryEnergySeries() - for ts in timestamps: - # Grid export - from_mp = series_as_dict["vebus_228_export"].get(ts) - grid_exports["From MP"].append(from_mp) - unaccounted_export = series_as_dict["grid_export"].get(ts, 0) - (from_mp or 0) - - # Grid import - to_mp = series_as_dict["vebus_228_import"].get(ts) - grid_imports["To MP"].append(to_mp) - grid_import = series_as_dict["grid_import"].get(ts) - if grid_import is not None: - grid_import -= (to_mp or 0) - unaccounted_export - grid_imports["Consumption"].append(grid_import) - - # Battery stats - battery_stats.energy_to_charger.append(to_mp) - battery_stats.energy_from_inverter.append(from_mp) - - return EnergyGraphResponse( - timestamps=timestamps, - grid_export=grid_exports, - grid_import=grid_imports, - battery_systems={"MultiPlus": battery_stats}, - ) + return await _get_energy_graph_legacy(db, battery_system, start, end, bucket_minutes) except Exception as e: logger.exception("Failed to get energy flow") raise HTTPException(status_code=500, detail=str(e)) from e +async def _get_energy_graph_timeseries( + timeseries: "TimeseriesBackend", + battery_system, + start: datetime, + end: datetime, + bucket_minutes: int, +) -> EnergyGraphResponse: + """Get energy graph data from timeseries backend.""" + device = battery_system.id + step = f"{bucket_minutes}m" + + # Query energy series using increase() to get per-bucket energy consumption + queries = battery_system.config.queries + grid_import_query = queries.energy_grid_import.replace("$device", device) + grid_export_query = queries.energy_grid_export.replace("$device", device) + to_mp_query = queries.energy_to_battery.replace("$device", device) + from_mp_query = queries.energy_from_battery.replace("$device", device) + + # Use increase() to get energy delta per bucket + grid_import_result = timeseries.query_range(f"increase({grid_import_query}[{step}])", start, end, step) + grid_export_result = timeseries.query_range(f"increase({grid_export_query}[{step}])", start, end, step) + to_mp_result = timeseries.query_range(f"increase({to_mp_query}[{step}])", start, end, step) + from_mp_result = timeseries.query_range(f"increase({from_mp_query}[{step}])", start, end, step) + + # Convert to dict for easier lookup + def result_to_dict(result: "QueryResult") -> dict[datetime, float]: + if not result.series or not result.series[0].values: + return {} + return {ts: val for ts, val in result.series[0].values} + + grid_import_data = result_to_dict(grid_import_result) + grid_export_data = result_to_dict(grid_export_result) + to_mp_data = result_to_dict(to_mp_result) + from_mp_data = result_to_dict(from_mp_result) + + # Collect all timestamps + all_timestamps: set[datetime] = set() + all_timestamps.update(grid_import_data.keys()) + all_timestamps.update(grid_export_data.keys()) + all_timestamps.update(to_mp_data.keys()) + all_timestamps.update(from_mp_data.keys()) + timestamps = sorted(all_timestamps) + + # Build response series + grid_exports: dict[str, list[float | None]] = {"From MP": []} + grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} + battery_stats = BatteryEnergySeries() + + for ts in timestamps: + from_mp = from_mp_data.get(ts) + grid_exports["From MP"].append(round(from_mp, 3) if from_mp else None) + unaccounted_export = grid_export_data.get(ts, 0) - (from_mp or 0) + + to_mp = to_mp_data.get(ts) + grid_imports["To MP"].append(round(to_mp, 3) if to_mp else None) + grid_import = grid_import_data.get(ts) + if grid_import is not None: + grid_import -= (to_mp or 0) - unaccounted_export + grid_imports["Consumption"].append(round(grid_import, 3) if grid_import else None) + + battery_stats.energy_to_charger.append(round(to_mp, 3) if to_mp else None) + battery_stats.energy_from_inverter.append(round(from_mp, 3) if from_mp else None) + + return EnergyGraphResponse( + timestamps=timestamps, + grid_export=grid_exports, + grid_import=grid_imports, + battery_systems={battery_system.config.name: battery_stats}, + ) + + +async def _get_energy_graph_legacy( + db: "DatabaseConnection", + battery_system, + start: datetime, + end: datetime, + bucket_minutes: int, +) -> EnergyGraphResponse: + """Get energy graph data from legacy database.""" + series = { + "grid_import": db.get_energy_aggregated( + "grid/energy/import/total", bucket_minutes * 60, start, end, center_buckets=True + ), + "grid_export": db.get_energy_aggregated( + "grid/energy/export/total", bucket_minutes * 60, start, end, center_buckets=True + ), + "vebus_228_import": db.get_energy_aggregated( + battery_system.config.metrics.energy_to_system, bucket_minutes * 60, start, end, center_buckets=True + ), + "vebus_228_export": db.get_energy_aggregated( + battery_system.config.metrics.energy_from_system, bucket_minutes * 60, start, end, center_buckets=True + ), + } + + timestamps: set[datetime] = set() + series_as_dict: dict[str, dict[datetime, float]] = {name: {} for name in series} + for name, s in series.items(): + for ts, v in s: + timestamps.add(ts) + series_as_dict[name][ts] = v + sorted_timestamps = sorted(timestamps) + + grid_exports: dict[str, list[float | None]] = {"From MP": []} + grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} + battery_stats = BatteryEnergySeries() + + for ts in sorted_timestamps: + from_mp = series_as_dict["vebus_228_export"].get(ts) + grid_exports["From MP"].append(from_mp) + unaccounted_export = series_as_dict["grid_export"].get(ts, 0) - (from_mp or 0) + + to_mp = series_as_dict["vebus_228_import"].get(ts) + grid_imports["To MP"].append(to_mp) + grid_import = series_as_dict["grid_import"].get(ts) + if grid_import is not None: + grid_import -= (to_mp or 0) - unaccounted_export + grid_imports["Consumption"].append(grid_import) + + battery_stats.energy_to_charger.append(to_mp) + battery_stats.energy_from_inverter.append(from_mp) + + return EnergyGraphResponse( + timestamps=sorted_timestamps, + grid_export=grid_exports, + grid_import=grid_imports, + battery_systems={"MultiPlus": battery_stats}, + ) + + +def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: + """Calculate query step from aggregate_minutes or time range.""" + if aggregate_minutes > 1: + return f"{aggregate_minutes}m" + # Auto-calculate based on range + duration = (end - start).total_seconds() + if duration <= 3600: # 1 hour + return "1m" + if duration <= 6 * 3600: # 6 hours + return "5m" + if duration <= 24 * 3600: # 24 hours + return "15m" + return "1h" + + @router.get("/power-graph", response_model=PowerResponse) async def get_power_graph( db: Database, + timeseries: TimeseriesDep, battery_systems: BatterySystemsDep, battery_id: str | None = Query(default=None), start: datetime | None = Query(default=None), @@ -330,27 +502,60 @@ async def get_power_graph( if end is None: end = now - bucket_seconds = aggregate_minutes * 60 - - series = {f"Grid L{i}": db.get_power(f"grid/power/l{i}", start, end, bucket_seconds) for i in (1, 2, 3)} - - series["To MP"] = db.get_power(battery_system.config.metrics.power_to_system, start, end, bucket_seconds) + # Schedule always comes from database + schedule_series: list[tuple[datetime, float]] = [] + for ts_start, ts_end, v, _ in db.get_schedule(battery_system.config.id, start): + schedule_series.extend([(ts_start, v), (ts_end, v)]) + + if timeseries is not None: + device = battery_system.id + step = _calculate_step(start, end, aggregate_minutes) + + series: dict[str, TimeSeries] = {} + + # Grid power per phase + for phase in ("L1", "L2", "L3"): + query = f'openess_power_watts{{from="grid", phase="{phase}", device="{device}"}}' + result = timeseries.query_range(query, start, end, step) + series[f"Grid {phase}"] = query_result_to_timeseries(result) + + # AC power (to/from MultiPlus) + ac_query = battery_system.config.queries.power_ac_in.replace("$device", device) + ac_result = timeseries.query_range(ac_query, start, end, step) + series["To MP"] = query_result_to_timeseries(ac_result) + + # Battery DC power + battery_query = battery_system.config.queries.power_battery.replace("$device", device) + battery_result = timeseries.query_range(battery_query, start, end, step) + series["Battery"] = query_result_to_timeseries(battery_result) + + # Solar power (negated for display) + solar_query = battery_system.config.queries.power_pv.replace("$device", device) + solar_result = timeseries.query_range(solar_query, start, end, step) + solar_ts = query_result_to_timeseries(solar_result) + series["Solar"] = TimeSeries( + timestamps=solar_ts.timestamps, + values=[-v for v in solar_ts.values], + ) - series["Battery"] = db.get_power(battery_system.config.metrics.power_to_battery, start, end, bucket_seconds) + series["Schedule"] = data_to_timeseries(schedule_series) + return PowerResponse(series=series) - series["Solar"] = [ + # Legacy database queries + bucket_seconds = aggregate_minutes * 60 + legacy_series: dict[str, list[tuple[datetime, float]]] = { + f"Grid L{i}": db.get_power(f"grid/power/l{i}", start, end, bucket_seconds) for i in (1, 2, 3) + } + legacy_series["To MP"] = db.get_power(battery_system.config.metrics.power_to_system, start, end, bucket_seconds) + legacy_series["Battery"] = db.get_power( + battery_system.config.metrics.power_to_battery, start, end, bucket_seconds + ) + legacy_series["Solar"] = [ (t, -p) for t, p in db.get_power("victron/pvinverter/31/power/l1", start, end, bucket_seconds) ] + legacy_series["Schedule"] = schedule_series - series["Schedule"] = [] - - print(type(battery_system)) - print(type(battery_system.config)) - - for ts_start, ts_end, v, _ in db.get_schedule(battery_system.config.id, start): - series["Schedule"].extend([(ts_start, v), (ts_end, v)]) - - return PowerResponse(series={k: data_to_timeseries(v) for k, v in series.items()}) + return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) except Exception as e: logger.exception("Failed to get power data") raise HTTPException(status_code=500, detail=str(e)) from e @@ -420,6 +625,7 @@ class BatteryGraphResponse(BaseModel): @router.get("/battery-graph", response_model=dict[str, BatteryGraphResponse]) async def get_battery_graph( db: Database, + timeseries: TimeseriesDep, battery_systems: BatterySystemsDep, battery_id: str | None = Query(default=None), start: datetime | None = Query(default=None), @@ -437,15 +643,33 @@ async def get_battery_graph( if battery_id is not None and battery_system.config.id != battery_id: continue - soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) + # Schedule still comes from database (not in timeseries) scheduled = [(t, soc) for _, t, _, soc in db.get_schedule(battery_system.config.id, start)] - voltage = db.get_voltage(battery_system.config.metrics.battery_voltage, start, end, bucket_seconds=60) - result[battery_system.config.name] = BatteryGraphResponse( - soc=data_to_timeseries(soc, rounding=1), - schedule=data_to_timeseries(scheduled, rounding=1), - voltage=data_to_timeseries(voltage, rounding=2), - ) + # SOC and voltage from timeseries backend + if timeseries is not None: + device = battery_system.id + soc_query = battery_system.config.queries.soc.replace("$device", device) + voltage_query = battery_system.config.queries.voltage.replace("$device", device) + + soc_result = timeseries.query_range(soc_query, start, end, step="1m") + voltage_result = timeseries.query_range(voltage_query, start, end, step="1m") + + result[battery_system.config.name] = BatteryGraphResponse( + soc=query_result_to_timeseries(soc_result, rounding=1), + schedule=data_to_timeseries(scheduled, rounding=1), + voltage=query_result_to_timeseries(voltage_result, rounding=2), + ) + else: + # Fallback to legacy database queries + soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) + voltage = db.get_voltage(battery_system.config.metrics.battery_voltage, start, end, bucket_seconds=60) + + result[battery_system.config.name] = BatteryGraphResponse( + soc=data_to_timeseries(soc, rounding=1), + schedule=data_to_timeseries(scheduled, rounding=1), + voltage=data_to_timeseries(voltage, rounding=2), + ) return result except Exception as e: logger.exception("Failed to get battery SOC") @@ -470,57 +694,185 @@ class EfficiencyScatterPoint(BaseModel): @router.get("/efficiency-scatter", response_model=list[EfficiencyScatterPoint]) async def get_efficiency_scatter( db: Database, - limit: int = Query(default=2000), + timeseries: TimeseriesDep, + battery_systems: BatterySystemsDep, + battery_id: str | None = Query(default=None), + start: datetime | None = Query(default=None), + end: datetime | None = Query(default=None), aggregate_minutes: int = Query(default=10), idle_threshold: int = Query(default=5), ) -> list[EfficiencyScatterPoint]: try: - ac_in = db.get_power("victron/vebus/228/power/ac_in/l1", bucket_seconds=aggregate_minutes * 60, limit=limit) - ac_out = db.get_power("victron/vebus/228/power/ac_out/l1", bucket_seconds=aggregate_minutes * 60, limit=limit) - dc = db.get_power("victron/vebus/228/power/battery", bucket_seconds=aggregate_minutes * 60, limit=limit) - # dc = db.get_power("victron/battery/225/power/battery", bucket_seconds=aggregate_minutes * 60, limit=limit) - - data = {ts: [v_in - v_out, None] for (ts, v_in), (_, v_out) in zip(ac_in, ac_out, strict=False)} - for ts, v in dc: - if ts in data: - data[ts][1] = v - - points = [] - for ts, (ac, dc) in data.items(): - if abs(dc) < idle_threshold: - category = "idling" - # elif dc > 0 and soc == 100 and abs(ac) < balancing_threshold: - # category = "balancing" - elif dc > 0: - category = "charging" + battery_system = None + if battery_id: + for bs in battery_systems: + if bs.id == battery_id: + battery_system = bs + break + elif len(battery_systems) == 1: + battery_system = battery_systems[0] + + if battery_system is None: + if battery_id: + raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") else: - category = "discharging" - - losses = ac - dc - efficiency = None - if category == "charging" and ac > 0: - efficiency = (dc / ac) * 100 - elif category == "discharging" and dc < 0: - efficiency = (ac / dc) * 100 - - points.append( - EfficiencyScatterPoint( - time=ts, - battery_power=round(abs(dc), 1), - inverter_charger_power=round(ac, 1), - losses=round(losses, 1), - efficiency=round(efficiency, 1) if efficiency is not None else None, - soc=None, - category=category, - ) + raise HTTPException(status_code=400, detail="Please provide a battery_id") + + now = datetime.now(UTC) + if start is None: + start = now - timedelta(days=7) + if end is None: + end = now + + if timeseries is not None: + return await _get_efficiency_scatter_timeseries( + timeseries, battery_system, start, end, aggregate_minutes, idle_threshold ) - return points + return await _get_efficiency_scatter_legacy(db, battery_system, start, end, aggregate_minutes, idle_threshold) except Exception as e: logger.exception("Failed to get efficiency scatter data") raise HTTPException(status_code=500, detail=str(e)) from e +async def _get_efficiency_scatter_timeseries( + timeseries: "TimeseriesBackend", + battery_system, + start: datetime, + end: datetime, + aggregate_minutes: int, + idle_threshold: int, +) -> list[EfficiencyScatterPoint]: + """Get efficiency scatter data from timeseries backend.""" + device = battery_system.id + step = f"{aggregate_minutes}m" + queries = battery_system.config.queries + + # Query AC in, AC out, and battery DC power + ac_in_query = queries.power_ac_in.replace("$device", device) + ac_out_query = queries.power_ac_out.replace("$device", device) + dc_query = queries.power_battery.replace("$device", device) + + ac_in_result = timeseries.query_range(ac_in_query, start, end, step) + ac_out_result = timeseries.query_range(ac_out_query, start, end, step) + dc_result = timeseries.query_range(dc_query, start, end, step) + + # Convert to dicts + def result_to_dict(result: "QueryResult") -> dict[datetime, float]: + if not result.series or not result.series[0].values: + return {} + return {ts: val for ts, val in result.series[0].values} + + ac_in_data = result_to_dict(ac_in_result) + ac_out_data = result_to_dict(ac_out_result) + dc_data = result_to_dict(dc_result) + + # Merge data by timestamp + all_timestamps = set(ac_in_data.keys()) & set(ac_out_data.keys()) & set(dc_data.keys()) + + points = [] + for ts in sorted(all_timestamps): + ac = ac_in_data[ts] - ac_out_data[ts] + dc = dc_data[ts] + + if abs(dc) < idle_threshold: + category = "idling" + elif dc > 0: + category = "charging" + else: + category = "discharging" + + losses = ac - dc + efficiency = None + if category == "charging" and ac > 0: + efficiency = (dc / ac) * 100 + elif category == "discharging" and dc < 0: + efficiency = (ac / dc) * 100 + + points.append( + EfficiencyScatterPoint( + time=ts, + battery_power=round(abs(dc), 1), + inverter_charger_power=round(ac, 1), + losses=round(losses, 1), + efficiency=round(efficiency, 1) if efficiency is not None else None, + soc=None, + category=category, + ) + ) + + return points + + +async def _get_efficiency_scatter_legacy( + db: "DatabaseConnection", + battery_system, + start: datetime, + end: datetime, + aggregate_minutes: int, + idle_threshold: int, +) -> list[EfficiencyScatterPoint]: + """Get efficiency scatter data from legacy database.""" + metrics = battery_system.config.metrics + bucket_seconds = aggregate_minutes * 60 + + # Use configured metrics paths + ac_in_path = metrics.power_to_system + if isinstance(ac_in_path, list): + ac_in_path = ac_in_path[0] + + # AC out is typically the same vebus but ac_out instead of ac_in + ac_out_path = ac_in_path.replace("ac_in", "ac_out") if ac_in_path else None + + dc_path = metrics.power_to_battery + if isinstance(dc_path, list): + dc_path = dc_path[0] + + ac_in = db.get_power(ac_in_path, start, end, bucket_seconds=bucket_seconds) if ac_in_path else [] + ac_out = db.get_power(ac_out_path, start, end, bucket_seconds=bucket_seconds) if ac_out_path else [] + dc = db.get_power(dc_path, start, end, bucket_seconds=bucket_seconds) if dc_path else [] + + data: dict[datetime, list[float | None]] = { + ts: [v_in - v_out, None] for (ts, v_in), (_, v_out) in zip(ac_in, ac_out, strict=False) + } + for ts, v in dc: + if ts in data: + data[ts][1] = v + + points = [] + for ts, (ac, dc_val) in data.items(): + if ac is None or dc_val is None: + continue + + if abs(dc_val) < idle_threshold: + category = "idling" + elif dc_val > 0: + category = "charging" + else: + category = "discharging" + + losses = ac - dc_val + efficiency = None + if category == "charging" and ac > 0: + efficiency = (dc_val / ac) * 100 + elif category == "discharging" and dc_val < 0: + efficiency = (ac / dc_val) * 100 + + points.append( + EfficiencyScatterPoint( + time=ts, + battery_power=round(abs(dc_val), 1), + inverter_charger_power=round(ac, 1), + losses=round(losses, 1), + efficiency=round(efficiency, 1) if efficiency is not None else None, + soc=None, + category=category, + ) + ) + + return points + + class BatteryCycle(BaseModel): start_time: datetime end_time: datetime @@ -541,6 +893,7 @@ class BatteryCycle(BaseModel): @router.get("/cycles", response_model=list[BatteryCycle]) async def get_battery_cycles( db: Database, + timeseries: TimeseriesDep, battery_systems: BatterySystemsDep, price_config: PriceConfigDep, battery_id: str | None = Query(default=None), @@ -570,7 +923,15 @@ async def get_battery_cycles( if end is None: end = now - battery_soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) + # Get SOC data from timeseries or legacy database + if timeseries is not None: + device = battery_system.id + soc_query = battery_system.config.queries.soc.replace("$device", device) + soc_result = timeseries.query_range(soc_query, start, end, step="1m") + battery_soc = [(ts, val) for ts, val in soc_result.series[0].values] if soc_result.series else [] + else: + battery_soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) + raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) cycles = [] @@ -667,6 +1028,8 @@ async def get_battery_cycles( @router.get("/power", response_model=PowerResponse) async def get_power( db: Database, + timeseries: TimeseriesDep, + battery_systems: BatterySystemsDep, start: datetime | None = Query(default=None), end: datetime | None = Query(default=None), aggregate_minutes: int = Query(default=1), @@ -677,8 +1040,40 @@ async def get_power( start = now - timedelta(hours=24) if end is None: end = now - series = db.get_all_power(start, end, aggregate_minutes * 60) - return PowerResponse(series={k: data_to_timeseries(v) for k, v in series.items()}) + + if timeseries is not None and battery_systems: + # Query all power metrics from timeseries + step = _calculate_step(start, end, aggregate_minutes) + series: dict[str, TimeSeries] = {} + + for battery_system in battery_systems: + device = battery_system.id + queries = battery_system.config.queries + prefix = battery_system.config.name or device + + # Query each power metric + power_queries = { + "Grid": queries.power_grid_total, + "Grid L1": f'openess_power_watts{{from="grid", phase="L1", device="{device}"}}', + "Grid L2": f'openess_power_watts{{from="grid", phase="L2", device="{device}"}}', + "Grid L3": f'openess_power_watts{{from="grid", phase="L3", device="{device}"}}', + "PV": queries.power_pv, + "Battery": queries.power_battery, + "AC In": queries.power_ac_in, + "AC Out": queries.power_ac_out, + } + + for name, query in power_queries.items(): + resolved_query = query.replace("$device", device) + result = timeseries.query_range(resolved_query, start, end, step) + label = f"{prefix}/{name}" if len(battery_systems) > 1 else name + series[label] = query_result_to_timeseries(result) + + return PowerResponse(series=series) + + # Legacy database fallback + legacy_series = db.get_all_power(start, end, aggregate_minutes * 60) + return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) except Exception as e: logger.exception("Failed to get debug power flows") raise HTTPException(status_code=500, detail=str(e)) from e @@ -689,8 +1084,11 @@ async def get_power( @router.get("/energy", response_model=EnergyResponse) async def get_energy( db: Database, + timeseries: TimeseriesDep, + battery_systems: BatterySystemsDep, start: datetime | None = Query(default=None), end: datetime | None = Query(default=None), + bucket_minutes: int = Query(default=60), ) -> EnergyResponse: try: now = datetime.now(UTC) @@ -699,14 +1097,66 @@ async def get_energy( if end is None: end = now - # Get counter-based energy flows - series = db.get_all_energy(start, end, normalize=True) + if timeseries is not None and battery_systems: + # Query all energy metrics from timeseries + step = f"{bucket_minutes}m" + series: dict[str, TimeSeries] = {} + + for battery_system in battery_systems: + device = battery_system.id + queries = battery_system.config.queries + prefix = battery_system.config.name or device + + energy_queries = { + "Grid Import": queries.energy_grid_import, + "Grid Export": queries.energy_grid_export, + "To Battery": queries.energy_to_battery, + "From Battery": queries.energy_from_battery, + } + + for name, query in energy_queries.items(): + resolved_query = query.replace("$device", device) + # Use increase() to get energy delta per bucket + result = timeseries.query_range(f"increase({resolved_query}[{step}])", start, end, step) + label = f"{prefix}/{name}" if len(battery_systems) > 1 else name + series[label] = query_result_to_timeseries(result, rounding=3) + + return EnergyResponse(series=series) + + # Legacy database fallback + legacy_series = db.get_all_energy(start, end, normalize=True) # Get integrated power flows for label in db.get_power_labels(start, end): - series[f"{label} [integrated]"] = db.integrate_power(label, start, end) + legacy_series[f"{label} [integrated]"] = db.integrate_power(label, start, end) - return EnergyResponse(series={k: data_to_timeseries(v) for k, v in series.items()}) + return EnergyResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) except Exception as e: logger.exception("Failed to get debug energy flows") raise HTTPException(status_code=500, detail=str(e)) from e + + +# -------------------------- # +# Timeseries query helpers # +# -------------------------- # + + +@router.get("/queries/{battery_id}") +async def get_queries( + battery_id: str, + battery_systems: BatterySystemsDep, +) -> dict[str, str]: + """Return resolved MetricsQL queries for a battery system. + + The queries are templated in BatterySystemConfig.queries and resolved + with the device serial number. Frontend can use these to query the + timeseries backend directly via /api/v1/query_range. + """ + battery = next((b for b in battery_systems if b.id == battery_id), None) + if not battery: + raise HTTPException(404, f"Battery system {battery_id} not found") + + device = battery.id + queries = battery.config.queries + + return {field: getattr(queries, field).replace("$device", device) for field in queries.model_fields} diff --git a/open_ess/frontend/routes/timeseries.py b/open_ess/frontend/routes/timeseries.py new file mode 100644 index 0000000..4dfa23a --- /dev/null +++ b/open_ess/frontend/routes/timeseries.py @@ -0,0 +1,85 @@ +"""Timeseries query proxy routes. + +These routes proxy queries to the timeseries backend (VictoriaMetrics). +For MetricSQLite, the native metricsqlite.fastapi routes are used instead. +""" + +from datetime import UTC, datetime + +from fastapi import APIRouter, HTTPException + +from ..dependencies import TimeseriesDep + +router = APIRouter() + + +def _parse_timestamp(value: float | str) -> datetime: + """Parse a timestamp from float (unix seconds) or ISO string.""" + if isinstance(value, str): + # Try parsing as ISO format + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + # Try parsing as float string + return datetime.fromtimestamp(float(value), tz=UTC) + return datetime.fromtimestamp(value, tz=UTC) + + +@router.get("/query") +async def query( + timeseries: TimeseriesDep, + query: str, + time: float | str | None = None, +) -> dict: + """Execute an instant query against the timeseries backend.""" + if timeseries is None: + raise HTTPException(503, "Timeseries backend not configured") + + eval_time = _parse_timestamp(time) if time is not None else None + result = timeseries.query(query, eval_time) + + return { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": series.metric, + "value": [series.values[0][0].timestamp(), series.values[0][1]] if series.values else None, + } + for series in result.series + ], + }, + } + + +@router.get("/query_range") +async def query_range( + timeseries: TimeseriesDep, + query: str, + start: float | str, + end: float | str, + step: str = "1m", +) -> dict: + """Execute a range query against the timeseries backend.""" + if timeseries is None: + raise HTTPException(503, "Timeseries backend not configured") + + start_time = _parse_timestamp(start) + end_time = _parse_timestamp(end) + + result = timeseries.query_range(query, start_time, end_time, step) + + return { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": series.metric, + "values": [[v[0].timestamp(), str(v[1])] for v in series.values], + } + for series in result.series + ], + }, + } diff --git a/open_ess/frontend/routes/util.py b/open_ess/frontend/routes/util.py index 204e038..6b242db 100644 --- a/open_ess/frontend/routes/util.py +++ b/open_ess/frontend/routes/util.py @@ -1,14 +1,39 @@ from collections.abc import Iterable from datetime import datetime +from typing import TYPE_CHECKING from pydantic import BaseModel +if TYPE_CHECKING: + from open_ess.timeseries import QueryResult + class TimeSeries(BaseModel): timestamps: list[datetime] values: list[float] +def query_result_to_timeseries(result: "QueryResult", rounding: int | None = None) -> TimeSeries: + """Convert a timeseries QueryResult to TimeSeries format. + + If multiple series are returned, they are merged (assumes same timestamps). + """ + timestamps: list[datetime] = [] + values: list[float] = [] + + # Take the first series (or merge if needed) + if result.series: + series = result.series[0] + for ts, val in series.values: + timestamps.append(ts) + if rounding is not None: + values.append(round(val, rounding)) + else: + values.append(val) + + return TimeSeries(timestamps=timestamps, values=values) + + def data_to_timeseries(data: Iterable[tuple[datetime, float]], rounding: int | None = None) -> TimeSeries: timestamps = [] values = [] diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 5667b61..4b18def 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -293,7 +293,10 @@ } } - async function loadSocChart(elementId, start, end) { + // Toggle: set to true to use new Timeseries API, false for legacy API + var USE_TIMESERIES_FOR_SOC = false; + + async function loadSocChartLegacy(elementId, start, end) { Utils.showLoading(elementId); try { @@ -362,6 +365,78 @@ } } + /** + * Load SOC chart using the new Timeseries API. + * This queries the timeseries backend directly via /api/v1/query_range. + */ + async function loadSocChartTimeseries(elementId, start, end, batteryId) { + Utils.showLoading(elementId); + + try { + // Initialize Timeseries with battery system ID + await Timeseries.init(batteryId); + + // Query SOC and voltage in parallel + var [socResult, voltageResult] = await Promise.all([ + Timeseries.queryRange('soc', start, end), + Timeseries.queryRange('voltage', start, end), + ]); + + var traces = []; + + // Convert SOC result to Plotly trace + var socTraces = Timeseries.toPlotlyTraces(socResult, { name: 'SoC' }); + socTraces.forEach(function(trace) { + trace.line = { color: '#3498db', width: 2 }; + trace.hovertemplate = '%{y:.1f}%SoC'; + traces.push(trace); + }); + + // Convert voltage result to Plotly trace (on secondary y-axis) + var voltTraces = Timeseries.toPlotlyTraces(voltageResult, { name: 'Voltage' }); + voltTraces.forEach(function(trace) { + trace.line = { color: '#ff7171', width: 2 }; + trace.hovertemplate = '%{y:.1f}VVoltage'; + trace.yaxis = 'y2'; + traces.push(trace); + }); + + var layout = Utils.getDefaultLayout(); + Utils.layoutSetXRange(layout, start, end); + Utils.layoutAddNowLine(layout, start, end); + layout.yaxis = layout.yaxis || {}; + layout.yaxis.side = 'left'; + layout.yaxis.range = [0, 100]; + layout.yaxis.title = { text: "SoC (%)" }; + layout.yaxis2 = { + overlaying: 'y', + side: 'right', + gridcolor: 'transparent', + title: { text: "Voltage (V)" }, + }; + Utils.makePlot(elementId, traces, layout); + } catch (error) { + console.error('Error loading SoC data via Timeseries:', error); + Utils.showError(elementId, 'Failed to load SoC data'); + } + } + + async function loadSocChart(elementId, start, end) { + if (USE_TIMESERIES_FOR_SOC) { + // Get battery system ID from system layout + var layout = await Api.systemLayout(); + var batteryId = layout.battery_systems && layout.battery_systems[0] + ? layout.battery_systems[0].id + : null; + + if (batteryId) { + return loadSocChartTimeseries(elementId, start, end, batteryId); + } + console.warn('No battery system found, falling back to legacy API'); + } + return loadSocChartLegacy(elementId, start, end); + } + async function loadAndCacheEnergyData(start, end, bucketMinutes) { try { cachedEnergyData = await Api.energyGraph({ diff --git a/open_ess/frontend/static/timeseries.js b/open_ess/frontend/static/timeseries.js new file mode 100644 index 0000000..58d2b16 --- /dev/null +++ b/open_ess/frontend/static/timeseries.js @@ -0,0 +1,201 @@ +/** + * Timeseries query helper for frontend visualization. + * + * This module provides a unified interface for querying the timeseries backend + * (VictoriaMetrics or MetricSQLite) using MetricsQL queries defined in the + * BatterySystemConfig. + * + * Usage: + * await Timeseries.init(batteryId); + * var result = await Timeseries.queryRange('power_grid', start, end); + * var traces = Timeseries.toPlotlyTraces(result, { name: 'Grid Power' }); + */ +var Timeseries = (function () { + /** @type {Object} */ + var queries = {}; + + /** @type {string|null} */ + var currentBatteryId = null; + + /** + * Initialize the timeseries helper for a battery system. + * Fetches the resolved MetricsQL queries from the backend. + * + * @param {string} batteryId - The battery system ID + * @returns {Promise} + */ + async function init(batteryId) { + if (currentBatteryId === batteryId && Object.keys(queries).length > 0) { + return; // Already initialized for this battery + } + + var response = await fetch("/api/queries/" + encodeURIComponent(batteryId)); + if (!response.ok) { + throw new Error("Failed to fetch queries: HTTP " + response.status); + } + queries = await response.json(); + currentBatteryId = batteryId; + } + + /** + * Calculate appropriate step based on time range. + * + * @param {Date} start - Start time + * @param {Date} end - End time + * @returns {string} Step string (e.g., "1m", "5m", "1h") + */ + function calculateStep(start, end) { + var durationMs = end.getTime() - start.getTime(); + var hour = 3600000; + + if (durationMs < hour) return "1m"; + if (durationMs < 6 * hour) return "5m"; + if (durationMs < 24 * hour) return "15m"; + if (durationMs < 7 * 24 * hour) return "1h"; + return "6h"; + } + + /** + * Execute a range query against the timeseries backend. + * + * @param {string} queryName - Name of the query (e.g., "power_grid", "soc") + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} [step] - Optional step override (e.g., "5m") + * @returns {Promise} Query result in Prometheus/VM format + */ + async function queryRange(queryName, start, end, step) { + var query = queries[queryName]; + if (!query) { + throw new Error("Unknown query: " + queryName + ". Available: " + Object.keys(queries).join(", ")); + } + + var params = new URLSearchParams({ + query: query, + start: (start.getTime() / 1000).toString(), + end: (end.getTime() / 1000).toString(), + step: step || calculateStep(start, end), + }); + + var response = await fetch("/api/v1/query_range?" + params); + if (!response.ok) { + throw new Error("Query failed: HTTP " + response.status); + } + return response.json(); + } + + /** + * Execute a raw MetricsQL query (not from the predefined queries). + * + * @param {string} query - MetricsQL query string + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} [step] - Optional step override + * @returns {Promise} Query result + */ + async function queryRangeRaw(query, start, end, step) { + var params = new URLSearchParams({ + query: query, + start: (start.getTime() / 1000).toString(), + end: (end.getTime() / 1000).toString(), + step: step || calculateStep(start, end), + }); + + var response = await fetch("/api/v1/query_range?" + params); + if (!response.ok) { + throw new Error("Query failed: HTTP " + response.status); + } + return response.json(); + } + + /** + * Format metric labels into a readable string. + * + * @param {Object} metric - Metric labels object + * @returns {string} Formatted label string + */ + function formatLabels(metric) { + var name = metric.__name__ || "unknown"; + var labels = Object.entries(metric) + .filter(function (e) { + return e[0] !== "__name__"; + }) + .map(function (e) { + return e[0] + "=" + e[1]; + }) + .join(", "); + return labels ? name + "{" + labels + "}" : name; + } + + /** + * Convert a query result to Plotly traces. + * + * @param {Object} result - Query result from queryRange + * @param {Object} [options] - Options + * @param {string} [options.name] - Override trace name (for single series) + * @param {function} [options.nameFormatter] - Function to format series name from labels + * @returns {Array} Array of Plotly trace objects + */ + function toPlotlyTraces(result, options) { + options = options || {}; + + if (!result || !result.data || !result.data.result) { + console.warn("Invalid query result:", result); + return []; + } + + return result.data.result.map(function (series, index) { + var name; + if (options.name && result.data.result.length === 1) { + name = options.name; + } else if (options.nameFormatter) { + name = options.nameFormatter(series.metric); + } else { + name = formatLabels(series.metric); + } + + return { + x: series.values.map(function (v) { + return new Date(v[0] * 1000); + }), + y: series.values.map(function (v) { + return parseFloat(v[1]); + }), + type: "scatter", + mode: "lines", + name: name, + line: { width: 1.5 }, + }; + }); + } + + /** + * Get the list of available query names. + * + * @returns {string[]} Array of query names + */ + function getQueryNames() { + return Object.keys(queries); + } + + /** + * Get a specific query string. + * + * @param {string} name - Query name + * @returns {string|undefined} The query string or undefined + */ + function getQuery(name) { + return queries[name]; + } + + return { + init: init, + queryRange: queryRange, + queryRangeRaw: queryRangeRaw, + toPlotlyTraces: toPlotlyTraces, + calculateStep: calculateStep, + formatLabels: formatLabels, + getQueryNames: getQueryNames, + getQuery: getQuery, + }; +})(); diff --git a/open_ess/frontend/templates/base.html b/open_ess/frontend/templates/base.html index c435c8a..1264520 100644 --- a/open_ess/frontend/templates/base.html +++ b/open_ess/frontend/templates/base.html @@ -36,6 +36,7 @@ + {% block scripts %}{% endblock %} diff --git a/open_ess/victron_modbus/config.py b/open_ess/victron_modbus/config.py index 23d6334..f66d7a0 100644 --- a/open_ess/victron_modbus/config.py +++ b/open_ess/victron_modbus/config.py @@ -18,11 +18,3 @@ class VictronConfig(BaseModel): disable_charger_when_idle: bool = False disable_inverter_when_idle: bool = False - - @property - def vebus_prefix(self) -> str: - return f"victron/vebus/{self.vebus_id}" - - @property - def battery_prefix(self) -> str | None: - return f"victron/battery/{self.battery_id}" if self.battery_id else None From e602a5098619d23a1d1adb22f4ae9b6fead2b9f9 Mon Sep 17 00:00:00 2001 From: david Date: Mon, 4 May 2026 22:53:39 +0200 Subject: [PATCH 05/18] delete database --- open_ess/battery_system/config.py | 55 +- open_ess/config.py | 2 - open_ess/database/__init__.py | 13 - open_ess/database/config.py | 13 - open_ess/database/database.py | 518 ---- open_ess/database/migration_runner.py | 68 - open_ess/database/migrations/001_initial.py | 202 -- .../database/migrations/002_rename_labels.py | 33 - open_ess/database/migrations/__init__.py | 1 - open_ess/database/service.py | 41 - open_ess/database/util.py | 29 - open_ess/frontend/app.py | 3 - open_ess/frontend/dependencies.py | 6 - open_ess/frontend/routes/api.py | 2250 ++++++++--------- open_ess/main.py | 71 +- open_ess/optimizer/optimizer.py | 6 +- open_ess/optimizer/service.py | 12 +- open_ess/pricing/client.py | 66 +- open_ess/pricing/service.py | 14 +- open_ess/timeseries/__init__.py | 10 +- open_ess/timeseries/config.py | 2 - open_ess/timeseries/metricsqlite/backend.py | 15 +- open_ess/timeseries/metricsqlite/config.py | 9 +- open_ess/timeseries/victoriametrics/config.py | 4 - open_ess/victron_modbus/client.py | 10 +- open_ess/victron_modbus/service.py | 4 +- 26 files changed, 1247 insertions(+), 2210 deletions(-) delete mode 100644 open_ess/database/__init__.py delete mode 100644 open_ess/database/config.py delete mode 100644 open_ess/database/database.py delete mode 100644 open_ess/database/migration_runner.py delete mode 100644 open_ess/database/migrations/001_initial.py delete mode 100644 open_ess/database/migrations/002_rename_labels.py delete mode 100644 open_ess/database/migrations/__init__.py delete mode 100644 open_ess/database/service.py delete mode 100644 open_ess/database/util.py diff --git a/open_ess/battery_system/config.py b/open_ess/battery_system/config.py index ea7e7be..6b508d1 100644 --- a/open_ess/battery_system/config.py +++ b/open_ess/battery_system/config.py @@ -10,38 +10,44 @@ class MqttControl(BaseModel): topic: str -class MetricsConfig(BaseModel): - battery_soc: str | list[str] | None = None - battery_voltage: str | list[str] | None = None - power_to_system: str | list[str] | None = None - power_to_battery: str | list[str] | None = None - energy_to_system: str | list[str] | None = None - energy_from_system: str | list[str] | None = None - energy_to_battery: str | list[str] | None = None - energy_from_battery: str | list[str] | None = None - - class QueriesConfig(BaseModel): # Battery state - soc: str = 'openess_soc_ratio{node="battery", device="$device"} * 100' - voltage: str = 'openess_voltage_volts{node="battery", device="$device"}' + # Readings from battery (BMS) have priority over readings from vebus (MultiPlus). + soc: str = """ + ( + openess_soc_ratio{device=~"$device", node="battery", unit="battery"} + or + openess_soc_ratio{device=~"$device", node="battery", unit="vebus"} + ) * 100 + """ # + voltage: str = """ + ( + openess_voltage_volts{device=~"$device", node="battery", unit="battery"} + or + openess_voltage_volts{device=~"$device", node="battery", unit="vebus"} + ) * 100 + """ # Power - power_grid: str = 'openess_power_watts{from="grid", device="$device"}' - power_pv: str = 'openess_power_watts{from="pvinverter", device="$device"}' - power_battery: str = 'openess_power_watts{from="system", to="battery", device="$device"}' - power_ac_in: str = 'openess_power_watts{from="ac_in", device="$device"}' - power_ac_out: str = 'openess_power_watts{from="ac_out", device="$device"}' + system_power: str = """ + openess_power_watts{device="$device", phase=~"$phase", from="ac_in", to="system"} + - + openess_power_watts{device="$device", phase=~"$phase", from="system", to="ac_out"} + """ + battery_power: str = """ + openess_power_watts{device="$device", from="system", to="battery", unit="battery"} + or + openess_power_watts{device="$device", from="system", to="battery", unit="vebus"} + """ - # Energy - energy_grid_import: str = 'openess_energy_kwh{from="grid", device="$device"}' - energy_grid_export: str = 'openess_energy_kwh{to="grid", device="$device"}' - energy_to_battery: str = 'openess_energy_kwh{to="system", device="$device"}' - energy_from_battery: str = 'openess_energy_kwh{from="system", device="$device"}' + # energy_to_system: str | list[str] | None = None + # energy_from_system: str | list[str] | None = None + # energy_to_battery: str | list[str] | None = None + # energy_from_battery: str | list[str] | None = None class BatterySystemConfig(BaseModel): - name: str | None = None # Is set to self.id if not provided. + name: str | None = None monitor_only: bool = False phases: int = 1 capacity_kwh: float | None = None @@ -52,7 +58,6 @@ class BatterySystemConfig(BaseModel): max_soc: int = 100 control: Annotated[VictronConfig | MqttControl, Field(discriminator="type")] - metrics: MetricsConfig = MetricsConfig() queries: QueriesConfig = QueriesConfig() @property diff --git a/open_ess/config.py b/open_ess/config.py index 95a050e..1fbd0f8 100644 --- a/open_ess/config.py +++ b/open_ess/config.py @@ -4,7 +4,6 @@ from pydantic import BaseModel from open_ess.battery_system import BatterySystemConfig -from open_ess.database import DatabaseConfig from open_ess.frontend import FrontendConfig from open_ess.pricing import PriceConfig from open_ess.timeseries import TimeseriesConfig @@ -14,7 +13,6 @@ class Config(BaseModel): - database: DatabaseConfig = DatabaseConfig() frontend: FrontendConfig prices: PriceConfig timeseries: TimeseriesConfig = MetricSQLiteConfig() diff --git a/open_ess/database/__init__.py b/open_ess/database/__init__.py deleted file mode 100644 index 1947c7a..0000000 --- a/open_ess/database/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .config import DatabaseConfig -from .database import Database, DatabaseConnection -from .service import DatabaseService -from .util import dt_to_ms, ms_to_dt - -__all__ = [ - "Database", - "DatabaseConnection", - "DatabaseConfig", - "DatabaseService", - "ms_to_dt", - "dt_to_ms", -] diff --git a/open_ess/database/config.py b/open_ess/database/config.py deleted file mode 100644 index 8fe81d2..0000000 --- a/open_ess/database/config.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path - -from pydantic import BaseModel - - -class DatabaseCompressionConfig(BaseModel): - enable: bool = True - bucket_seconds: int = 60 - - -class DatabaseConfig(BaseModel): - path: Path = Path("./data.db") - compression: DatabaseCompressionConfig = DatabaseCompressionConfig() diff --git a/open_ess/database/database.py b/open_ess/database/database.py deleted file mode 100644 index 146795f..0000000 --- a/open_ess/database/database.py +++ /dev/null @@ -1,518 +0,0 @@ -import logging -import sqlite3 -from datetime import datetime, timedelta -from pathlib import Path - -from .config import DatabaseConfig -from .migration_runner import run_migrations -from .util import base_conditions, dt_to_ms, ms_to_dt - -logger = logging.getLogger(__name__) - - -class Database: - def __init__(self, config: DatabaseConfig): - self._config = config - config.path.parent.mkdir(parents=True, exist_ok=True) - with self.connect() as conn: - conn.execute("PRAGMA journal_mode=WAL") - # ^ WAL mode allows concurrent reads/writes without blocking - conn.execute("PRAGMA busy_timeout = 30000") - # ^ Wait up to 30 seconds for locks instead of failing immediately - - @property - def config(self) -> DatabaseConfig: - return self._config - - def connect(self) -> "DatabaseConnection": - return DatabaseConnection(self._config.path) - - def run_migrations(self) -> None: - with self.connect() as conn: - run_migrations(conn) - - -class DatabaseConnection: - def __init__(self, path: Path): - self._conn = sqlite3.connect(path) - self._conn.row_factory = sqlite3.Row - # ^ Makes column access by name possible - - def close(self) -> None: - self._conn.close() - - def __enter__(self) -> "DatabaseConnection": - return self - - def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: - self.close() - - def execute(self, sql: str, parameters: list | tuple | None = None) -> sqlite3.Cursor: - if parameters is None: - parameters = [] - return self._conn.execute(sql, parameters) - - def commit(self) -> None: - self._conn.commit() - - def vacuum(self) -> None: - self._conn.execute("PRAGMA incremental_vacuum") - - def _get_labels( - self, table_name: str, timestamp_name: str, start: datetime | None = None, end: datetime | None = None - ) -> list[str]: - conditions = [] - params = [] - if start is not None: - conditions.append(f"{timestamp_name} >= ?") - params.append(dt_to_ms(start)) - if end is not None: - conditions.append(f"{timestamp_name} < ?") - params.append(dt_to_ms(end)) - - where_clause = "WHERE " + " AND ".join(conditions) if conditions else "" - query = f""" - SELECT DISTINCT label - FROM {table_name} - {where_clause} - """ - cursor = self._conn.execute(query, params) - return [row[0] for row in cursor.fetchall()] - - # ------------------------------------------------------------------------- - # Power - # ------------------------------------------------------------------------- - - def insert_power(self, label: str, timestamp: datetime, power: float | None) -> None: - if power is None: - return - self._conn.execute( - "INSERT INTO power (label, start_time, sample_count, value) VALUES (?, ?, 1, ?)", - (label, dt_to_ms(timestamp), power), - ) - self._conn.commit() - - def get_power( - self, - label: str, - start: datetime | None = None, - end: datetime | None = None, - bucket_seconds: float | None = 60, - limit: int | None = None, - ) -> list[tuple[datetime, float]]: - if isinstance(label, list): - label = label[0] - - conditions, params = base_conditions(label, start, end, timestamp_name="start_time") - - if bucket_seconds is not None: - bucket_ms = round(bucket_seconds * 1000) - select_clause = "(start_time / ?) * ? as bucket, AVG(value) as avg_value" - params = [bucket_ms, bucket_ms, *params] - group_by = "GROUP BY bucket" - order_by = "bucket" - else: - select_clause = "start_time, value" - group_by = "" - order_by = "start_time" - - limit_clause = "" - if limit: - limit_clause = "LIMIT ?" - params.append(limit) - - where_clause = " AND ".join(conditions) - query = f""" - SELECT {select_clause} - FROM power - WHERE {where_clause} - {group_by} - ORDER BY {order_by} - {limit_clause} - """ - cursor = self._conn.execute(query, params) - return [(ms_to_dt(row[0]), row[1]) for row in cursor.fetchall()] - - def get_power_labels(self, start: datetime | None = None, end: datetime | None = None) -> list[str]: - return self._get_labels("power", "start_time", start, end) - - def get_all_power( - self, start: datetime, end: datetime | None = None, bucket_seconds: float | None = None - ) -> dict[str, list[tuple[datetime, float]]]: - power_series = {} - for label in self.get_power_labels(start, end): - power_series[label] = self.get_power(label, start, end, bucket_seconds) - return power_series - - def compress_power(self, older_than: datetime, bucket_seconds: float) -> tuple[int, int]: - older_than = older_than.replace(second=0, microsecond=0) - bucket_ms = round(bucket_seconds * 1000) - cutoff_ms = dt_to_ms(older_than) - # ^ cutoff alignment with bucket size enforces that we never cross bucket boundaries. This - # makes calculating average power much easier since we don't need to work with weighted - # averages and take duration of semi-bucket into account. - - bucket_query = """ - SELECT - label_id, - (start_time / ?) * ? AS bucket, - SUM(sample_count) AS total_samples, - COUNT(*) AS row_count - FROM _power - WHERE start_time < ? - GROUP BY label_id, bucket - HAVING row_count > 1 - ORDER BY bucket - """ - cursor = self._conn.execute(bucket_query, (bucket_ms, bucket_ms, cutoff_ms)) - buckets = cursor.fetchall() - - total_sample_count = 0 - total_bucket_count = 0 - for row in buckets: - label_id = row["label_id"] - bucket_start = row["bucket"] - bucket_end = bucket_start + bucket_ms - total_samples = row["total_samples"] - - cursor = self._conn.execute( - """ - SELECT start_time, end_time, sample_count, value - FROM _power - WHERE label_id = ? AND start_time >= ? AND start_time < ? - ORDER BY start_time - """, - (label_id, bucket_start, bucket_end), - ) - samples = cursor.fetchall() - - sample_count = 0 - total_power = 0.0 - for sample in samples: - sample_count += sample["sample_count"] - total_power += sample["value"] - average_power = total_power / sample_count - - self._conn.execute( - "DELETE FROM _power WHERE label_id = ? AND start_time >= ? AND start_time < ?", - (label_id, bucket_start, bucket_end), - ) - self._conn.execute( - "INSERT INTO _power (label_id, start_time, end_time, sample_count, value) VALUES (?, ?, ?, ?, ?)", - (label_id, bucket_start, bucket_end, total_samples, average_power), - ) - - total_sample_count += sample_count - total_bucket_count += 1 - - self._conn.commit() - return total_sample_count, total_bucket_count - - # "Abuse" power table for voltages because the compression algorithm also works perfectly fine for voltages. - insert_voltage = insert_power - get_voltage = get_power - - # ------------------------------------------------------------------------- - # Energy - # ------------------------------------------------------------------------- - - def insert_energy( - self, - label: str, - timestamp: datetime, - energy: float | None, - ) -> None: - if energy is None: - return - self._conn.execute( - """ - INSERT INTO energy (label, timestamp, value) - SELECT ?, ?, ? - WHERE ? != COALESCE( - (SELECT value FROM energy - WHERE label = ? - ORDER BY timestamp DESC LIMIT 1), - -1 - ) - """, - (label, dt_to_ms(timestamp), energy, energy, label), - ) - self._conn.commit() - - def get_energy( - self, - label: str, - start: datetime | None, - end: datetime | None, - normalize: bool = False, - ) -> list[tuple[datetime, float]]: - conditions, params = base_conditions(label, start, end) - where_clause = "WHERE " + " AND ".join(conditions) - query = f""" - SELECT timestamp, value - FROM energy - {where_clause} - ORDER BY timestamp - """ - cursor = self._conn.execute(query, params) - - result = [(row[0], row[1]) for row in cursor.fetchall()] - if normalize and result: - start_energy = result[0][1] - result = [(t, v - start_energy) for t, v in result] - return result - - def get_energy_aggregated( - self, - label: str, - aggregation_seconds: float, - start: datetime | None, - end: datetime | None, - center_buckets: bool = False, - ) -> list[tuple[datetime, float]]: - if start: - start -= timedelta(seconds=aggregation_seconds) - if end: - end += timedelta(seconds=aggregation_seconds) - - agg_ms = int(aggregation_seconds * 1000) - conditions, params = base_conditions(label, start, end) - where_clause = "WHERE " + " AND ".join(conditions) - query = f""" - SELECT - (timestamp / ?) * ? AS bucket, - SUM(delta) AS energy_sum - FROM ( - SELECT - timestamp, - CASE - WHEN prev IS NULL THEN 0 -- first value - WHEN value < prev THEN value -- time series was reset to zero - ELSE value - prev - END AS delta - FROM ( - SELECT - timestamp, - value, - LAG(value) OVER (ORDER BY timestamp) AS prev - FROM energy - {where_clause} - ) - ) - GROUP BY bucket - ORDER BY bucket - """ - cursor = self._conn.execute(query, [agg_ms, agg_ms, *params]) - - center_offset = agg_ms // 2 if center_buckets else 0 - return [(ms_to_dt(r[0] + center_offset), round(r[1], 3)) for r in cursor.fetchall()] - - def get_energy_labels(self, start: datetime | None, end: datetime | None = None) -> list[str]: - return self._get_labels("energy", "timestamp", start, end) - - def get_all_energy( - self, start: datetime, end: datetime | None = None, normalize: bool = False - ) -> dict[str, list[tuple[datetime, float]]]: - energy_series = {} - for label in self.get_energy_labels(start, end): - energy_series[label] = self.get_energy(label, start, end, normalize) - return energy_series - - def get_grid_energy_total( - self, start: datetime | None, end: datetime | None = None, normalize: bool = False - ) -> dict[str, list[tuple[datetime, float]]]: - # TODO: per phase and total - return { - "from_net_total": self.get_energy("from_net_total", start, end, normalize), - "to_net_total": self.get_energy("to_net_total", start, end, normalize), - } - - def integrate_power( - self, label: str, start: datetime, end: datetime, bucket_seconds: int = 60 - ) -> list[tuple[datetime, float]]: - power_series = self.get_power(label, start, end, bucket_seconds=bucket_seconds) - if not power_series: - return [] - - energy_series = [(power_series[0][0] - timedelta(seconds=bucket_seconds), 0.0)] - for ts, v in power_series: - energy_series.append((ts, energy_series[-1][-1] + v / 1000 / (3600 / bucket_seconds))) - return energy_series - - # ------------------------------------------------------------------------- - # Day-ahead prices - # ------------------------------------------------------------------------- - - def insert_price(self, area: str, start_time: datetime, end_time: datetime, price: float) -> None: - self._conn.execute( - """ - INSERT INTO day_ahead_prices (area, start_time, end_time, price) - VALUES (?, ?, ?, ?) - ON CONFLICT (area, start_time) DO UPDATE SET - end_time = excluded.end_time, price = excluded.price - """, - (area, dt_to_ms(start_time), dt_to_ms(end_time), price), - ) - self._conn.commit() - - def insert_prices(self, area: str, prices: list[tuple[datetime, datetime, float]]) -> None: - self._conn.executemany( - """ - INSERT INTO day_ahead_prices (area, start_time, end_time, price) - VALUES (?, ?, ?, ?) - ON CONFLICT (area, start_time) DO UPDATE SET - end_time = excluded.end_time, price = excluded.price - """, - [(area, dt_to_ms(start), dt_to_ms(end), price) for start, end, price in prices], - ) - self._conn.commit() - logger.debug(f"Inserted {len(prices)} price records") - - def get_prices( - self, area: str, start: datetime, end: datetime | None = None, aggregate_minutes: float | None = None - ) -> list[tuple[datetime, float]]: - conditions, params = base_conditions(area, start, end, label_name="area", timestamp_name="start_time") - - if aggregate_minutes is not None: - timestamp_column = "bucket" - value_column = "avg_value" - select_clause = f"(start_time / ?) * ? as {timestamp_column}, AVG(price) as {value_column}" - group_by = f"GROUP BY {timestamp_column}" - agg_ms = round(aggregate_minutes * 60000) - params = [agg_ms, agg_ms, *params] - else: - timestamp_column = "start_time" - group_by = "" - value_column = "price" - select_clause = f"{timestamp_column}, {value_column}" - - where_clause = "WHERE " + " AND ".join(conditions) - cursor = self._conn.execute( - f""" - SELECT {select_clause} - FROM day_ahead_prices - {where_clause} - {group_by} - ORDER BY {timestamp_column} - """, - params, - ) - # TODO: store prices in €/kWh instead of €/MWh - return [(ms_to_dt(row[timestamp_column]), row[value_column] / 1000) for row in cursor.fetchall()] - - def get_hourly_prices( - self, area: str, start: datetime, end: datetime | None = None - ) -> list[tuple[datetime, float]]: - """Get hourly aggregated prices. Returns list of (hour_start, price_eur_per_kwh).""" - params = [area, dt_to_ms(start)] - if end is not None: - params.append(dt_to_ms(end)) - cursor = self._conn.execute( - f""" - SELECT (start_time / 3600000) * 3600000 AS hour, AVG(price) / 1000.0 AS price - FROM day_ahead_prices - WHERE area = ? AND start_time >= ?{" AND start_time < ?" if end is not None else ""} - GROUP BY hour ORDER BY hour - """, - params, - ) - return [(ms_to_dt(row["hour"]), row["price"]) for row in cursor.fetchall()] - - def get_latest_price_time(self, area: str) -> datetime | None: - cursor = self._conn.execute( - "SELECT MAX(end_time) as latest FROM day_ahead_prices WHERE area = ?", - (area,), - ) - row = cursor.fetchone() - if row and row["latest"]: - return ms_to_dt(row["latest"]) - return None - - # ------------------------------------------------------------------------- - # Battery SOC - # ------------------------------------------------------------------------- - - def insert_soc(self, label: str, timestamp: datetime, soc: int) -> None: - # TODO: also insert if last update was more than 5 minutes ago - self._conn.execute( - """ - INSERT INTO battery_soc (label, timestamp, value) - SELECT ?, ?, ? - WHERE ? != COALESCE( - (SELECT value FROM battery_soc WHERE label = ? ORDER BY timestamp DESC LIMIT 1), - -1 - ) - """, - (label, dt_to_ms(timestamp), soc, soc, label), - ) - self._conn.commit() - - def get_battery_soc(self, label: str, start: datetime, end: datetime) -> list[tuple[datetime, float]]: - if isinstance(label, list): - label = label[0] - cursor = self._conn.execute( - "SELECT timestamp, value FROM battery_soc WHERE label = ? AND timestamp >= ? AND timestamp < ? ORDER BY timestamp", - [label, dt_to_ms(start), dt_to_ms(end)], - ) - return [(ms_to_dt(row["timestamp"]), row["value"]) for row in cursor.fetchall()] - - def get_current_soc(self) -> int | None: - cursor = self._conn.execute("SELECT value FROM battery_soc ORDER BY timestamp DESC LIMIT 1") - row = cursor.fetchone() - return row["value"] if row else None - - def get_soc_at(self, timestamp: datetime) -> int | None: - """Get battery SOC reading at or after timestamp. - Before would seemingly make more sense but might return a very out of data SoC value after a - cold-start which would mess with the optimizer.""" - ts_ms = dt_to_ms(timestamp) - cursor = self._conn.execute( - "SELECT value FROM battery_soc WHERE timestamp >= ? ORDER BY timestamp ASC LIMIT 1", - [ts_ms], - ) - row = cursor.fetchone() - if not row: - cursor = self._conn.execute( - "SELECT value FROM battery_soc WHERE timestamp <= ? ORDER BY timestamp DESC LIMIT 1", - [ts_ms], - ) - row = cursor.fetchone() - return row["value"] if row else None - - # ------------------------------------------------------------------------- - # Charge schedule - # ------------------------------------------------------------------------- - - def set_schedule(self, battery_id: str, entries: list[tuple[datetime, datetime, int, float]]) -> None: - # The view on the actual table already handles OR REPLACE and add OR REPLACE to this query messes up the - # insertion... The table_id is then recreated on every conflict. - if not entries: - return - - entries = sorted(entries, key=lambda row: row[0]) - first_ts, _, _, _ = entries[0] - - self._conn.execute( - "DELETE FROM charge_schedule WHERE label = ? AND start_time >= ?", [battery_id, dt_to_ms(first_ts)] - ) - self._conn.executemany( - "INSERT INTO charge_schedule (label, start_time, end_time, power, expected_soc) VALUES (?, ?, ?, ?, ?)", - [(battery_id, dt_to_ms(start), dt_to_ms(end), power, soc) for start, end, power, soc in entries], - ) - self._conn.commit() - - def get_schedule( - self, battery_id: str, start: datetime | None = None, end: datetime | None = None - ) -> list[tuple[datetime, datetime, int, int]]: - conditions, params = base_conditions(battery_id, start, end, timestamp_name="start_time") - where_clause = "WHERE " + " AND ".join(conditions) - cursor = self._conn.execute( - f""" - SELECT start_time, end_time, power, expected_soc - FROM charge_schedule - {where_clause} - ORDER BY start_time - """, - params, - ) - return [(ms_to_dt(row[0]), ms_to_dt(row[1]), row[2], row[3]) for row in cursor.fetchall()] diff --git a/open_ess/database/migration_runner.py b/open_ess/database/migration_runner.py deleted file mode 100644 index 406de11..0000000 --- a/open_ess/database/migration_runner.py +++ /dev/null @@ -1,68 +0,0 @@ -import logging -from datetime import UTC, datetime -from importlib import import_module -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .database import DatabaseConnection - -MIGRATIONS_DIR = Path(__file__).parent / "migrations" - -logger = logging.getLogger(__name__) - - -def get_migrations() -> list[tuple[int, str]]: - """Discover all migration files in the migrations directory. - - Returns: - List of (version, module_name) tuples, sorted by version. - """ - migrations = [] - for file in MIGRATIONS_DIR.glob("*.py"): - if file.name.startswith("_"): - continue - # Parse version from filename like "001_initial.py" - parts = file.stem.split("_", 1) - if len(parts) >= 1 and parts[0].isdigit(): - version = int(parts[0]) - module_name = f"open_ess.database.migrations.{file.stem}" - migrations.append((version, module_name)) - return sorted(migrations) - - -def run_migration(version: int, module_name: str, conn: "DatabaseConnection") -> None: - """Run a single migration. - - Args: - version: Migration version number - module_name: Full module name to import - conn: SQLite connection - """ - module = import_module(module_name) - module.upgrade(conn) - - -def run_migrations(conn: "DatabaseConnection") -> None: - conn.execute(""" - CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER PRIMARY KEY, - applied_at TEXT NOT NULL - ) - """) - conn.commit() - - cursor = conn.execute("SELECT MAX(version) as version FROM schema_version") - row = cursor.fetchone() - current_version = row["version"] or 0 - - for version, module_name in get_migrations(): - if version > current_version: - logger.info(f"Running migration {version}: {module_name}") - run_migration(version, module_name, conn) - conn.execute( - "INSERT INTO schema_version (version, applied_at) VALUES (?, ?)", - (version, datetime.now(UTC)), - ) - conn.commit() - logger.info(f"Migration {version} complete") diff --git a/open_ess/database/migrations/001_initial.py b/open_ess/database/migrations/001_initial.py deleted file mode 100644 index c5721b7..0000000 --- a/open_ess/database/migrations/001_initial.py +++ /dev/null @@ -1,202 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ..database import DatabaseConnection - - -def upgrade(conn: "DatabaseConnection") -> None: - conn.execute("PRAGMA foreign_keys = ON") - - # ------------- - # Labels - # ------------- - - conn.execute(""" - CREATE TABLE labels ( - label_id INTEGER PRIMARY KEY, - label TEXT UNIQUE NOT NULL - ) - """) - - # ------------- - # Day-ahead prices - # ------------- - - conn.execute(""" - CREATE TABLE day_ahead_prices ( - area TEXT NOT NULL, - start_time INTEGER NOT NULL, - end_time INTEGER NOT NULL, - price REAL NOT NULL, - PRIMARY KEY (area, start_time) - ) - """) - - # ------------- - # Charge schedule - # ------------- - - conn.execute(""" - CREATE TABLE _charge_schedule ( - label_id INTEGER NOT NULL, - start_time INTEGER NOT NULL, - end_time INTEGER NOT NULL, - power INTEGER NOT NULL, - expected_soc REAL NOT NULL, - PRIMARY KEY (label_id, start_time), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW charge_schedule AS - SELECT l.label, cs.start_time, cs.end_time, cs.power, cs.expected_soc - FROM _charge_schedule AS cs - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER charge_schedule_insert - INSTEAD OF INSERT ON charge_schedule - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT OR REPLACE INTO _charge_schedule(label_id, start_time, end_time, power, expected_soc) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.start_time, - NEW.end_time, - NEW.power, - NEW.expected_soc - ); - END - """) - conn.execute(""" - CREATE TRIGGER charge_schedule_delete - INSTEAD OF DELETE ON charge_schedule - BEGIN - DELETE FROM _charge_schedule - WHERE label_id = (SELECT label_id FROM labels WHERE label = OLD.label) - AND start_time = OLD.start_time; - END - """) - - # ------------- - # Power - # ------------- - - conn.execute(""" - CREATE TABLE _power ( - label_id INTEGER NOT NULL, - start_time INTEGER NOT NULL, - end_time INTEGER, - sample_count INTEGER, - value REAL NOT NULL, - PRIMARY KEY (label_id, start_time), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW power AS - SELECT l.label, p.start_time, p.end_time, p.sample_count, p.value - FROM _power AS p - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER power_insert - INSTEAD OF INSERT ON power - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT INTO _power(label_id, start_time, end_time, sample_count, value) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.start_time, - NEW.end_time, - NEW.sample_count, - NEW.value - ); - END - """) - conn.execute(""" - CREATE TRIGGER power_delete - INSTEAD OF DELETE ON power - BEGIN - DELETE FROM _power - WHERE label_id = (SELECT label_id FROM labels WHERE label = OLD.label) - AND start_time = OLD.start_time; - END - """) - - # ------------- - # Energy - # ------------- - - conn.execute(""" - CREATE TABLE _energy ( - label_id INTEGER NOT NULL, - timestamp INTEGER NOT NULL, - value REAL NOT NULL, - PRIMARY KEY (label_id, timestamp), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW energy AS - SELECT l.label, e.timestamp, e.value - FROM _energy AS e - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER energy_insert - INSTEAD OF INSERT ON energy - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT INTO _energy(label_id, timestamp, value) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.timestamp, - NEW.value - ); - END - """) - conn.execute(""" - CREATE TRIGGER energy_delete - INSTEAD OF DELETE ON energy - BEGIN - DELETE FROM _energy - WHERE label_id = (SELECT label_id FROM labels WHERE label = OLD.label) - AND timestamp = OLD.timestamp; - END - """) - - # ------------- - # Battery SoC - # ------------- - - conn.execute(""" - CREATE TABLE _battery_soc ( - label_id INTEGER NOT NULL, - timestamp INTEGER NOT NULL, - value REAL NOT NULL, - PRIMARY KEY (label_id, timestamp), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW battery_soc AS - SELECT l.label, e.timestamp, e.value - FROM _battery_soc AS e - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER battery_soc_insert - INSTEAD OF INSERT ON battery_soc - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT INTO _battery_soc(label_id, timestamp, value) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.timestamp, - NEW.value - ); - END - """) - - conn.commit() diff --git a/open_ess/database/migrations/002_rename_labels.py b/open_ess/database/migrations/002_rename_labels.py deleted file mode 100644 index 80cab7d..0000000 --- a/open_ess/database/migrations/002_rename_labels.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Rename labels.""" - -from open_ess.database import DatabaseConnection - - -def upgrade(conn: DatabaseConnection) -> None: - renames = [ - ("victron/vebus/228", "victron/c0619ab2f37c"), - ("victron/vebus/228/power/ac_in/l1", "victron/c0619ab2f37c/228/power/ac_in/l1"), - ("victron/vebus/228/power/ac_out/l1", "victron/c0619ab2f37c/228/power/ac_out/l1"), - ("victron/vebus/228/soc", "victron/c0619ab2f37c/228/soc"), - ("victron/vebus/228/power/battery", "victron/c0619ab2f37c/228/power/battery"), - ("victron/vebus/228/energy/ac_in_to_ac_out", "victron/c0619ab2f37c/228/energy/ac_in_to_ac_out"), - ("victron/vebus/228/energy/ac_in_import", "victron/c0619ab2f37c/228/energy/ac_in_import"), - ("victron/vebus/228/energy/ac_out_to_ac_in", "victron/c0619ab2f37c/228/energy/ac_out_to_ac_in"), - ("victron/vebus/228/energy/ac_in_export", "victron/c0619ab2f37c/228/energy/ac_in_export"), - ("victron/vebus/228/energy/ac_out_export", "victron/c0619ab2f37c/228/energy/ac_out_export"), - ("victron/vebus/228/energy/ac_out_import", "victron/c0619ab2f37c/228/energy/ac_out_import"), - ("victron/vebus/228/voltage/battery", "victron/c0619ab2f37c/228/voltage/battery"), - ("victron/battery/225/power/battery", "victron/c0619ab2f37c/225/power/battery"), - ("victron/battery/225/voltage/battery", "victron/c0619ab2f37c/225/voltage/battery"), - ("victron/battery/225/soc", "victron/c0619ab2f37c/225/soc"), - ("victron/pvinverter/31/power/l1", "victron/c0619ab2f37c/31/power/l1"), - ("victron/pvinverter/31/energy/l1", "victron/c0619ab2f37c/31/energy/l1"), - ] - - for old_label, new_label in renames: - conn.execute( - "UPDATE labels SET label = ? WHERE label = ?", - (new_label, old_label), - ) - - conn.commit() diff --git a/open_ess/database/migrations/__init__.py b/open_ess/database/migrations/__init__.py deleted file mode 100644 index ea0d785..0000000 --- a/open_ess/database/migrations/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Migration scripts.""" diff --git a/open_ess/database/service.py b/open_ess/database/service.py deleted file mode 100644 index b88ee5b..0000000 --- a/open_ess/database/service.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging -from datetime import UTC, datetime, timedelta - -from open_ess.service import Service - -from .database import Database, DatabaseConnection - -logger = logging.getLogger(__name__) - - -class DatabaseService(Service): - def __init__(self, database: Database): - super().__init__("DatabaseService") - self._database = database - self._config = database.config - self._db_conn: DatabaseConnection | None = None - - def on_start(self) -> None: - self._db_conn = self._database.connect() - logger.info("DatabaseService started") - - def tick(self) -> None: - self._run_compression() - - def _run_compression(self) -> None: - if self._db_conn is None: - return None - if self._config.compression.enable: - n_samples, _n_buckets = self._db_conn.compress_power( - datetime.now(UTC), self._config.compression.bucket_seconds - ) - if n_samples > 0: - self._db_conn.vacuum() - - def wait_until_next(self) -> None: - now = datetime.now(UTC) - next_run = now.replace(second=0, microsecond=0) + timedelta(minutes=1, seconds=10) - # ^ Run next compression 10 seconds after a new minute starts. This ensures that all new metrics - # have been written to the database. - - self.wait_seconds((next_run - now).total_seconds()) diff --git a/open_ess/database/util.py b/open_ess/database/util.py deleted file mode 100644 index ac8d692..0000000 --- a/open_ess/database/util.py +++ /dev/null @@ -1,29 +0,0 @@ -from datetime import UTC, datetime - - -def dt_to_ms(dt: datetime) -> int: - """UTC datetime to Unix milliseconds.""" - return int(dt.timestamp() * 1000) - - -def ms_to_dt(ms: int) -> datetime: - """Unix milliseconds to UTC datetime.""" - return datetime.fromtimestamp(ms / 1000, tz=UTC) - - -def base_conditions( - label: str, - start: datetime | None, - end: datetime | None, - label_name: str = "label", - timestamp_name: str = "timestamp", -) -> tuple[list[str], list[str | int]]: - conditions = [f"{label_name} = ?"] - params: list = [label] - if start is not None: - conditions.append(f"{timestamp_name} >= ?") - params.append(dt_to_ms(start)) - if end is not None: - conditions.append(f"{timestamp_name} < ?") - params.append(dt_to_ms(end)) - return conditions, params diff --git a/open_ess/frontend/app.py b/open_ess/frontend/app.py index bf7fc2b..fb4721a 100644 --- a/open_ess/frontend/app.py +++ b/open_ess/frontend/app.py @@ -7,7 +7,6 @@ from fastapi.staticfiles import StaticFiles from open_ess.battery_system import BatterySystem -from open_ess.database import Database from open_ess.timeseries import TimeseriesBackend from open_ess.timeseries.metricsqlite.backend import MetricSQLiteBackend @@ -20,14 +19,12 @@ def create_app( - database: Database, config: "Config", battery_systems: list[BatterySystem], timeseries: TimeseriesBackend | None = None, ) -> FastAPI: @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: - _app.state.database = database.connect() _app.state.price_config = config.prices _app.state.battery_systems = battery_systems _app.state.timeseries = timeseries diff --git a/open_ess/frontend/dependencies.py b/open_ess/frontend/dependencies.py index 72678e2..6a7084d 100644 --- a/open_ess/frontend/dependencies.py +++ b/open_ess/frontend/dependencies.py @@ -3,15 +3,10 @@ from fastapi import Depends, Request from open_ess.battery_system import BatterySystem -from open_ess.database import DatabaseConnection from open_ess.pricing import PriceConfig from open_ess.timeseries import TimeseriesBackend -def get_database(request: Request) -> DatabaseConnection: - return request.app.state.database # type: ignore[no-any-return] - - def get_price_config(request: Request) -> PriceConfig: return request.app.state.price_config # type: ignore[no-any-return] @@ -25,7 +20,6 @@ def get_timeseries(request: Request) -> TimeseriesBackend | None: # Type aliases for cleaner route signatures -Database = Annotated[DatabaseConnection, Depends(get_database)] PriceConfigDep = Annotated[PriceConfig, Depends(get_price_config)] BatterySystemsDep = Annotated[list[BatterySystem], Depends(get_battery_systems)] TimeseriesDep = Annotated[TimeseriesBackend | None, Depends(get_timeseries)] diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index 249f2f6..ce1c5f5 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,18 +1,13 @@ import logging -from datetime import UTC, datetime, timedelta -from enum import StrEnum from typing import TYPE_CHECKING -from fastapi import APIRouter, HTTPException, Query +from fastapi import APIRouter, HTTPException from pydantic import BaseModel -from open_ess.frontend.dependencies import BatterySystemsDep, Database, PriceConfigDep, TimeseriesDep - -from .util import TimeSeries, data_to_timeseries, find_full_battery_cycles, query_result_to_timeseries +from .util import TimeSeries if TYPE_CHECKING: - from open_ess.database import DatabaseConnection - from open_ess.timeseries import QueryResult, TimeseriesBackend + pass logger = logging.getLogger(__name__) @@ -34,1129 +29,1128 @@ class HealthResponse(BaseModel): @router.get("/health", response_model=HealthResponse) -async def health_check(db: Database) -> HealthResponse: - try: - # TODO: - cursor = db.execute("SELECT name FROM sqlite_master WHERE type='table'") - tables = [row["name"] for row in cursor.fetchall()] - return HealthResponse(status="ok", database="connected", tables=tables) - except Exception as e: - logger.exception("Health check failed") - raise HTTPException(status_code=500, detail=str(e)) from e - - -# ---------------------------- # -# Power overview (Dashboard) # -# ---------------------------- # - - -class BatterySystemInfo(BaseModel): - id: str - name: str - - -class SystemLayoutData(BaseModel): - phases: list[int] - # TODO: grid_labels: list[str] # ["L1", "L2", "L3"] - has_solar: bool - battery_systems: list[BatterySystemInfo] - - -@router.get("/system-layout", response_model=SystemLayoutData) -async def get_system_layout(battery_systems: BatterySystemsDep) -> SystemLayoutData: - return SystemLayoutData( - phases=[1, 2, 3], - # grid_labels=["L1", "L2", "L3"], - has_solar=True, # TODO - battery_systems=[BatterySystemInfo(id=b.id, name=b.name) for b in battery_systems], - ) - - -class BatteryPowerValues(BaseModel): - charger: float | None - inverter: float | None - battery: float | None - losses: float | None - - -class PowerFlowData(BaseModel): - grid: dict[str, float | None] - solar: float | None - consumption: dict[str, float] # e.g. {"L1": 800, "L2": 300, "L3": 200} - batteries: dict[str, BatteryPowerValues] - - -def _get_instant_value(result: "QueryResult") -> float | None: - """Extract the latest value from an instant query result.""" - if result.series and result.series[0].values: - return result.series[0].values[-1][1] - return None - - -@router.get("/power-flow", response_model=PowerFlowData) -async def get_power_flow( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, -) -> PowerFlowData: - if timeseries is not None: - return await _get_power_flow_timeseries(timeseries, battery_systems) - return await _get_power_flow_legacy(db, battery_systems) - - -async def _get_power_flow_timeseries( - timeseries: "TimeseriesBackend", - battery_systems: list, -) -> PowerFlowData: - """Get power flow data from timeseries backend.""" - now = datetime.now(UTC) - - # Grid power per phase - grid_power: dict[str, float | None] = {} - for phase in ("L1", "L2", "L3"): - query = f'openess_power_watts{{from="grid", phase="{phase}"}}' - result = timeseries.query(query, now) - grid_power[phase] = _get_instant_value(result) - - # Solar power - solar_query = 'openess_power_watts{from="pvinverter"}' - solar_result = timeseries.query(solar_query, now) - solar_power = _get_instant_value(solar_result) - - # Battery power for each system - batteries: dict[str, BatteryPowerValues] = {} - for battery_system in battery_systems: - device = battery_system.id - - # AC power (charger/inverter) - ac_in_query = battery_system.config.queries.power_ac_in.replace("$device", device) - ac_in_result = timeseries.query(ac_in_query, now) - system = _get_instant_value(ac_in_result) or 0 - - charger = -system if system < 0 else 0 - inverter = system if system > 0 else 0 - - # DC battery power - battery_query = battery_system.config.queries.power_battery.replace("$device", device) - battery_result = timeseries.query(battery_query, now) - battery = _get_instant_value(battery_result) or 0 - - losses = battery - system - - batteries[battery_system.id] = BatteryPowerValues( - charger=charger, - inverter=inverter, - battery=battery, - losses=losses, - ) - - return PowerFlowData( - grid=grid_power, - solar=solar_power, - consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, - batteries=batteries, - ) - - -async def _get_power_flow_legacy(db: "DatabaseConnection", battery_systems: list) -> PowerFlowData: - """Get power flow data from legacy database.""" - start = datetime.now(UTC) - timedelta(seconds=10) - - grid_power: dict[str, float | None] = {} - for i in (1, 2, 3): - power = None - result = db.get_power(f"grid/power/l{i}", start=start, bucket_seconds=None) - if result: - _, power = result[-1] - grid_power[f"L{i}"] = power - - solar_power = None - result = db.get_power("victron/pvinverter/31/power/l1", start=start, bucket_seconds=None) - if result: - _, solar_power = result[-1] - - batteries: dict[str, BatteryPowerValues] = {} - for battery_system in battery_systems: - charger = 0 - inverter = 0 - battery = 0 - losses = 0 - system = 0 - result = db.get_power(battery_system.config.metrics.power_to_system, start=start, bucket_seconds=None) - if result: - _, system = result[-1] - if system < 0: - charger = -system - if system > 0: - inverter = system - - result = db.get_power(battery_system.config.metrics.power_to_battery, start=start, bucket_seconds=None) - if result: - _, battery = result[-1] - losses = battery - system - - batteries[battery_system.id] = BatteryPowerValues( - charger=charger, - inverter=inverter, - battery=battery, - losses=losses, - ) - - return PowerFlowData( - grid=grid_power, - solar=solar_power, - consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, - batteries=batteries, - ) - - -# ------------------------------- # -# Services overview (Dashboard) # -# ------------------------------- # - - -class Status(StrEnum): - OK = "ok" - WARNING = "warning" - ERROR = "error" - - -class ServiceMessage(BaseModel): - timestamp: datetime - status: Status - message: str - - -class ServiceStatus(BaseModel): - status: Status - messages: list[ServiceMessage] - - -class ServicesStatusResponse(BaseModel): - database: ServiceStatus | None - optimizer: ServiceStatus | None - - -@router.get("/services-status", response_model=ServicesStatusResponse) -async def services_status() -> ServicesStatusResponse: +async def health_check() -> HealthResponse: try: - return ServicesStatusResponse( - database=ServiceStatus(status=Status.OK, messages=[]), - optimizer=ServiceStatus(status=Status.OK, messages=[]), - ) + # TODO + # cursor = db.execute("SELECT name FROM sqlite_master WHERE type='table'") + # tables = [row["name"] for row in cursor.fetchall()] + return HealthResponse(status="ok", database="connected", tables=[]) except Exception as e: logger.exception("Health check failed") raise HTTPException(status_code=500, detail=str(e)) from e -@router.get("/battery-ids", response_model=list[str]) -async def get_battery_ids(battery_systems: BatterySystemsDep) -> list[str]: - try: - return [s.id for s in battery_systems] - except Exception as e: - logger.exception("Failed to get battery ids") - raise HTTPException(status_code=500, detail=str(e)) from e - - -# ------------------------ # -# Metrics page endpoints # -# ------------------------ # - - -class BatteryEnergySeries(BaseModel): - energy_to_charger: list[float | None] = [] - energy_from_inverter: list[float | None] = [] - energy_to_battery: list[float | None] = [] - energy_from_battery: list[float | None] = [] - energy_loss_to_battery: list[float | None] = [] - energy_loss_from_battery: list[float | None] = [] - - -class EnergyGraphResponse(BaseModel): - timestamps: list[datetime] - - grid_import: dict[str, list[float | None]] - grid_export: dict[str, list[float | None]] - - battery_systems: dict[str, BatteryEnergySeries] - - solar: list[float | None] = [] - to_consumption: list[float | None] = [] - from_consumption: list[float | None] = [] - - -@router.get("/energy-graph", response_model=EnergyGraphResponse) -async def get_energy_flow_endpoint( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - bucket_minutes: int = Query(default=60), -) -> EnergyGraphResponse: - try: - battery_system = None - if battery_id: - for bs in battery_systems: - if bs.id == battery_id: - battery_system = bs - break - elif len(battery_systems) == 1: - battery_system = battery_systems[0] - - if battery_system is None: - if battery_id: - raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") - else: - raise HTTPException(status_code=400, detail="Please provide a battery_id") - - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now - - if timeseries is not None: - return await _get_energy_graph_timeseries(timeseries, battery_system, start, end, bucket_minutes) - - return await _get_energy_graph_legacy(db, battery_system, start, end, bucket_minutes) - except Exception as e: - logger.exception("Failed to get energy flow") - raise HTTPException(status_code=500, detail=str(e)) from e - - -async def _get_energy_graph_timeseries( - timeseries: "TimeseriesBackend", - battery_system, - start: datetime, - end: datetime, - bucket_minutes: int, -) -> EnergyGraphResponse: - """Get energy graph data from timeseries backend.""" - device = battery_system.id - step = f"{bucket_minutes}m" - - # Query energy series using increase() to get per-bucket energy consumption - queries = battery_system.config.queries - grid_import_query = queries.energy_grid_import.replace("$device", device) - grid_export_query = queries.energy_grid_export.replace("$device", device) - to_mp_query = queries.energy_to_battery.replace("$device", device) - from_mp_query = queries.energy_from_battery.replace("$device", device) - - # Use increase() to get energy delta per bucket - grid_import_result = timeseries.query_range(f"increase({grid_import_query}[{step}])", start, end, step) - grid_export_result = timeseries.query_range(f"increase({grid_export_query}[{step}])", start, end, step) - to_mp_result = timeseries.query_range(f"increase({to_mp_query}[{step}])", start, end, step) - from_mp_result = timeseries.query_range(f"increase({from_mp_query}[{step}])", start, end, step) - - # Convert to dict for easier lookup - def result_to_dict(result: "QueryResult") -> dict[datetime, float]: - if not result.series or not result.series[0].values: - return {} - return {ts: val for ts, val in result.series[0].values} - - grid_import_data = result_to_dict(grid_import_result) - grid_export_data = result_to_dict(grid_export_result) - to_mp_data = result_to_dict(to_mp_result) - from_mp_data = result_to_dict(from_mp_result) - - # Collect all timestamps - all_timestamps: set[datetime] = set() - all_timestamps.update(grid_import_data.keys()) - all_timestamps.update(grid_export_data.keys()) - all_timestamps.update(to_mp_data.keys()) - all_timestamps.update(from_mp_data.keys()) - timestamps = sorted(all_timestamps) - - # Build response series - grid_exports: dict[str, list[float | None]] = {"From MP": []} - grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} - battery_stats = BatteryEnergySeries() - - for ts in timestamps: - from_mp = from_mp_data.get(ts) - grid_exports["From MP"].append(round(from_mp, 3) if from_mp else None) - unaccounted_export = grid_export_data.get(ts, 0) - (from_mp or 0) - - to_mp = to_mp_data.get(ts) - grid_imports["To MP"].append(round(to_mp, 3) if to_mp else None) - grid_import = grid_import_data.get(ts) - if grid_import is not None: - grid_import -= (to_mp or 0) - unaccounted_export - grid_imports["Consumption"].append(round(grid_import, 3) if grid_import else None) - - battery_stats.energy_to_charger.append(round(to_mp, 3) if to_mp else None) - battery_stats.energy_from_inverter.append(round(from_mp, 3) if from_mp else None) - - return EnergyGraphResponse( - timestamps=timestamps, - grid_export=grid_exports, - grid_import=grid_imports, - battery_systems={battery_system.config.name: battery_stats}, - ) - - -async def _get_energy_graph_legacy( - db: "DatabaseConnection", - battery_system, - start: datetime, - end: datetime, - bucket_minutes: int, -) -> EnergyGraphResponse: - """Get energy graph data from legacy database.""" - series = { - "grid_import": db.get_energy_aggregated( - "grid/energy/import/total", bucket_minutes * 60, start, end, center_buckets=True - ), - "grid_export": db.get_energy_aggregated( - "grid/energy/export/total", bucket_minutes * 60, start, end, center_buckets=True - ), - "vebus_228_import": db.get_energy_aggregated( - battery_system.config.metrics.energy_to_system, bucket_minutes * 60, start, end, center_buckets=True - ), - "vebus_228_export": db.get_energy_aggregated( - battery_system.config.metrics.energy_from_system, bucket_minutes * 60, start, end, center_buckets=True - ), - } - - timestamps: set[datetime] = set() - series_as_dict: dict[str, dict[datetime, float]] = {name: {} for name in series} - for name, s in series.items(): - for ts, v in s: - timestamps.add(ts) - series_as_dict[name][ts] = v - sorted_timestamps = sorted(timestamps) - - grid_exports: dict[str, list[float | None]] = {"From MP": []} - grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} - battery_stats = BatteryEnergySeries() - - for ts in sorted_timestamps: - from_mp = series_as_dict["vebus_228_export"].get(ts) - grid_exports["From MP"].append(from_mp) - unaccounted_export = series_as_dict["grid_export"].get(ts, 0) - (from_mp or 0) - - to_mp = series_as_dict["vebus_228_import"].get(ts) - grid_imports["To MP"].append(to_mp) - grid_import = series_as_dict["grid_import"].get(ts) - if grid_import is not None: - grid_import -= (to_mp or 0) - unaccounted_export - grid_imports["Consumption"].append(grid_import) - - battery_stats.energy_to_charger.append(to_mp) - battery_stats.energy_from_inverter.append(from_mp) - - return EnergyGraphResponse( - timestamps=sorted_timestamps, - grid_export=grid_exports, - grid_import=grid_imports, - battery_systems={"MultiPlus": battery_stats}, - ) - - -def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: - """Calculate query step from aggregate_minutes or time range.""" - if aggregate_minutes > 1: - return f"{aggregate_minutes}m" - # Auto-calculate based on range - duration = (end - start).total_seconds() - if duration <= 3600: # 1 hour - return "1m" - if duration <= 6 * 3600: # 6 hours - return "5m" - if duration <= 24 * 3600: # 24 hours - return "15m" - return "1h" - - -@router.get("/power-graph", response_model=PowerResponse) -async def get_power_graph( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int = Query(default=1), -) -> PowerResponse: - try: - battery_system = None - if battery_id: - for bs in battery_systems: - if bs.id == battery_id: - battery_system = bs - break - elif len(battery_systems) == 1: - battery_system = battery_systems[0] - - if battery_system is None: - if battery_id: - raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") - else: - raise HTTPException(status_code=400, detail="Please provide a battery_id") - - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now - - # Schedule always comes from database - schedule_series: list[tuple[datetime, float]] = [] - for ts_start, ts_end, v, _ in db.get_schedule(battery_system.config.id, start): - schedule_series.extend([(ts_start, v), (ts_end, v)]) - - if timeseries is not None: - device = battery_system.id - step = _calculate_step(start, end, aggregate_minutes) - - series: dict[str, TimeSeries] = {} - - # Grid power per phase - for phase in ("L1", "L2", "L3"): - query = f'openess_power_watts{{from="grid", phase="{phase}", device="{device}"}}' - result = timeseries.query_range(query, start, end, step) - series[f"Grid {phase}"] = query_result_to_timeseries(result) - - # AC power (to/from MultiPlus) - ac_query = battery_system.config.queries.power_ac_in.replace("$device", device) - ac_result = timeseries.query_range(ac_query, start, end, step) - series["To MP"] = query_result_to_timeseries(ac_result) - - # Battery DC power - battery_query = battery_system.config.queries.power_battery.replace("$device", device) - battery_result = timeseries.query_range(battery_query, start, end, step) - series["Battery"] = query_result_to_timeseries(battery_result) - - # Solar power (negated for display) - solar_query = battery_system.config.queries.power_pv.replace("$device", device) - solar_result = timeseries.query_range(solar_query, start, end, step) - solar_ts = query_result_to_timeseries(solar_result) - series["Solar"] = TimeSeries( - timestamps=solar_ts.timestamps, - values=[-v for v in solar_ts.values], - ) - - series["Schedule"] = data_to_timeseries(schedule_series) - return PowerResponse(series=series) - - # Legacy database queries - bucket_seconds = aggregate_minutes * 60 - legacy_series: dict[str, list[tuple[datetime, float]]] = { - f"Grid L{i}": db.get_power(f"grid/power/l{i}", start, end, bucket_seconds) for i in (1, 2, 3) - } - legacy_series["To MP"] = db.get_power(battery_system.config.metrics.power_to_system, start, end, bucket_seconds) - legacy_series["Battery"] = db.get_power( - battery_system.config.metrics.power_to_battery, start, end, bucket_seconds - ) - legacy_series["Solar"] = [ - (t, -p) for t, p in db.get_power("victron/pvinverter/31/power/l1", start, end, bucket_seconds) - ] - legacy_series["Schedule"] = schedule_series - - return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) - except Exception as e: - logger.exception("Failed to get power data") - raise HTTPException(status_code=500, detail=str(e)) from e - - -class PricePoint(BaseModel): - time: datetime - market: float | None - buy: float | None - sell: float | None - - -class PricesResponse(BaseModel): - area: str - aggregate_minutes: int - unit: str = "€/kWh" # TODO: based on area - timeseries: list[PricePoint] - - -@router.get("/prices", response_model=PricesResponse) -async def get_price_data( - db: Database, - price_config: PriceConfigDep, - area: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int | None = Query(default=None), -) -> PricesResponse: - try: - if area is None: - area = price_config.area - now = datetime.now(UTC) - if start is None: - start = now - timedelta(days=7) - if end is None: - end = now + timedelta(days=2) - if aggregate_minutes is None: - aggregate_minutes = price_config.aggregate_minutes - - timeseries = [] - for timestamp, price in db.get_prices(area, start, end, aggregate_minutes=aggregate_minutes): - timeseries.append( - PricePoint( - time=timestamp, - market=round(price, 4), - buy=round(price_config.buy_price(price), 4), - sell=round(price_config.sell_price(price), 4), - ) - ) - - return PricesResponse( - area=area, - aggregate_minutes=aggregate_minutes, - timeseries=timeseries, - ) - except Exception as e: - logger.exception("Failed to get prices") - raise HTTPException(status_code=500, detail=str(e)) from e - - -class BatteryGraphResponse(BaseModel): - soc: TimeSeries - schedule: TimeSeries # Scheduled (past and future) SoC - voltage: TimeSeries - - -@router.get("/battery-graph", response_model=dict[str, BatteryGraphResponse]) -async def get_battery_graph( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), -) -> dict[str, BatteryGraphResponse]: - try: - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=48) - if end is None: - end = now + timedelta(hours=24) - - result = {} - for battery_system in battery_systems: - if battery_id is not None and battery_system.config.id != battery_id: - continue - - # Schedule still comes from database (not in timeseries) - scheduled = [(t, soc) for _, t, _, soc in db.get_schedule(battery_system.config.id, start)] - - # SOC and voltage from timeseries backend - if timeseries is not None: - device = battery_system.id - soc_query = battery_system.config.queries.soc.replace("$device", device) - voltage_query = battery_system.config.queries.voltage.replace("$device", device) - - soc_result = timeseries.query_range(soc_query, start, end, step="1m") - voltage_result = timeseries.query_range(voltage_query, start, end, step="1m") - - result[battery_system.config.name] = BatteryGraphResponse( - soc=query_result_to_timeseries(soc_result, rounding=1), - schedule=data_to_timeseries(scheduled, rounding=1), - voltage=query_result_to_timeseries(voltage_result, rounding=2), - ) - else: - # Fallback to legacy database queries - soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) - voltage = db.get_voltage(battery_system.config.metrics.battery_voltage, start, end, bucket_seconds=60) - - result[battery_system.config.name] = BatteryGraphResponse( - soc=data_to_timeseries(soc, rounding=1), - schedule=data_to_timeseries(scheduled, rounding=1), - voltage=data_to_timeseries(voltage, rounding=2), - ) - return result - except Exception as e: - logger.exception("Failed to get battery SOC") - raise HTTPException(status_code=500, detail=str(e)) from e - - -# ---------------# -# Cycles page # -# ---------------# - - -class EfficiencyScatterPoint(BaseModel): - time: datetime - battery_power: float - inverter_charger_power: float - losses: float - efficiency: float | None - soc: int | None - category: str - - -@router.get("/efficiency-scatter", response_model=list[EfficiencyScatterPoint]) -async def get_efficiency_scatter( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int = Query(default=10), - idle_threshold: int = Query(default=5), -) -> list[EfficiencyScatterPoint]: - try: - battery_system = None - if battery_id: - for bs in battery_systems: - if bs.id == battery_id: - battery_system = bs - break - elif len(battery_systems) == 1: - battery_system = battery_systems[0] - - if battery_system is None: - if battery_id: - raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") - else: - raise HTTPException(status_code=400, detail="Please provide a battery_id") - - now = datetime.now(UTC) - if start is None: - start = now - timedelta(days=7) - if end is None: - end = now - - if timeseries is not None: - return await _get_efficiency_scatter_timeseries( - timeseries, battery_system, start, end, aggregate_minutes, idle_threshold - ) - - return await _get_efficiency_scatter_legacy(db, battery_system, start, end, aggregate_minutes, idle_threshold) - except Exception as e: - logger.exception("Failed to get efficiency scatter data") - raise HTTPException(status_code=500, detail=str(e)) from e - - -async def _get_efficiency_scatter_timeseries( - timeseries: "TimeseriesBackend", - battery_system, - start: datetime, - end: datetime, - aggregate_minutes: int, - idle_threshold: int, -) -> list[EfficiencyScatterPoint]: - """Get efficiency scatter data from timeseries backend.""" - device = battery_system.id - step = f"{aggregate_minutes}m" - queries = battery_system.config.queries - - # Query AC in, AC out, and battery DC power - ac_in_query = queries.power_ac_in.replace("$device", device) - ac_out_query = queries.power_ac_out.replace("$device", device) - dc_query = queries.power_battery.replace("$device", device) - - ac_in_result = timeseries.query_range(ac_in_query, start, end, step) - ac_out_result = timeseries.query_range(ac_out_query, start, end, step) - dc_result = timeseries.query_range(dc_query, start, end, step) - - # Convert to dicts - def result_to_dict(result: "QueryResult") -> dict[datetime, float]: - if not result.series or not result.series[0].values: - return {} - return {ts: val for ts, val in result.series[0].values} - - ac_in_data = result_to_dict(ac_in_result) - ac_out_data = result_to_dict(ac_out_result) - dc_data = result_to_dict(dc_result) - - # Merge data by timestamp - all_timestamps = set(ac_in_data.keys()) & set(ac_out_data.keys()) & set(dc_data.keys()) - - points = [] - for ts in sorted(all_timestamps): - ac = ac_in_data[ts] - ac_out_data[ts] - dc = dc_data[ts] - - if abs(dc) < idle_threshold: - category = "idling" - elif dc > 0: - category = "charging" - else: - category = "discharging" - - losses = ac - dc - efficiency = None - if category == "charging" and ac > 0: - efficiency = (dc / ac) * 100 - elif category == "discharging" and dc < 0: - efficiency = (ac / dc) * 100 - - points.append( - EfficiencyScatterPoint( - time=ts, - battery_power=round(abs(dc), 1), - inverter_charger_power=round(ac, 1), - losses=round(losses, 1), - efficiency=round(efficiency, 1) if efficiency is not None else None, - soc=None, - category=category, - ) - ) - - return points - - -async def _get_efficiency_scatter_legacy( - db: "DatabaseConnection", - battery_system, - start: datetime, - end: datetime, - aggregate_minutes: int, - idle_threshold: int, -) -> list[EfficiencyScatterPoint]: - """Get efficiency scatter data from legacy database.""" - metrics = battery_system.config.metrics - bucket_seconds = aggregate_minutes * 60 - - # Use configured metrics paths - ac_in_path = metrics.power_to_system - if isinstance(ac_in_path, list): - ac_in_path = ac_in_path[0] - - # AC out is typically the same vebus but ac_out instead of ac_in - ac_out_path = ac_in_path.replace("ac_in", "ac_out") if ac_in_path else None - - dc_path = metrics.power_to_battery - if isinstance(dc_path, list): - dc_path = dc_path[0] - - ac_in = db.get_power(ac_in_path, start, end, bucket_seconds=bucket_seconds) if ac_in_path else [] - ac_out = db.get_power(ac_out_path, start, end, bucket_seconds=bucket_seconds) if ac_out_path else [] - dc = db.get_power(dc_path, start, end, bucket_seconds=bucket_seconds) if dc_path else [] - - data: dict[datetime, list[float | None]] = { - ts: [v_in - v_out, None] for (ts, v_in), (_, v_out) in zip(ac_in, ac_out, strict=False) - } - for ts, v in dc: - if ts in data: - data[ts][1] = v - - points = [] - for ts, (ac, dc_val) in data.items(): - if ac is None or dc_val is None: - continue - - if abs(dc_val) < idle_threshold: - category = "idling" - elif dc_val > 0: - category = "charging" - else: - category = "discharging" - - losses = ac - dc_val - efficiency = None - if category == "charging" and ac > 0: - efficiency = (dc_val / ac) * 100 - elif category == "discharging" and dc_val < 0: - efficiency = (ac / dc_val) * 100 - - points.append( - EfficiencyScatterPoint( - time=ts, - battery_power=round(abs(dc_val), 1), - inverter_charger_power=round(ac, 1), - losses=round(losses, 1), - efficiency=round(efficiency, 1) if efficiency is not None else None, - soc=None, - category=category, - ) - ) - - return points - - -class BatteryCycle(BaseModel): - start_time: datetime - end_time: datetime - duration_hours: float - min_soc: float - ac_energy_in: float | None - ac_energy_out: float | None - dc_energy_in: float - dc_energy_out: float - system_efficiency: float | None - battery_efficiency: float | None - charger_efficiency: float | None - inverter_efficiency: float | None - profit: float | None - scheduled_profit: float | None - - -@router.get("/cycles", response_model=list[BatteryCycle]) -async def get_battery_cycles( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - price_config: PriceConfigDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - min_soc_swing: int = Query(default=10), -) -> list[BatteryCycle]: - try: - battery_system = None - if battery_id: - for bs in battery_systems: - if bs.id == battery_id: - battery_system = bs - break - elif len(battery_systems) == 1: - battery_system = battery_systems[0] - - if battery_system is None: - if battery_id: - raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") - else: - raise HTTPException(status_code=400, detail="Please provide a battery_id") - - now = datetime.now(UTC) - if start is None: - start = now - timedelta(days=30) - if end is None: - end = now - - # Get SOC data from timeseries or legacy database - if timeseries is not None: - device = battery_system.id - soc_query = battery_system.config.queries.soc.replace("$device", device) - soc_result = timeseries.query_range(soc_query, start, end, step="1m") - battery_soc = [(ts, val) for ts, val in soc_result.series[0].values] if soc_result.series else [] - else: - battery_soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) - - raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) - - cycles = [] - for cycle_start, cycle_end, min_soc in raw_cycles: - duration = (cycle_end - cycle_start).total_seconds() / 3600.0 - - # TODO: this is cursed AF and should be fixed - dc_energy_in = 0.0 - dc_energy_out = 0.0 - for _, p in db.get_power(battery_system.config.metrics.power_to_battery[0], cycle_start, cycle_end): - # for _, p in db.get_power("vebus_228_battery", cycle_start, cycle_end): - if p > 0: - dc_energy_in += p - else: - dc_energy_out += -p - dc_energy_in /= 60000 - dc_energy_out /= 60000 - - ac_in_import = db.get_energy( - battery_system.config.metrics.energy_to_system, cycle_start, cycle_end, normalize=True - ) - # ac_out_import = db.get_energy("vebus_228_ac_out_import", cycle_start, cycle_end, normalize=True) - ac_in_export = db.get_energy( - battery_system.config.metrics.energy_from_system, cycle_start, cycle_end, normalize=True - ) - # ac_out_export = db.get_energy("vebus_228_ac_out_export", cycle_start, cycle_end, normalize=True) - - ac_energy_in = 0.0 - if ac_in_import: - ac_energy_in += ac_in_import[-1][1] - # if ac_out_import: - # ac_energy_in += ac_out_import[-1][1] - ac_energy_out = 0.0 - if ac_in_export: - ac_energy_out += ac_in_export[-1][1] - # if ac_out_export: - # ac_energy_out += ac_out_export[-1][1] - - profit = 0.0 - scheduled_profit = 0.0 - e_in = { - ts: v - for ts, v in db.get_energy_aggregated( - battery_system.config.metrics.energy_to_system, 3600, cycle_start, cycle_end - ) - } - e_out = { - ts: v - for ts, v in db.get_energy_aggregated( - battery_system.config.metrics.energy_from_system, 3600, cycle_start, cycle_end - ) - } - scheduled = {ts: v for ts, _, v, _ in db.get_schedule(battery_system.config.id, cycle_start)} - for ts, v in db.get_prices(price_config.area, cycle_start, cycle_end, aggregate_minutes=60): - profit -= price_config.buy_price(v) * e_in.get(ts, 0) - profit += price_config.sell_price(v) * e_out.get(ts, 0) - scheduled_power = scheduled.get(ts, 0) - if scheduled_power > 0: - scheduled_profit -= price_config.buy_price(v) * scheduled_power / 1000 - if scheduled_power < 0: - scheduled_profit += price_config.sell_price(v) * -scheduled_power / 1000 - - cycles.append( - BatteryCycle( - start_time=cycle_start, - end_time=cycle_end, - duration_hours=round(duration, 2), - min_soc=round(min_soc, 1), - ac_energy_in=round(ac_energy_in, 2) if ac_energy_in else None, - ac_energy_out=round(ac_energy_out, 2) if ac_energy_out else None, - dc_energy_in=round(dc_energy_in, 2), - dc_energy_out=round(dc_energy_out, 2), - system_efficiency=round(ac_energy_out / ac_energy_in * 100, 1) if ac_energy_in else None, - battery_efficiency=round(dc_energy_out / dc_energy_in * 100, 1) if dc_energy_in else None, - charger_efficiency=round(dc_energy_in / ac_energy_in * 100, 1) if ac_energy_in else None, - inverter_efficiency=round(ac_energy_out / dc_energy_out * 100, 1) if dc_energy_out else None, - profit=round(profit, 2), - scheduled_profit=round(scheduled_profit, 2), - ) - ) - - return cycles - except Exception as e: - logger.exception("Failed to get battery cycles") - raise HTTPException(status_code=500, detail=str(e)) from e - - -# -------------------------# -# Generalized endpoints # -# -------------------------# - - -# TODO: add parameter to select subset of series -@router.get("/power", response_model=PowerResponse) -async def get_power( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int = Query(default=1), -) -> PowerResponse: - try: - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now - - if timeseries is not None and battery_systems: - # Query all power metrics from timeseries - step = _calculate_step(start, end, aggregate_minutes) - series: dict[str, TimeSeries] = {} - - for battery_system in battery_systems: - device = battery_system.id - queries = battery_system.config.queries - prefix = battery_system.config.name or device - - # Query each power metric - power_queries = { - "Grid": queries.power_grid_total, - "Grid L1": f'openess_power_watts{{from="grid", phase="L1", device="{device}"}}', - "Grid L2": f'openess_power_watts{{from="grid", phase="L2", device="{device}"}}', - "Grid L3": f'openess_power_watts{{from="grid", phase="L3", device="{device}"}}', - "PV": queries.power_pv, - "Battery": queries.power_battery, - "AC In": queries.power_ac_in, - "AC Out": queries.power_ac_out, - } - - for name, query in power_queries.items(): - resolved_query = query.replace("$device", device) - result = timeseries.query_range(resolved_query, start, end, step) - label = f"{prefix}/{name}" if len(battery_systems) > 1 else name - series[label] = query_result_to_timeseries(result) - - return PowerResponse(series=series) - - # Legacy database fallback - legacy_series = db.get_all_power(start, end, aggregate_minutes * 60) - return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) - except Exception as e: - logger.exception("Failed to get debug power flows") - raise HTTPException(status_code=500, detail=str(e)) from e - - -# TODO: add parameter to select subset of series -# TODO: add normalize parameter -@router.get("/energy", response_model=EnergyResponse) -async def get_energy( - db: Database, - timeseries: TimeseriesDep, - battery_systems: BatterySystemsDep, - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - bucket_minutes: int = Query(default=60), -) -> EnergyResponse: - try: - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now - - if timeseries is not None and battery_systems: - # Query all energy metrics from timeseries - step = f"{bucket_minutes}m" - series: dict[str, TimeSeries] = {} - - for battery_system in battery_systems: - device = battery_system.id - queries = battery_system.config.queries - prefix = battery_system.config.name or device - - energy_queries = { - "Grid Import": queries.energy_grid_import, - "Grid Export": queries.energy_grid_export, - "To Battery": queries.energy_to_battery, - "From Battery": queries.energy_from_battery, - } - - for name, query in energy_queries.items(): - resolved_query = query.replace("$device", device) - # Use increase() to get energy delta per bucket - result = timeseries.query_range(f"increase({resolved_query}[{step}])", start, end, step) - label = f"{prefix}/{name}" if len(battery_systems) > 1 else name - series[label] = query_result_to_timeseries(result, rounding=3) - - return EnergyResponse(series=series) - - # Legacy database fallback - legacy_series = db.get_all_energy(start, end, normalize=True) - - # Get integrated power flows - for label in db.get_power_labels(start, end): - legacy_series[f"{label} [integrated]"] = db.integrate_power(label, start, end) - - return EnergyResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) - except Exception as e: - logger.exception("Failed to get debug energy flows") - raise HTTPException(status_code=500, detail=str(e)) from e - - -# -------------------------- # -# Timeseries query helpers # -# -------------------------- # - - -@router.get("/queries/{battery_id}") -async def get_queries( - battery_id: str, - battery_systems: BatterySystemsDep, -) -> dict[str, str]: - """Return resolved MetricsQL queries for a battery system. - - The queries are templated in BatterySystemConfig.queries and resolved - with the device serial number. Frontend can use these to query the - timeseries backend directly via /api/v1/query_range. - """ - battery = next((b for b in battery_systems if b.id == battery_id), None) - if not battery: - raise HTTPException(404, f"Battery system {battery_id} not found") - - device = battery.id - queries = battery.config.queries - - return {field: getattr(queries, field).replace("$device", device) for field in queries.model_fields} +# # ---------------------------- # +# # Power overview (Dashboard) # +# # ---------------------------- # +# +# +# class BatterySystemInfo(BaseModel): +# id: str +# name: str +# +# +# class SystemLayoutData(BaseModel): +# phases: list[int] +# # TODO: grid_labels: list[str] # ["L1", "L2", "L3"] +# has_solar: bool +# battery_systems: list[BatterySystemInfo] +# +# +# @router.get("/system-layout", response_model=SystemLayoutData) +# async def get_system_layout(battery_systems: BatterySystemsDep) -> SystemLayoutData: +# return SystemLayoutData( +# phases=[1, 2, 3], +# # grid_labels=["L1", "L2", "L3"], +# has_solar=True, # TODO +# battery_systems=[BatterySystemInfo(id=b.id, name=b.name) for b in battery_systems], +# ) +# +# +# class BatteryPowerValues(BaseModel): +# charger: float | None +# inverter: float | None +# battery: float | None +# losses: float | None +# +# +# class PowerFlowData(BaseModel): +# grid: dict[str, float | None] +# solar: float | None +# consumption: dict[str, float] # e.g. {"L1": 800, "L2": 300, "L3": 200} +# batteries: dict[str, BatteryPowerValues] +# +# +# def _get_instant_value(result: "QueryResult") -> float | None: +# """Extract the latest value from an instant query result.""" +# if result.series and result.series[0].values: +# return result.series[0].values[-1][1] +# return None +# +# +# @router.get("/power-flow", response_model=PowerFlowData) +# async def get_power_flow( +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# ) -> PowerFlowData: +# if timeseries is not None: +# return await _get_power_flow_timeseries(timeseries, battery_systems) +# return await _get_power_flow_legacy(battery_systems) +# +# +# async def _get_power_flow_timeseries( +# timeseries: "TimeseriesBackend", +# battery_systems: list, +# ) -> PowerFlowData: +# """Get power flow data from timeseries backend.""" +# now = datetime.now(UTC) +# +# # Grid power per phase +# grid_power: dict[str, float | None] = {} +# for phase in ("L1", "L2", "L3"): +# query = f'openess_power_watts{{from="grid", phase="{phase}"}}' +# result = timeseries.query(query, now) +# grid_power[phase] = _get_instant_value(result) +# +# # Solar power +# solar_query = 'openess_power_watts{from="pvinverter"}' +# solar_result = timeseries.query(solar_query, now) +# solar_power = _get_instant_value(solar_result) +# +# # Battery power for each system +# batteries: dict[str, BatteryPowerValues] = {} +# for battery_system in battery_systems: +# device = battery_system.id +# +# # AC power (charger/inverter) +# ac_in_query = battery_system.config.queries.power_ac_in.replace("$device", device) +# ac_in_result = timeseries.query(ac_in_query, now) +# system = _get_instant_value(ac_in_result) or 0 +# +# charger = -system if system < 0 else 0 +# inverter = system if system > 0 else 0 +# +# # DC battery power +# battery_query = battery_system.config.queries.power_battery.replace("$device", device) +# battery_result = timeseries.query(battery_query, now) +# battery = _get_instant_value(battery_result) or 0 +# +# losses = battery - system +# +# batteries[battery_system.id] = BatteryPowerValues( +# charger=charger, +# inverter=inverter, +# battery=battery, +# losses=losses, +# ) +# +# return PowerFlowData( +# grid=grid_power, +# solar=solar_power, +# consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, +# batteries=batteries, +# ) +# +# +# async def _get_power_flow_legacy(battery_systems: list) -> PowerFlowData: +# """Get power flow data from legacy database.""" +# start = datetime.now(UTC) - timedelta(seconds=10) +# +# grid_power: dict[str, float | None] = {} +# for i in (1, 2, 3): +# power = None +# result = db.get_power(f"grid/power/l{i}", start=start, bucket_seconds=None) +# if result: +# _, power = result[-1] +# grid_power[f"L{i}"] = power +# +# solar_power = None +# result = db.get_power("victron/pvinverter/31/power/l1", start=start, bucket_seconds=None) +# if result: +# _, solar_power = result[-1] +# +# batteries: dict[str, BatteryPowerValues] = {} +# for battery_system in battery_systems: +# charger = 0 +# inverter = 0 +# battery = 0 +# losses = 0 +# system = 0 +# result = db.get_power(battery_system.config.metrics.power_to_system, start=start, bucket_seconds=None) +# if result: +# _, system = result[-1] +# if system < 0: +# charger = -system +# if system > 0: +# inverter = system +# +# result = db.get_power(battery_system.config.metrics.power_to_battery, start=start, bucket_seconds=None) +# if result: +# _, battery = result[-1] +# losses = battery - system +# +# batteries[battery_system.id] = BatteryPowerValues( +# charger=charger, +# inverter=inverter, +# battery=battery, +# losses=losses, +# ) +# +# return PowerFlowData( +# grid=grid_power, +# solar=solar_power, +# consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, +# batteries=batteries, +# ) +# +# +# # ------------------------------- # +# # Services overview (Dashboard) # +# # ------------------------------- # +# +# +# class Status(StrEnum): +# OK = "ok" +# WARNING = "warning" +# ERROR = "error" +# +# +# class ServiceMessage(BaseModel): +# timestamp: datetime +# status: Status +# message: str +# +# +# class ServiceStatus(BaseModel): +# status: Status +# messages: list[ServiceMessage] +# +# +# class ServicesStatusResponse(BaseModel): +# database: ServiceStatus | None +# optimizer: ServiceStatus | None +# +# +# @router.get("/services-status", response_model=ServicesStatusResponse) +# async def services_status() -> ServicesStatusResponse: +# try: +# return ServicesStatusResponse( +# database=ServiceStatus(status=Status.OK, messages=[]), +# optimizer=ServiceStatus(status=Status.OK, messages=[]), +# ) +# except Exception as e: +# logger.exception("Health check failed") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# @router.get("/battery-ids", response_model=list[str]) +# async def get_battery_ids(battery_systems: BatterySystemsDep) -> list[str]: +# try: +# return [s.id for s in battery_systems] +# except Exception as e: +# logger.exception("Failed to get battery ids") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# # ------------------------ # +# # Metrics page endpoints # +# # ------------------------ # +# +# +# class BatteryEnergySeries(BaseModel): +# energy_to_charger: list[float | None] = [] +# energy_from_inverter: list[float | None] = [] +# energy_to_battery: list[float | None] = [] +# energy_from_battery: list[float | None] = [] +# energy_loss_to_battery: list[float | None] = [] +# energy_loss_from_battery: list[float | None] = [] +# +# +# class EnergyGraphResponse(BaseModel): +# timestamps: list[datetime] +# +# grid_import: dict[str, list[float | None]] +# grid_export: dict[str, list[float | None]] +# +# battery_systems: dict[str, BatteryEnergySeries] +# +# solar: list[float | None] = [] +# to_consumption: list[float | None] = [] +# from_consumption: list[float | None] = [] +# +# +# @router.get("/energy-graph", response_model=EnergyGraphResponse) +# async def get_energy_flow_endpoint( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# battery_id: str | None = Query(default=None), +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# bucket_minutes: int = Query(default=60), +# ) -> EnergyGraphResponse: +# try: +# battery_system = None +# if battery_id: +# for bs in battery_systems: +# if bs.id == battery_id: +# battery_system = bs +# break +# elif len(battery_systems) == 1: +# battery_system = battery_systems[0] +# +# if battery_system is None: +# if battery_id: +# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") +# else: +# raise HTTPException(status_code=400, detail="Please provide a battery_id") +# +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(hours=24) +# if end is None: +# end = now +# +# if timeseries is not None: +# return await _get_energy_graph_timeseries(timeseries, battery_system, start, end, bucket_minutes) +# +# return await _get_energy_graph_legacy(db, battery_system, start, end, bucket_minutes) +# except Exception as e: +# logger.exception("Failed to get energy flow") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# async def _get_energy_graph_timeseries( +# timeseries: "TimeseriesBackend", +# battery_system, +# start: datetime, +# end: datetime, +# bucket_minutes: int, +# ) -> EnergyGraphResponse: +# """Get energy graph data from timeseries backend.""" +# device = battery_system.id +# step = f"{bucket_minutes}m" +# +# # Query energy series using increase() to get per-bucket energy consumption +# queries = battery_system.config.queries +# grid_import_query = queries.energy_grid_import.replace("$device", device) +# grid_export_query = queries.energy_grid_export.replace("$device", device) +# to_mp_query = queries.energy_to_battery.replace("$device", device) +# from_mp_query = queries.energy_from_battery.replace("$device", device) +# +# # Use increase() to get energy delta per bucket +# grid_import_result = timeseries.query_range(f"increase({grid_import_query}[{step}])", start, end, step) +# grid_export_result = timeseries.query_range(f"increase({grid_export_query}[{step}])", start, end, step) +# to_mp_result = timeseries.query_range(f"increase({to_mp_query}[{step}])", start, end, step) +# from_mp_result = timeseries.query_range(f"increase({from_mp_query}[{step}])", start, end, step) +# +# # Convert to dict for easier lookup +# def result_to_dict(result: "QueryResult") -> dict[datetime, float]: +# if not result.series or not result.series[0].values: +# return {} +# return {ts: val for ts, val in result.series[0].values} +# +# grid_import_data = result_to_dict(grid_import_result) +# grid_export_data = result_to_dict(grid_export_result) +# to_mp_data = result_to_dict(to_mp_result) +# from_mp_data = result_to_dict(from_mp_result) +# +# # Collect all timestamps +# all_timestamps: set[datetime] = set() +# all_timestamps.update(grid_import_data.keys()) +# all_timestamps.update(grid_export_data.keys()) +# all_timestamps.update(to_mp_data.keys()) +# all_timestamps.update(from_mp_data.keys()) +# timestamps = sorted(all_timestamps) +# +# # Build response series +# grid_exports: dict[str, list[float | None]] = {"From MP": []} +# grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} +# battery_stats = BatteryEnergySeries() +# +# for ts in timestamps: +# from_mp = from_mp_data.get(ts) +# grid_exports["From MP"].append(round(from_mp, 3) if from_mp else None) +# unaccounted_export = grid_export_data.get(ts, 0) - (from_mp or 0) +# +# to_mp = to_mp_data.get(ts) +# grid_imports["To MP"].append(round(to_mp, 3) if to_mp else None) +# grid_import = grid_import_data.get(ts) +# if grid_import is not None: +# grid_import -= (to_mp or 0) - unaccounted_export +# grid_imports["Consumption"].append(round(grid_import, 3) if grid_import else None) +# +# battery_stats.energy_to_charger.append(round(to_mp, 3) if to_mp else None) +# battery_stats.energy_from_inverter.append(round(from_mp, 3) if from_mp else None) +# +# return EnergyGraphResponse( +# timestamps=timestamps, +# grid_export=grid_exports, +# grid_import=grid_imports, +# battery_systems={battery_system.config.name: battery_stats}, +# ) +# +# +# async def _get_energy_graph_legacy( +# db: "DatabaseConnection", +# battery_system, +# start: datetime, +# end: datetime, +# bucket_minutes: int, +# ) -> EnergyGraphResponse: +# """Get energy graph data from legacy database.""" +# series = { +# "grid_import": db.get_energy_aggregated( +# "grid/energy/import/total", bucket_minutes * 60, start, end, center_buckets=True +# ), +# "grid_export": db.get_energy_aggregated( +# "grid/energy/export/total", bucket_minutes * 60, start, end, center_buckets=True +# ), +# "vebus_228_import": db.get_energy_aggregated( +# battery_system.config.metrics.energy_to_system, bucket_minutes * 60, start, end, center_buckets=True +# ), +# "vebus_228_export": db.get_energy_aggregated( +# battery_system.config.metrics.energy_from_system, bucket_minutes * 60, start, end, center_buckets=True +# ), +# } +# +# timestamps: set[datetime] = set() +# series_as_dict: dict[str, dict[datetime, float]] = {name: {} for name in series} +# for name, s in series.items(): +# for ts, v in s: +# timestamps.add(ts) +# series_as_dict[name][ts] = v +# sorted_timestamps = sorted(timestamps) +# +# grid_exports: dict[str, list[float | None]] = {"From MP": []} +# grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} +# battery_stats = BatteryEnergySeries() +# +# for ts in sorted_timestamps: +# from_mp = series_as_dict["vebus_228_export"].get(ts) +# grid_exports["From MP"].append(from_mp) +# unaccounted_export = series_as_dict["grid_export"].get(ts, 0) - (from_mp or 0) +# +# to_mp = series_as_dict["vebus_228_import"].get(ts) +# grid_imports["To MP"].append(to_mp) +# grid_import = series_as_dict["grid_import"].get(ts) +# if grid_import is not None: +# grid_import -= (to_mp or 0) - unaccounted_export +# grid_imports["Consumption"].append(grid_import) +# +# battery_stats.energy_to_charger.append(to_mp) +# battery_stats.energy_from_inverter.append(from_mp) +# +# return EnergyGraphResponse( +# timestamps=sorted_timestamps, +# grid_export=grid_exports, +# grid_import=grid_imports, +# battery_systems={"MultiPlus": battery_stats}, +# ) +# +# +# def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: +# """Calculate query step from aggregate_minutes or time range.""" +# if aggregate_minutes > 1: +# return f"{aggregate_minutes}m" +# # Auto-calculate based on range +# duration = (end - start).total_seconds() +# if duration <= 3600: # 1 hour +# return "1m" +# if duration <= 6 * 3600: # 6 hours +# return "5m" +# if duration <= 24 * 3600: # 24 hours +# return "15m" +# return "1h" +# +# +# @router.get("/power-graph", response_model=PowerResponse) +# async def get_power_graph( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# battery_id: str | None = Query(default=None), +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# aggregate_minutes: int = Query(default=1), +# ) -> PowerResponse: +# try: +# battery_system = None +# if battery_id: +# for bs in battery_systems: +# if bs.id == battery_id: +# battery_system = bs +# break +# elif len(battery_systems) == 1: +# battery_system = battery_systems[0] +# +# if battery_system is None: +# if battery_id: +# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") +# else: +# raise HTTPException(status_code=400, detail="Please provide a battery_id") +# +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(hours=24) +# if end is None: +# end = now +# +# # Schedule always comes from database +# schedule_series: list[tuple[datetime, float]] = [] +# for ts_start, ts_end, v, _ in db.get_schedule(battery_system.config.id, start): +# schedule_series.extend([(ts_start, v), (ts_end, v)]) +# +# if timeseries is not None: +# device = battery_system.id +# step = _calculate_step(start, end, aggregate_minutes) +# +# series: dict[str, TimeSeries] = {} +# +# # Grid power per phase +# for phase in ("L1", "L2", "L3"): +# query = f'openess_power_watts{{from="grid", phase="{phase}", device="{device}"}}' +# result = timeseries.query_range(query, start, end, step) +# series[f"Grid {phase}"] = query_result_to_timeseries(result) +# +# # AC power (to/from MultiPlus) +# ac_query = battery_system.config.queries.power_ac_in.replace("$device", device) +# ac_result = timeseries.query_range(ac_query, start, end, step) +# series["To MP"] = query_result_to_timeseries(ac_result) +# +# # Battery DC power +# battery_query = battery_system.config.queries.power_battery.replace("$device", device) +# battery_result = timeseries.query_range(battery_query, start, end, step) +# series["Battery"] = query_result_to_timeseries(battery_result) +# +# # Solar power (negated for display) +# solar_query = battery_system.config.queries.power_pv.replace("$device", device) +# solar_result = timeseries.query_range(solar_query, start, end, step) +# solar_ts = query_result_to_timeseries(solar_result) +# series["Solar"] = TimeSeries( +# timestamps=solar_ts.timestamps, +# values=[-v for v in solar_ts.values], +# ) +# +# series["Schedule"] = data_to_timeseries(schedule_series) +# return PowerResponse(series=series) +# +# # Legacy database queries +# bucket_seconds = aggregate_minutes * 60 +# legacy_series: dict[str, list[tuple[datetime, float]]] = { +# f"Grid L{i}": db.get_power(f"grid/power/l{i}", start, end, bucket_seconds) for i in (1, 2, 3) +# } +# legacy_series["To MP"] = db.get_power(battery_system.config.metrics.power_to_system, start, end, bucket_seconds) +# legacy_series["Battery"] = db.get_power( +# battery_system.config.metrics.power_to_battery, start, end, bucket_seconds +# ) +# legacy_series["Solar"] = [ +# (t, -p) for t, p in db.get_power("victron/pvinverter/31/power/l1", start, end, bucket_seconds) +# ] +# legacy_series["Schedule"] = schedule_series +# +# return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) +# except Exception as e: +# logger.exception("Failed to get power data") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# class PricePoint(BaseModel): +# time: datetime +# market: float | None +# buy: float | None +# sell: float | None +# +# +# class PricesResponse(BaseModel): +# area: str +# aggregate_minutes: int +# unit: str = "€/kWh" # TODO: based on area +# timeseries: list[PricePoint] +# +# +# @router.get("/prices", response_model=PricesResponse) +# async def get_price_data( +# db: Database, +# price_config: PriceConfigDep, +# area: str | None = Query(default=None), +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# aggregate_minutes: int | None = Query(default=None), +# ) -> PricesResponse: +# try: +# if area is None: +# area = price_config.area +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(days=7) +# if end is None: +# end = now + timedelta(days=2) +# if aggregate_minutes is None: +# aggregate_minutes = price_config.aggregate_minutes +# +# timeseries = [] +# for timestamp, price in db.get_prices(area, start, end, aggregate_minutes=aggregate_minutes): +# timeseries.append( +# PricePoint( +# time=timestamp, +# market=round(price, 4), +# buy=round(price_config.buy_price(price), 4), +# sell=round(price_config.sell_price(price), 4), +# ) +# ) +# +# return PricesResponse( +# area=area, +# aggregate_minutes=aggregate_minutes, +# timeseries=timeseries, +# ) +# except Exception as e: +# logger.exception("Failed to get prices") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# class BatteryGraphResponse(BaseModel): +# soc: TimeSeries +# schedule: TimeSeries # Scheduled (past and future) SoC +# voltage: TimeSeries +# +# +# @router.get("/battery-graph", response_model=dict[str, BatteryGraphResponse]) +# async def get_battery_graph( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# battery_id: str | None = Query(default=None), +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# ) -> dict[str, BatteryGraphResponse]: +# try: +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(hours=48) +# if end is None: +# end = now + timedelta(hours=24) +# +# result = {} +# for battery_system in battery_systems: +# if battery_id is not None and battery_system.config.id != battery_id: +# continue +# +# # Schedule still comes from database (not in timeseries) +# scheduled = [(t, soc) for _, t, _, soc in db.get_schedule(battery_system.config.id, start)] +# +# # SOC and voltage from timeseries backend +# if timeseries is not None: +# device = battery_system.id +# soc_query = battery_system.config.queries.soc.replace("$device", device) +# voltage_query = battery_system.config.queries.voltage.replace("$device", device) +# +# soc_result = timeseries.query_range(soc_query, start, end, step="1m") +# voltage_result = timeseries.query_range(voltage_query, start, end, step="1m") +# +# result[battery_system.config.name] = BatteryGraphResponse( +# soc=query_result_to_timeseries(soc_result, rounding=1), +# schedule=data_to_timeseries(scheduled, rounding=1), +# voltage=query_result_to_timeseries(voltage_result, rounding=2), +# ) +# else: +# # Fallback to legacy database queries +# soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) +# voltage = db.get_voltage(battery_system.config.metrics.battery_voltage, start, end, bucket_seconds=60) +# +# result[battery_system.config.name] = BatteryGraphResponse( +# soc=data_to_timeseries(soc, rounding=1), +# schedule=data_to_timeseries(scheduled, rounding=1), +# voltage=data_to_timeseries(voltage, rounding=2), +# ) +# return result +# except Exception as e: +# logger.exception("Failed to get battery SOC") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# # ---------------# +# # Cycles page # +# # ---------------# +# +# +# class EfficiencyScatterPoint(BaseModel): +# time: datetime +# battery_power: float +# inverter_charger_power: float +# losses: float +# efficiency: float | None +# soc: int | None +# category: str +# +# +# @router.get("/efficiency-scatter", response_model=list[EfficiencyScatterPoint]) +# async def get_efficiency_scatter( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# battery_id: str | None = Query(default=None), +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# aggregate_minutes: int = Query(default=10), +# idle_threshold: int = Query(default=5), +# ) -> list[EfficiencyScatterPoint]: +# try: +# battery_system = None +# if battery_id: +# for bs in battery_systems: +# if bs.id == battery_id: +# battery_system = bs +# break +# elif len(battery_systems) == 1: +# battery_system = battery_systems[0] +# +# if battery_system is None: +# if battery_id: +# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") +# else: +# raise HTTPException(status_code=400, detail="Please provide a battery_id") +# +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(days=7) +# if end is None: +# end = now +# +# if timeseries is not None: +# return await _get_efficiency_scatter_timeseries( +# timeseries, battery_system, start, end, aggregate_minutes, idle_threshold +# ) +# +# return await _get_efficiency_scatter_legacy(db, battery_system, start, end, aggregate_minutes, idle_threshold) +# except Exception as e: +# logger.exception("Failed to get efficiency scatter data") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# async def _get_efficiency_scatter_timeseries( +# timeseries: "TimeseriesBackend", +# battery_system, +# start: datetime, +# end: datetime, +# aggregate_minutes: int, +# idle_threshold: int, +# ) -> list[EfficiencyScatterPoint]: +# """Get efficiency scatter data from timeseries backend.""" +# device = battery_system.id +# step = f"{aggregate_minutes}m" +# queries = battery_system.config.queries +# +# # Query AC in, AC out, and battery DC power +# ac_in_query = queries.power_ac_in.replace("$device", device) +# ac_out_query = queries.power_ac_out.replace("$device", device) +# dc_query = queries.power_battery.replace("$device", device) +# +# ac_in_result = timeseries.query_range(ac_in_query, start, end, step) +# ac_out_result = timeseries.query_range(ac_out_query, start, end, step) +# dc_result = timeseries.query_range(dc_query, start, end, step) +# +# # Convert to dicts +# def result_to_dict(result: "QueryResult") -> dict[datetime, float]: +# if not result.series or not result.series[0].values: +# return {} +# return {ts: val for ts, val in result.series[0].values} +# +# ac_in_data = result_to_dict(ac_in_result) +# ac_out_data = result_to_dict(ac_out_result) +# dc_data = result_to_dict(dc_result) +# +# # Merge data by timestamp +# all_timestamps = set(ac_in_data.keys()) & set(ac_out_data.keys()) & set(dc_data.keys()) +# +# points = [] +# for ts in sorted(all_timestamps): +# ac = ac_in_data[ts] - ac_out_data[ts] +# dc = dc_data[ts] +# +# if abs(dc) < idle_threshold: +# category = "idling" +# elif dc > 0: +# category = "charging" +# else: +# category = "discharging" +# +# losses = ac - dc +# efficiency = None +# if category == "charging" and ac > 0: +# efficiency = (dc / ac) * 100 +# elif category == "discharging" and dc < 0: +# efficiency = (ac / dc) * 100 +# +# points.append( +# EfficiencyScatterPoint( +# time=ts, +# battery_power=round(abs(dc), 1), +# inverter_charger_power=round(ac, 1), +# losses=round(losses, 1), +# efficiency=round(efficiency, 1) if efficiency is not None else None, +# soc=None, +# category=category, +# ) +# ) +# +# return points +# +# +# async def _get_efficiency_scatter_legacy( +# db: "DatabaseConnection", +# battery_system, +# start: datetime, +# end: datetime, +# aggregate_minutes: int, +# idle_threshold: int, +# ) -> list[EfficiencyScatterPoint]: +# """Get efficiency scatter data from legacy database.""" +# metrics = battery_system.config.metrics +# bucket_seconds = aggregate_minutes * 60 +# +# # Use configured metrics paths +# ac_in_path = metrics.power_to_system +# if isinstance(ac_in_path, list): +# ac_in_path = ac_in_path[0] +# +# # AC out is typically the same vebus but ac_out instead of ac_in +# ac_out_path = ac_in_path.replace("ac_in", "ac_out") if ac_in_path else None +# +# dc_path = metrics.power_to_battery +# if isinstance(dc_path, list): +# dc_path = dc_path[0] +# +# ac_in = db.get_power(ac_in_path, start, end, bucket_seconds=bucket_seconds) if ac_in_path else [] +# ac_out = db.get_power(ac_out_path, start, end, bucket_seconds=bucket_seconds) if ac_out_path else [] +# dc = db.get_power(dc_path, start, end, bucket_seconds=bucket_seconds) if dc_path else [] +# +# data: dict[datetime, list[float | None]] = { +# ts: [v_in - v_out, None] for (ts, v_in), (_, v_out) in zip(ac_in, ac_out, strict=False) +# } +# for ts, v in dc: +# if ts in data: +# data[ts][1] = v +# +# points = [] +# for ts, (ac, dc_val) in data.items(): +# if ac is None or dc_val is None: +# continue +# +# if abs(dc_val) < idle_threshold: +# category = "idling" +# elif dc_val > 0: +# category = "charging" +# else: +# category = "discharging" +# +# losses = ac - dc_val +# efficiency = None +# if category == "charging" and ac > 0: +# efficiency = (dc_val / ac) * 100 +# elif category == "discharging" and dc_val < 0: +# efficiency = (ac / dc_val) * 100 +# +# points.append( +# EfficiencyScatterPoint( +# time=ts, +# battery_power=round(abs(dc_val), 1), +# inverter_charger_power=round(ac, 1), +# losses=round(losses, 1), +# efficiency=round(efficiency, 1) if efficiency is not None else None, +# soc=None, +# category=category, +# ) +# ) +# +# return points +# +# +# class BatteryCycle(BaseModel): +# start_time: datetime +# end_time: datetime +# duration_hours: float +# min_soc: float +# ac_energy_in: float | None +# ac_energy_out: float | None +# dc_energy_in: float +# dc_energy_out: float +# system_efficiency: float | None +# battery_efficiency: float | None +# charger_efficiency: float | None +# inverter_efficiency: float | None +# profit: float | None +# scheduled_profit: float | None +# +# +# @router.get("/cycles", response_model=list[BatteryCycle]) +# async def get_battery_cycles( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# price_config: PriceConfigDep, +# battery_id: str | None = Query(default=None), +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# min_soc_swing: int = Query(default=10), +# ) -> list[BatteryCycle]: +# try: +# battery_system = None +# if battery_id: +# for bs in battery_systems: +# if bs.id == battery_id: +# battery_system = bs +# break +# elif len(battery_systems) == 1: +# battery_system = battery_systems[0] +# +# if battery_system is None: +# if battery_id: +# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") +# else: +# raise HTTPException(status_code=400, detail="Please provide a battery_id") +# +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(days=30) +# if end is None: +# end = now +# +# # Get SOC data from timeseries or legacy database +# if timeseries is not None: +# device = battery_system.id +# soc_query = battery_system.config.queries.soc.replace("$device", device) +# soc_result = timeseries.query_range(soc_query, start, end, step="1m") +# battery_soc = [(ts, val) for ts, val in soc_result.series[0].values] if soc_result.series else [] +# else: +# battery_soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) +# +# raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) +# +# cycles = [] +# for cycle_start, cycle_end, min_soc in raw_cycles: +# duration = (cycle_end - cycle_start).total_seconds() / 3600.0 +# +# # TODO: this is cursed AF and should be fixed +# dc_energy_in = 0.0 +# dc_energy_out = 0.0 +# for _, p in db.get_power(battery_system.config.metrics.power_to_battery[0], cycle_start, cycle_end): +# # for _, p in db.get_power("vebus_228_battery", cycle_start, cycle_end): +# if p > 0: +# dc_energy_in += p +# else: +# dc_energy_out += -p +# dc_energy_in /= 60000 +# dc_energy_out /= 60000 +# +# ac_in_import = db.get_energy( +# battery_system.config.metrics.energy_to_system, cycle_start, cycle_end, normalize=True +# ) +# # ac_out_import = db.get_energy("vebus_228_ac_out_import", cycle_start, cycle_end, normalize=True) +# ac_in_export = db.get_energy( +# battery_system.config.metrics.energy_from_system, cycle_start, cycle_end, normalize=True +# ) +# # ac_out_export = db.get_energy("vebus_228_ac_out_export", cycle_start, cycle_end, normalize=True) +# +# ac_energy_in = 0.0 +# if ac_in_import: +# ac_energy_in += ac_in_import[-1][1] +# # if ac_out_import: +# # ac_energy_in += ac_out_import[-1][1] +# ac_energy_out = 0.0 +# if ac_in_export: +# ac_energy_out += ac_in_export[-1][1] +# # if ac_out_export: +# # ac_energy_out += ac_out_export[-1][1] +# +# profit = 0.0 +# scheduled_profit = 0.0 +# e_in = { +# ts: v +# for ts, v in db.get_energy_aggregated( +# battery_system.config.metrics.energy_to_system, 3600, cycle_start, cycle_end +# ) +# } +# e_out = { +# ts: v +# for ts, v in db.get_energy_aggregated( +# battery_system.config.metrics.energy_from_system, 3600, cycle_start, cycle_end +# ) +# } +# scheduled = {ts: v for ts, _, v, _ in db.get_schedule(battery_system.config.id, cycle_start)} +# for ts, v in db.get_prices(price_config.area, cycle_start, cycle_end, aggregate_minutes=60): +# profit -= price_config.buy_price(v) * e_in.get(ts, 0) +# profit += price_config.sell_price(v) * e_out.get(ts, 0) +# scheduled_power = scheduled.get(ts, 0) +# if scheduled_power > 0: +# scheduled_profit -= price_config.buy_price(v) * scheduled_power / 1000 +# if scheduled_power < 0: +# scheduled_profit += price_config.sell_price(v) * -scheduled_power / 1000 +# +# cycles.append( +# BatteryCycle( +# start_time=cycle_start, +# end_time=cycle_end, +# duration_hours=round(duration, 2), +# min_soc=round(min_soc, 1), +# ac_energy_in=round(ac_energy_in, 2) if ac_energy_in else None, +# ac_energy_out=round(ac_energy_out, 2) if ac_energy_out else None, +# dc_energy_in=round(dc_energy_in, 2), +# dc_energy_out=round(dc_energy_out, 2), +# system_efficiency=round(ac_energy_out / ac_energy_in * 100, 1) if ac_energy_in else None, +# battery_efficiency=round(dc_energy_out / dc_energy_in * 100, 1) if dc_energy_in else None, +# charger_efficiency=round(dc_energy_in / ac_energy_in * 100, 1) if ac_energy_in else None, +# inverter_efficiency=round(ac_energy_out / dc_energy_out * 100, 1) if dc_energy_out else None, +# profit=round(profit, 2), +# scheduled_profit=round(scheduled_profit, 2), +# ) +# ) +# +# return cycles +# except Exception as e: +# logger.exception("Failed to get battery cycles") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# # -------------------------# +# # Generalized endpoints # +# # -------------------------# +# +# +# # TODO: add parameter to select subset of series +# @router.get("/power", response_model=PowerResponse) +# async def get_power( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# aggregate_minutes: int = Query(default=1), +# ) -> PowerResponse: +# try: +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(hours=24) +# if end is None: +# end = now +# +# if timeseries is not None and battery_systems: +# # Query all power metrics from timeseries +# step = _calculate_step(start, end, aggregate_minutes) +# series: dict[str, TimeSeries] = {} +# +# for battery_system in battery_systems: +# device = battery_system.id +# queries = battery_system.config.queries +# prefix = battery_system.config.name or device +# +# # Query each power metric +# power_queries = { +# "Grid": queries.power_grid_total, +# "Grid L1": f'openess_power_watts{{from="grid", phase="L1", device="{device}"}}', +# "Grid L2": f'openess_power_watts{{from="grid", phase="L2", device="{device}"}}', +# "Grid L3": f'openess_power_watts{{from="grid", phase="L3", device="{device}"}}', +# "PV": queries.power_pv, +# "Battery": queries.power_battery, +# "AC In": queries.power_ac_in, +# "AC Out": queries.power_ac_out, +# } +# +# for name, query in power_queries.items(): +# resolved_query = query.replace("$device", device) +# result = timeseries.query_range(resolved_query, start, end, step) +# label = f"{prefix}/{name}" if len(battery_systems) > 1 else name +# series[label] = query_result_to_timeseries(result) +# +# return PowerResponse(series=series) +# +# # Legacy database fallback +# legacy_series = db.get_all_power(start, end, aggregate_minutes * 60) +# return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) +# except Exception as e: +# logger.exception("Failed to get debug power flows") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# # TODO: add parameter to select subset of series +# # TODO: add normalize parameter +# @router.get("/energy", response_model=EnergyResponse) +# async def get_energy( +# db: Database, +# timeseries: TimeseriesDep, +# battery_systems: BatterySystemsDep, +# start: datetime | None = Query(default=None), +# end: datetime | None = Query(default=None), +# bucket_minutes: int = Query(default=60), +# ) -> EnergyResponse: +# try: +# now = datetime.now(UTC) +# if start is None: +# start = now - timedelta(hours=24) +# if end is None: +# end = now +# +# if timeseries is not None and battery_systems: +# # Query all energy metrics from timeseries +# step = f"{bucket_minutes}m" +# series: dict[str, TimeSeries] = {} +# +# for battery_system in battery_systems: +# device = battery_system.id +# queries = battery_system.config.queries +# prefix = battery_system.config.name or device +# +# energy_queries = { +# "Grid Import": queries.energy_grid_import, +# "Grid Export": queries.energy_grid_export, +# "To Battery": queries.energy_to_battery, +# "From Battery": queries.energy_from_battery, +# } +# +# for name, query in energy_queries.items(): +# resolved_query = query.replace("$device", device) +# # Use increase() to get energy delta per bucket +# result = timeseries.query_range(f"increase({resolved_query}[{step}])", start, end, step) +# label = f"{prefix}/{name}" if len(battery_systems) > 1 else name +# series[label] = query_result_to_timeseries(result, rounding=3) +# +# return EnergyResponse(series=series) +# +# # Legacy database fallback +# legacy_series = db.get_all_energy(start, end, normalize=True) +# +# # Get integrated power flows +# for label in db.get_power_labels(start, end): +# legacy_series[f"{label} [integrated]"] = db.integrate_power(label, start, end) +# +# return EnergyResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) +# except Exception as e: +# logger.exception("Failed to get debug energy flows") +# raise HTTPException(status_code=500, detail=str(e)) from e +# +# +# # -------------------------- # +# # Timeseries query helpers # +# # -------------------------- # +# +# +# @router.get("/queries/{battery_id}") +# async def get_queries( +# battery_id: str, +# battery_systems: BatterySystemsDep, +# ) -> dict[str, str]: +# """Return resolved MetricsQL queries for a battery system. +# +# The queries are templated in BatterySystemConfig.queries and resolved +# with the device serial number. Frontend can use these to query the +# timeseries backend directly via /api/v1/query_range. +# """ +# battery = next((b for b in battery_systems if b.id == battery_id), None) +# if not battery: +# raise HTTPException(404, f"Battery system {battery_id} not found") +# +# device = battery.id +# queries = battery.config.queries +# +# return {field: getattr(queries, field).replace("$device", device) for field in queries.model_fields} diff --git a/open_ess/main.py b/open_ess/main.py index 36e7afc..98677c3 100644 --- a/open_ess/main.py +++ b/open_ess/main.py @@ -1,18 +1,11 @@ import logging import signal -import uvicorn - -from open_ess.battery_system import BatterySystem, VictronBatterySystem from open_ess.config import Config -from open_ess.database import Database, DatabaseService -from open_ess.frontend import create_app -from open_ess.optimizer import OptimizerService from open_ess.pricing import EntsoeService from open_ess.service import ServiceManager from open_ess.timeseries import TimeseriesBackend, create_backend -from open_ess.util import EndpointFilter, parse_args, setup_logging -from open_ess.victron_modbus import VictronService +from open_ess.util import parse_args, setup_logging setup_logging() logger = logging.getLogger(__name__) @@ -22,32 +15,28 @@ def main() -> None: args = parse_args("Open Energy Storage System - optimize charging based on day-ahead prices") config = Config.from_file(args.config) - database = Database(config.database) - database.run_migrations() - # Create timeseries backend - timeseries: TimeseriesBackend = create_backend(config.timeseries, db_path=config.database.path) + timeseries: TimeseriesBackend = create_backend(config.timeseries) logger.info(f"Using timeseries backend: {config.timeseries.backend}") # Create services service_manager = ServiceManager() - service_manager.register_service(DatabaseService(database)) - service_manager.register_service(EntsoeService(database, config.prices)) - battery_systems: list[BatterySystem] = [] - for battery_config in config.battery_systems: - if battery_config.is_victron: - victron_service = VictronService(database, battery_config, timeseries) - service_manager.register_service(victron_service) - battery_system = VictronBatterySystem(battery_config, victron_service.client) - battery_systems.append(battery_system) - service_manager.register_service( - OptimizerService( - database, - battery_system=battery_system, - price_config=config.prices, - ), - requires=victron_service, - ) + service_manager.register_service(EntsoeService(timeseries, config.prices)) + # battery_systems: list[BatterySystem] = [] + # for battery_config in config.battery_systems: + # if battery_config.is_victron: + # victron_service = VictronService(timeseries, battery_config, timeseries) + # service_manager.register_service(victron_service) + # battery_system = VictronBatterySystem(battery_config, victron_service.client) + # battery_systems.append(battery_system) + # service_manager.register_service( + # OptimizerService( + # timeseries, + # battery_system=battery_system, + # price_config=config.prices, + # ), + # requires=victron_service, + # ) # Shutdown handler def shutdown(signum: int, frame: object) -> None: @@ -61,18 +50,18 @@ def shutdown(signum: int, frame: object) -> None: service_manager.start() # Frontend - if config.frontend.enable: - logger.info(f"Starting web server on http://{config.frontend.host}:{config.frontend.port}") - - logging.getLogger("uvicorn.access").addFilter(EndpointFilter(["/api/power-flow"])) - - app = create_app(database, config, battery_systems) - uvicorn.run( - app, - host=config.frontend.host, - port=config.frontend.port, - log_level="info", - ) + # if config.frontend.enable: + # logger.info(f"Starting web server on http://{config.frontend.host}:{config.frontend.port}") + # + # logging.getLogger("uvicorn.access").addFilter(EndpointFilter(["/api/power-flow"])) + # + # app = create_app(timeseries, config, battery_systems) + # uvicorn.run( + # app, + # host=config.frontend.host, + # port=config.frontend.port, + # log_level="info", + # ) service_manager.wait_for_stop() logger.info("Shutdown complete") diff --git a/open_ess/optimizer/optimizer.py b/open_ess/optimizer/optimizer.py index ce9195b..d0b0b3d 100644 --- a/open_ess/optimizer/optimizer.py +++ b/open_ess/optimizer/optimizer.py @@ -9,8 +9,8 @@ from pyomo.opt import SolverFactory from open_ess.battery_system import BatterySystemConfig -from open_ess.database import DatabaseConnection from open_ess.pricing import PriceConfig +from open_ess.timeseries import TimeseriesBackend logger = logging.getLogger(__name__) @@ -26,8 +26,8 @@ class Optimizer: the need for a separate binary variable. """ - def __init__(self, db: DatabaseConnection, price_config: PriceConfig, battery_config: BatterySystemConfig): - self._database = db + def __init__(self, mql_client: TimeseriesBackend, price_config: PriceConfig, battery_config: BatterySystemConfig): + self._mql_client = mql_client self._price_config = price_config self._battery_config = battery_config # TODO: check for cbc diff --git a/open_ess/optimizer/service.py b/open_ess/optimizer/service.py index 136bdc3..6c65a95 100644 --- a/open_ess/optimizer/service.py +++ b/open_ess/optimizer/service.py @@ -2,9 +2,9 @@ from datetime import UTC, datetime, timedelta from open_ess.battery_system import BatterySystem -from open_ess.database import Database, DatabaseConnection from open_ess.pricing import PriceConfig from open_ess.service import Service +from open_ess.timeseries import TimeseriesBackend from .optimizer import Optimizer @@ -14,26 +14,24 @@ class OptimizerService(Service): def __init__( self, - db: Database, + mql_client: TimeseriesBackend, battery_system: BatterySystem, price_config: PriceConfig, ): super().__init__("OptimizerService") - self._db = db + self._mql_client = mql_client self._battery_system = battery_system self._price_config = price_config - self._db_conn: DatabaseConnection | None = None self._optimizer: Optimizer | None = None def on_start(self) -> None: - self._db_conn = self._db.connect() self._optimizer = Optimizer( - self._db_conn, price_config=self._price_config, battery_config=self._battery_system.config + self._mql_client, price_config=self._price_config, battery_config=self._battery_system.config ) def tick(self) -> None: - if self._optimizer is None or self._db_conn is None: + if self._optimizer is None: return logger.debug("Running charge optimizer(s)") diff --git a/open_ess/pricing/client.py b/open_ess/pricing/client.py index edf0031..565de32 100644 --- a/open_ess/pricing/client.py +++ b/open_ess/pricing/client.py @@ -8,7 +8,7 @@ from entsoe.utils import add_timestamps, extract_records from pandas import DataFrame -from open_ess.database import DatabaseConnection +from open_ess.timeseries import Sample, TimeseriesBackend from .areas import AREAS from .config import PriceConfig @@ -17,21 +17,19 @@ class EntsoeClient: - def __init__(self, config: PriceConfig, db: DatabaseConnection): + def __init__(self, config: PriceConfig, mql_client: TimeseriesBackend): if config.area not in AREAS: raise ValueError(f"Unknown area code: '{config.area}'") self._config = config - self._db = db - - self._eic_code, tz_name = AREAS[config.area] - self._tz = ZoneInfo(tz_name) + self._mql_client = mql_client if config.entsoe_api_key: set_config(security_token=config.entsoe_api_key) + @staticmethod def fetch_day_ahead_prices( - self, + area: str, start: datetime, end: datetime, ) -> list[tuple[datetime, datetime, float]]: @@ -39,29 +37,33 @@ def fetch_day_ahead_prices( Fetch day-ahead prices from ENTSO-E for a given area and time range. Args: + area: start: Start datetime (UTC) end: End datetime (UTC) Returns: List of (start_time, end_time, price) tuples with prices in EUR/MWh """ + eic_code, tz_name = AREAS[area] + tz = ZoneInfo(tz_name) # ENTSO-E expects timestamps formatted as YYYYMMDDhhmm in the area's local timezone - start_local = start.astimezone(self._tz) - end_local = end.astimezone(self._tz) + start_local = start.astimezone(tz) + end_local = end.astimezone(tz) period_start = int(start_local.strftime("%Y%m%d%H%M")) period_end = int(end_local.strftime("%Y%m%d%H%M")) try: result = EnergyPrices( - in_domain=self._eic_code, - out_domain=self._eic_code, + in_domain=eic_code, + out_domain=eic_code, period_start=period_start, period_end=period_end, ).query_api() except ET.ParseError: - # On a 404, entsoe-apy still tries to parse the result which fails. Other errors such as 503 and timeouts are retried + # On a 404, entsoe-apy still tries to parse the result which fails. + # Other errors such as 503 and timeouts are retried. return [] records = extract_records(result) @@ -79,23 +81,45 @@ def fetch_day_ahead_prices( prices.append((row_start, row_end, price)) return prices - def fetch_missing_prices(self) -> None: + def fetch_missing_prices(self, area: str) -> None: now = datetime.now(UTC) end_of_tomorrow = (now + timedelta(days=2)).replace(hour=0, minute=0, second=0, microsecond=0) - latest = self._db.get_latest_price_time(self._config.area) - if latest is None: + # TODO: prevent injection via area variable + result = self._mql_client.query(f'openess_prices{{area="{area}"}}') + if len(result.series) == 0: fetch_start = now.replace(hour=0, minute=0, second=0, microsecond=0) - fetch_start -= timedelta(days=14) - elif latest >= end_of_tomorrow: - return + fetch_start -= timedelta(weeks=6) else: - fetch_start = latest + latest, _ = result.series[0].values + if latest >= end_of_tomorrow: + return + else: + fetch_start = latest logger.info(f"Fetching prices for {self._config.area} from {fetch_start} to {end_of_tomorrow}") - prices = self.fetch_day_ahead_prices(fetch_start, end_of_tomorrow) + prices = self.fetch_day_ahead_prices(area, fetch_start, end_of_tomorrow) if prices: - self._db.insert_prices(self._config.area, prices) + self._upsert_prices(area, prices) + + def _upsert_prices(self, area: str, prices: list[tuple[datetime, datetime, float]]): + def make_sample(_ts: datetime, _price_type: str, _price: float): + return Sample( + metric="openess_prices", + labels={ + "area": area, + "price": _price_type, + }, + timestamp=_ts, + value=_price, + ) + + samples: list[Sample] = [] + for ts, _, price in prices: + samples.append(make_sample(ts, "market", price)) + samples.append(make_sample(ts, "buy", self._config.buy_price(price))) + samples.append(make_sample(ts, "sell", self._config.sell_price(price))) + self._mql_client.write(samples) def _parse_resolution(resolution: str) -> int: diff --git a/open_ess/pricing/service.py b/open_ess/pricing/service.py index bcb9b10..3e5a619 100644 --- a/open_ess/pricing/service.py +++ b/open_ess/pricing/service.py @@ -1,7 +1,7 @@ import logging -from open_ess.database import Database, DatabaseConnection from open_ess.service import Service +from open_ess.timeseries import TimeseriesBackend from .client import EntsoeClient from .config import PriceConfig @@ -10,19 +10,15 @@ class EntsoeService(Service): - """Fetches day-ahead prices from ENTSO-E at regular intervals.""" - - def __init__(self, db: Database, config: PriceConfig): + def __init__(self, mql_client: TimeseriesBackend, config: PriceConfig): super().__init__("EntsoeService") - self._db = db + self._mql_client = mql_client self._config = config self._check_interval = 3600 self._client: EntsoeClient | None = None - self._db_conn: DatabaseConnection | None = None def on_start(self) -> None: - self._db_conn = self._db.connect() - self._client = EntsoeClient(self._config, self._db_conn) + self._client = EntsoeClient(self._config, self._mql_client) self._fetch_prices() def tick(self) -> None: @@ -32,7 +28,7 @@ def _fetch_prices(self) -> None: if self._client is None: return None try: - self._client.fetch_missing_prices() + self._client.fetch_missing_prices(self._config.area) except Exception as e: logger.error(f"Failed to fetch ENTSO-E prices: {e}") diff --git a/open_ess/timeseries/__init__.py b/open_ess/timeseries/__init__.py index 007dab4..2764349 100644 --- a/open_ess/timeseries/__init__.py +++ b/open_ess/timeseries/__init__.py @@ -1,7 +1,3 @@ -"""Timeseries backend abstraction.""" - -from pathlib import Path - from .base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend from .config import TimeseriesConfig from .metricsqlite.config import MetricSQLiteConfig @@ -10,13 +6,11 @@ def create_backend( config: TimeseriesConfig, - db_path: Path | None = None, ) -> TimeseriesBackend: """Create a timeseries backend from config. Args: config: Timeseries configuration (MetricSQLiteConfig or VictoriaMetricsConfig). - db_path: Database path for MetricSQLite backend. Required if using MetricSQLite. Returns: Configured backend instance. @@ -28,9 +22,7 @@ def create_backend( elif isinstance(config, MetricSQLiteConfig): from .metricsqlite.backend import MetricSQLiteBackend - if db_path is None: - raise ValueError("db_path is required for MetricSQLite backend") - return MetricSQLiteBackend(config, db_path) + return MetricSQLiteBackend(config) else: raise ValueError(f"Unknown timeseries config type: {type(config)}") diff --git a/open_ess/timeseries/config.py b/open_ess/timeseries/config.py index caeb115..804803d 100644 --- a/open_ess/timeseries/config.py +++ b/open_ess/timeseries/config.py @@ -1,5 +1,3 @@ -"""Timeseries backend configuration.""" - from typing import Annotated from pydantic import Field diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py index a73272d..cf1e85b 100644 --- a/open_ess/timeseries/metricsqlite/backend.py +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -1,7 +1,4 @@ -"""MetricSQLite timeseries backend implementation.""" - from datetime import datetime -from pathlib import Path from metricsqlite import MetricsQLiteClient from metricsqlite.engine import InstantVector, MatrixResult, RangeVectorResult, ScalarResult @@ -11,17 +8,9 @@ class MetricSQLiteBackend(TimeseriesBackend): - """MetricSQLite backend using SQLite for storage.""" - - def __init__(self, config: MetricSQLiteConfig, db_path: Path): - """Initialize MetricSQLite backend. - - Args: - config: MetricSQLite configuration. - db_path: Path to SQLite database file. - """ + def __init__(self, config: MetricSQLiteConfig): self.config = config - self._client = MetricsQLiteClient(db_path, enable_wal=True) + self._client = MetricsQLiteClient(config.db_path, enable_wal=True) self._client.connect() self._client.create_tables() diff --git a/open_ess/timeseries/metricsqlite/config.py b/open_ess/timeseries/metricsqlite/config.py index d274705..20ad67a 100644 --- a/open_ess/timeseries/metricsqlite/config.py +++ b/open_ess/timeseries/metricsqlite/config.py @@ -1,14 +1,9 @@ -"""MetricSQLite timeseries backend configuration.""" - +from pathlib import Path from typing import Literal from pydantic import BaseModel class MetricSQLiteConfig(BaseModel): - """Configuration for MetricSQLite backend. - - Uses the database path from DatabaseConfig. - """ - backend: Literal["metricsqlite"] = "metricsqlite" + db_path: Path = Path("openess.db") diff --git a/open_ess/timeseries/victoriametrics/config.py b/open_ess/timeseries/victoriametrics/config.py index 5ab2543..3271bd9 100644 --- a/open_ess/timeseries/victoriametrics/config.py +++ b/open_ess/timeseries/victoriametrics/config.py @@ -1,13 +1,9 @@ -"""VictoriaMetrics timeseries backend configuration.""" - from typing import Literal from pydantic import BaseModel class VictoriaMetricsConfig(BaseModel): - """Configuration for VictoriaMetrics backend.""" - backend: Literal["victoriametrics"] url: str username: str | None = None diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index 6ce02a5..601548e 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -3,7 +3,6 @@ from threading import Lock from typing import TYPE_CHECKING -from open_ess.database import Database, DatabaseConnection from open_ess.timeseries import Sample, TimeseriesBackend from .config import VictronConfig @@ -30,19 +29,16 @@ def _get_float(values: dict[Register, float | bytes | None], key: Register) -> f class VictronClient: def __init__( self, - database: Database, config: "BatterySystemConfig", timeseries: TimeseriesBackend | None = None, ): if not isinstance(config.control, VictronConfig): raise TypeError(f"VictronClient requires VictronConfig, got {type(config.control).__name__}") - self._db = database self._config = config self._control: VictronConfig = config.control self._client = VictronModbusClient(self._control) self._timeseries = timeseries - self._db_conn: DatabaseConnection | None = None self._serial: str | None = None self._setpoint: float = 0.0 # In Watt @@ -51,7 +47,6 @@ def __init__( self._lock = Lock() def initialize(self) -> bool: - self._db_conn = self._db.connect() if not self.connect(): return False @@ -103,9 +98,6 @@ def set_ess_setpoint(self, power: float, until: datetime) -> None: self._setpoint_expiration = until def write_setpoints(self) -> None: - if self._db_conn is None: - return - if self._config.monitor_only: return @@ -207,7 +199,7 @@ def add(metric: str, value: float | None, labels: dict[str, str]) -> None: add( POWER_METRIC, _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1), - {"from": "ac_out", "to": "system", "phase": "L1"}, + {"from": "system", "to": "ac_out", "phase": "L1"}, ) # VEBus battery diff --git a/open_ess/victron_modbus/service.py b/open_ess/victron_modbus/service.py index 7ad304a..19c7af4 100644 --- a/open_ess/victron_modbus/service.py +++ b/open_ess/victron_modbus/service.py @@ -2,7 +2,6 @@ import time from typing import TYPE_CHECKING -from open_ess.database import Database from open_ess.service import Service from open_ess.timeseries import TimeseriesBackend @@ -19,13 +18,12 @@ class VictronService(Service): def __init__( self, - db: Database, config: "BatterySystemConfig", timeseries: TimeseriesBackend | None = None, ): super().__init__("VictronService") self._config = config - self._client = VictronClient(db, config, timeseries) + self._client = VictronClient(config, timeseries) @property def client(self) -> VictronClient: From bdb22a23a529d8983f24c2474f61946265e58a74 Mon Sep 17 00:00:00 2001 From: david Date: Mon, 4 May 2026 23:29:59 +0200 Subject: [PATCH 06/18] entsoe client: use timeseries backend --- open_ess/frontend/routes/timeseries.py | 42 +++++--- open_ess/frontend/routes/util.py | 24 ++++- open_ess/pricing/client.py | 19 ++-- open_ess/pricing/service.py | 1 - open_ess/timeseries/__init__.py | 19 +++- open_ess/timeseries/base.py | 60 +++++++++--- open_ess/timeseries/metricsqlite/backend.py | 79 +++++++++------ .../timeseries/victoriametrics/backend.py | 95 ++++++++++++++----- 8 files changed, 241 insertions(+), 98 deletions(-) diff --git a/open_ess/frontend/routes/timeseries.py b/open_ess/frontend/routes/timeseries.py index 4dfa23a..47d98bd 100644 --- a/open_ess/frontend/routes/timeseries.py +++ b/open_ess/frontend/routes/timeseries.py @@ -8,6 +8,8 @@ from fastapi import APIRouter, HTTPException +from open_ess.timeseries import ScalarResult, VectorResult + from ..dependencies import TimeseriesDep router = APIRouter() @@ -38,19 +40,31 @@ async def query( eval_time = _parse_timestamp(time) if time is not None else None result = timeseries.query(query, eval_time) - return { - "status": "success", - "data": { - "resultType": "vector", - "result": [ - { - "metric": series.metric, - "value": [series.values[0][0].timestamp(), series.values[0][1]] if series.values else None, - } - for series in result.series - ], - }, - } + if isinstance(result, ScalarResult): + return { + "status": "success", + "data": { + "resultType": "scalar", + "result": [result.timestamp.timestamp(), str(result.value)], + }, + } + + if isinstance(result, VectorResult): + return { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": series.metric, + "value": [series.timestamp.timestamp(), str(series.value)], + } + for series in result.series + ], + }, + } + + raise HTTPException(500, f"Unexpected result type: {type(result)}") @router.get("/query_range") @@ -77,7 +91,7 @@ async def query_range( "result": [ { "metric": series.metric, - "values": [[v[0].timestamp(), str(v[1])] for v in series.values], + "values": [[ts.timestamp(), str(val)] for ts, val in series.values], } for series in result.series ], diff --git a/open_ess/frontend/routes/util.py b/open_ess/frontend/routes/util.py index 6b242db..3adce21 100644 --- a/open_ess/frontend/routes/util.py +++ b/open_ess/frontend/routes/util.py @@ -5,7 +5,7 @@ from pydantic import BaseModel if TYPE_CHECKING: - from open_ess.timeseries import QueryResult + from open_ess.timeseries import InstantQueryResult, RangeQueryResult class TimeSeries(BaseModel): @@ -13,15 +13,14 @@ class TimeSeries(BaseModel): values: list[float] -def query_result_to_timeseries(result: "QueryResult", rounding: int | None = None) -> TimeSeries: - """Convert a timeseries QueryResult to TimeSeries format. +def range_result_to_timeseries(result: "RangeQueryResult", rounding: int | None = None) -> TimeSeries: + """Convert a RangeQueryResult to TimeSeries format. - If multiple series are returned, they are merged (assumes same timestamps). + Takes the first series if multiple are returned. """ timestamps: list[datetime] = [] values: list[float] = [] - # Take the first series (or merge if needed) if result.series: series = result.series[0] for ts, val in series.values: @@ -34,6 +33,21 @@ def query_result_to_timeseries(result: "QueryResult", rounding: int | None = Non return TimeSeries(timestamps=timestamps, values=values) +def instant_result_to_value(result: "InstantQueryResult") -> float | None: + """Extract the value from an instant query result. + + For ScalarResult, returns the scalar value. + For VectorResult, returns the first series' value. + """ + from open_ess.timeseries import ScalarResult, VectorResult + + if isinstance(result, ScalarResult): + return result.value + if isinstance(result, VectorResult) and result.series: + return result.series[0].value + return None + + def data_to_timeseries(data: Iterable[tuple[datetime, float]], rounding: int | None = None) -> TimeSeries: timestamps = [] values = [] diff --git a/open_ess/pricing/client.py b/open_ess/pricing/client.py index 565de32..5e6e840 100644 --- a/open_ess/pricing/client.py +++ b/open_ess/pricing/client.py @@ -8,7 +8,7 @@ from entsoe.utils import add_timestamps, extract_records from pandas import DataFrame -from open_ess.timeseries import Sample, TimeseriesBackend +from open_ess.timeseries import Sample, TimeseriesBackend, VectorResult from .areas import AREAS from .config import PriceConfig @@ -77,21 +77,25 @@ def fetch_day_ahead_prices( row_start = datetime.fromisoformat(row["timestamp"]) interval_minutes = _parse_resolution(row["time_series.period.resolution"]) row_end = row_start + timedelta(minutes=interval_minutes) - price = float(row["time_series.period.point.price_amount"]) + price = float(row["time_series.period.point.price_amount"]) / 1000 prices.append((row_start, row_end, price)) return prices def fetch_missing_prices(self, area: str) -> None: + if area not in AREAS: + raise ValueError(f"Unknown area code: '{area}'") + now = datetime.now(UTC) end_of_tomorrow = (now + timedelta(days=2)).replace(hour=0, minute=0, second=0, microsecond=0) - # TODO: prevent injection via area variable - result = self._mql_client.query(f'openess_prices{{area="{area}"}}') + result: VectorResult = self._mql_client.query( + f'last_over_time(openess_prices{{area="{area}"}}[8w])', time=end_of_tomorrow + ) if len(result.series) == 0: fetch_start = now.replace(hour=0, minute=0, second=0, microsecond=0) - fetch_start -= timedelta(weeks=6) + fetch_start -= timedelta(weeks=8) else: - latest, _ = result.series[0].values + latest = result.series[0].timestamp if latest >= end_of_tomorrow: return else: @@ -115,7 +119,8 @@ def make_sample(_ts: datetime, _price_type: str, _price: float): ) samples: list[Sample] = [] - for ts, _, price in prices: + for ts_start, ts_end, price in prices: + ts = ts_start + (ts_end - ts_start) / 2 samples.append(make_sample(ts, "market", price)) samples.append(make_sample(ts, "buy", self._config.buy_price(price))) samples.append(make_sample(ts, "sell", self._config.sell_price(price))) diff --git a/open_ess/pricing/service.py b/open_ess/pricing/service.py index 3e5a619..2ec37b5 100644 --- a/open_ess/pricing/service.py +++ b/open_ess/pricing/service.py @@ -19,7 +19,6 @@ def __init__(self, mql_client: TimeseriesBackend, config: PriceConfig): def on_start(self) -> None: self._client = EntsoeClient(self._config, self._mql_client) - self._fetch_prices() def tick(self) -> None: self._fetch_prices() diff --git a/open_ess/timeseries/__init__.py b/open_ess/timeseries/__init__.py index 2764349..7b23084 100644 --- a/open_ess/timeseries/__init__.py +++ b/open_ess/timeseries/__init__.py @@ -1,4 +1,13 @@ -from .base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend +from .base import ( + InstantQueryResult, + InstantSeries, + RangeQueryResult, + RangeSeries, + Sample, + ScalarResult, + TimeseriesBackend, + VectorResult, +) from .config import TimeseriesConfig from .metricsqlite.config import MetricSQLiteConfig from .victoriametrics.config import VictoriaMetricsConfig @@ -28,10 +37,14 @@ def create_backend( __all__ = [ - "QueryResult", - "QueryResultSeries", + "InstantQueryResult", + "InstantSeries", + "RangeQueryResult", + "RangeSeries", "Sample", + "ScalarResult", "TimeseriesBackend", "TimeseriesConfig", + "VectorResult", "create_backend", ] diff --git a/open_ess/timeseries/base.py b/open_ess/timeseries/base.py index 25ed41f..6b73943 100644 --- a/open_ess/timeseries/base.py +++ b/open_ess/timeseries/base.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime +from typing import Literal @dataclass @@ -15,25 +16,54 @@ class Sample: labels: dict[str, str] = field(default_factory=dict) +# --- Instant Query Result Types --- + + +@dataclass +class ScalarResult: + """Result of an instant query returning a scalar value.""" + + result_type: Literal["scalar"] = field(default="scalar", repr=False) + timestamp: datetime = field(default_factory=datetime.now) + value: float = 0.0 + + @dataclass -class QueryResultSeries: - """A single series from a query result.""" +class InstantSeries: + """A series with a single value (instant vector element).""" metric: dict[str, str] - values: list[tuple[datetime, float]] # (timestamp, value) pairs + timestamp: datetime + value: float + + +@dataclass +class VectorResult: + """Result of an instant query returning an instant vector.""" + + result_type: Literal["vector"] = field(default="vector", repr=False) + series: list[InstantSeries] = field(default_factory=list) @dataclass -class QueryResult: - """Result of a query or query_range call.""" +class RangeSeries: + """A series with multiple values over time (range vector/matrix element).""" + + metric: dict[str, str] + values: list[tuple[datetime, float]] # (timestamp, value) pairs + - series: list[QueryResultSeries] +InstantQueryResult = ScalarResult | VectorResult + + +# --- Range Query Result Type --- + + +@dataclass +class RangeQueryResult: + """Result of a range query (always returns a matrix).""" - def scalar(self) -> float | None: - """Get single scalar value if result has exactly one series with one value.""" - if len(self.series) == 1 and len(self.series[0].values) == 1: - return self.series[0].values[0][1] - return None + series: list[RangeSeries] = field(default_factory=list) class TimeseriesBackend(ABC): @@ -53,7 +83,7 @@ def write(self, samples: list[Sample]) -> None: ... @abstractmethod - def query(self, query: str, time: datetime | None = None) -> QueryResult: + def query(self, query: str, time: datetime | None = None) -> InstantQueryResult: """Execute an instant query. Args: @@ -61,7 +91,7 @@ def query(self, query: str, time: datetime | None = None) -> QueryResult: time: Evaluation timestamp. Defaults to now. Returns: - Query result containing matching series. + ScalarResult, VectorResult, or MatrixResult depending on query. """ ... @@ -72,7 +102,7 @@ def query_range( start: datetime, end: datetime, step: str = "1m", - ) -> QueryResult: + ) -> RangeQueryResult: """Execute a range query. Args: @@ -82,7 +112,7 @@ def query_range( step: Query resolution (e.g., "1m", "5m", "1h"). Returns: - Query result containing matching series with values at each step. + RangeQueryResult containing series with values at each step. """ ... diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py index cf1e85b..4d97701 100644 --- a/open_ess/timeseries/metricsqlite/backend.py +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -1,11 +1,23 @@ -from datetime import datetime +import logging +from datetime import UTC, datetime from metricsqlite import MetricsQLiteClient from metricsqlite.engine import InstantVector, MatrixResult, RangeVectorResult, ScalarResult -from ..base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend +from ..base import ( + InstantQueryResult, + InstantSeries, + RangeQueryResult, + RangeSeries, + Sample, + TimeseriesBackend, + VectorResult, +) +from ..base import ScalarResult as BaseScalarResult from .config import MetricSQLiteConfig +logger = logging.getLogger(__name__) + class MetricSQLiteBackend(TimeseriesBackend): def __init__(self, config: MetricSQLiteConfig): @@ -25,14 +37,14 @@ def write(self, samples: list[Sample]) -> None: labels=sample.labels if sample.labels else None, ) - def query(self, query: str, time: datetime | None = None) -> QueryResult: + def query(self, query: str, time: datetime | None = None) -> InstantQueryResult: """Execute an instant query.""" eval_time: float | None = None if time is not None: eval_time = time.timestamp() * 1000 result = self._client.query(query, time=eval_time) - return self._convert_result(result) + return self._convert_instant_result(result) def query_range( self, @@ -40,53 +52,58 @@ def query_range( start: datetime, end: datetime, step: str = "1m", - ) -> QueryResult: + ) -> RangeQueryResult: """Execute a range query.""" start_ms = start.timestamp() * 1000 end_ms = end.timestamp() * 1000 result = self._client.query_range(query, start=start_ms, end=end_ms, step=step) - return self._convert_matrix_result(result) + return self._convert_range_result(result) - def _convert_result(self, result: InstantVector | RangeVectorResult | ScalarResult) -> QueryResult: - """Convert metricsqlite result to QueryResult.""" + def _convert_instant_result(self, result: InstantVector | RangeVectorResult | ScalarResult) -> InstantQueryResult: + """Convert metricsqlite instant query result.""" if isinstance(result, ScalarResult): - # Scalar result - single value - return QueryResult( - series=[ - QueryResultSeries( - metric={}, - values=[(datetime.fromtimestamp(result.timestamp / 1000), result.value)], - ) - ] + return BaseScalarResult( + timestamp=datetime.fromtimestamp(result.timestamp / 1000, tz=UTC), + value=result.value, ) if isinstance(result, RangeVectorResult): # Range vector from instant query (e.g., metric[5m]) - series_list = [] + # Convert to VectorResult by taking the last value + logger.warning("Instant query returned range vector, taking last value") + series = [] for labels, samples in result.series: - values = [(datetime.fromtimestamp(sample.timestamp / 1000), sample.value) for sample in samples] - series_list.append(QueryResultSeries(metric=labels, values=values)) - return QueryResult(series=series_list) + if samples: + last = samples[-1] + series.append( + InstantSeries( + metric=labels, + timestamp=datetime.fromtimestamp(last.timestamp / 1000, tz=UTC), + value=last.value, + ) + ) + return VectorResult(series=series) # InstantVector - series_list = [] + series = [] for labels, sample in result.series: - series_list.append( - QueryResultSeries( + series.append( + InstantSeries( metric=labels, - values=[(datetime.fromtimestamp(sample.timestamp / 1000), sample.value)], + timestamp=datetime.fromtimestamp(sample.timestamp / 1000, tz=UTC), + value=sample.value, ) ) - return QueryResult(series=series_list) + return VectorResult(series=series) - def _convert_matrix_result(self, result: MatrixResult) -> QueryResult: - """Convert metricsqlite MatrixResult to QueryResult.""" - series_list = [] + def _convert_range_result(self, result: MatrixResult) -> RangeQueryResult: + """Convert metricsqlite range query result.""" + series = [] for labels, samples in result.series: - values = [(datetime.fromtimestamp(sample.timestamp / 1000), sample.value) for sample in samples] - series_list.append(QueryResultSeries(metric=labels, values=values)) - return QueryResult(series=series_list) + values = [(datetime.fromtimestamp(sample.timestamp / 1000, tz=UTC), sample.value) for sample in samples] + series.append(RangeSeries(metric=labels, values=values)) + return RangeQueryResult(series=series) def close(self) -> None: """Close the database connection.""" diff --git a/open_ess/timeseries/victoriametrics/backend.py b/open_ess/timeseries/victoriametrics/backend.py index c5003b1..07a4f60 100644 --- a/open_ess/timeseries/victoriametrics/backend.py +++ b/open_ess/timeseries/victoriametrics/backend.py @@ -2,12 +2,21 @@ import json import logging -from datetime import datetime +from datetime import UTC, datetime from urllib3 import HTTPConnectionPool, HTTPSConnectionPool from urllib3.util import parse_url -from ..base import QueryResult, QueryResultSeries, Sample, TimeseriesBackend +from ..base import ( + InstantQueryResult, + InstantSeries, + RangeQueryResult, + RangeSeries, + Sample, + ScalarResult, + TimeseriesBackend, + VectorResult, +) from .client import RemoteWriteClient from .client import Sample as RemoteWriteSample from .config import VictoriaMetricsConfig @@ -76,7 +85,7 @@ def write(self, samples: list[Sample]) -> None: ] self._write_client.write(remote_samples) - def query(self, query: str, time: datetime | None = None) -> QueryResult: + def query(self, query: str, time: datetime | None = None) -> InstantQueryResult: """Execute an instant query.""" params = {"query": query} if time is not None: @@ -92,7 +101,7 @@ def query(self, query: str, time: datetime | None = None) -> QueryResult: raise RuntimeError(f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}") data = json.loads(response.data.decode("utf-8")) - return self._parse_response(data) + return self._parse_instant_response(data) def query_range( self, @@ -100,7 +109,7 @@ def query_range( start: datetime, end: datetime, step: str = "1m", - ) -> QueryResult: + ) -> RangeQueryResult: """Execute a range query.""" params = { "query": query, @@ -119,34 +128,76 @@ def query_range( raise RuntimeError(f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}") data = json.loads(response.data.decode("utf-8")) - return self._parse_response(data) + return self._parse_range_response(data) - def _parse_response(self, data: dict) -> QueryResult: - """Parse VictoriaMetrics/Prometheus API response.""" + def _parse_instant_response(self, data: dict) -> InstantQueryResult: + """Parse VictoriaMetrics/Prometheus instant query response.""" if data.get("status") != "success": error = data.get("error", "Unknown error") raise RuntimeError(f"Query error: {error}") + result_type = data.get("data", {}).get("resultType", "vector") result = data.get("data", {}).get("result", []) - series_list = [] - for item in result: - metric = item.get("metric", {}) + if result_type == "scalar": + # Scalar: [timestamp, value] + ts, val = result + return ScalarResult( + timestamp=datetime.fromtimestamp(float(ts), tz=UTC), + value=float(val), + ) - # Handle both instant query (value) and range query (values) - if "value" in item: - # Instant query: [timestamp, value] + if result_type == "vector": + # Vector: list of {metric, value: [timestamp, value]} + series = [] + for item in result: + metric = item.get("metric", {}) ts, val = item["value"] - values = [(datetime.fromtimestamp(float(ts)), float(val))] - elif "values" in item: - # Range query: [[timestamp, value], ...] - values = [(datetime.fromtimestamp(float(ts)), float(val)) for ts, val in item["values"]] - else: - values = [] + series.append( + InstantSeries( + metric=metric, + timestamp=datetime.fromtimestamp(float(ts), tz=UTC), + value=float(val), + ) + ) + return VectorResult(series=series) + + if result_type == "matrix": + # Matrix from instant query (range selector like [5m]) + # Convert to VectorResult by taking the last value from each series + logger.warning("Instant query returned matrix (range selector?), taking last value") + series = [] + for item in result: + metric = item.get("metric", {}) + values = item.get("values", []) + if values: + ts, val = values[-1] + series.append( + InstantSeries( + metric=metric, + timestamp=datetime.fromtimestamp(float(ts), tz=UTC), + value=float(val), + ) + ) + return VectorResult(series=series) + + raise RuntimeError(f"Unknown result type: {result_type}") + + def _parse_range_response(self, data: dict) -> RangeQueryResult: + """Parse VictoriaMetrics/Prometheus range query response.""" + if data.get("status") != "success": + error = data.get("error", "Unknown error") + raise RuntimeError(f"Query error: {error}") + + result = data.get("data", {}).get("result", []) + series = [] - series_list.append(QueryResultSeries(metric=metric, values=values)) + for item in result: + metric = item.get("metric", {}) + values = [(datetime.fromtimestamp(float(ts), tz=UTC), float(val)) for ts, val in item.get("values", [])] + series.append(RangeSeries(metric=metric, values=values)) - return QueryResult(series=series_list) + return RangeQueryResult(series=series) def close(self) -> None: """Close connections.""" From 354146888003b9cc0a262b7e95c51c75c18ad524 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 5 May 2026 08:55:40 +0200 Subject: [PATCH 07/18] optimizer: use timeseries backend --- open_ess/battery_system/battery_system.py | 6 ++ open_ess/frontend/app.py | 10 ++-- open_ess/frontend/routes/timeseries.py | 4 +- open_ess/main.py | 68 ++++++++++++----------- open_ess/optimizer/optimizer.py | 42 +++++++------- open_ess/optimizer/service.py | 10 ++-- open_ess/pricing/__init__.py | 3 +- open_ess/pricing/util.py | 33 +++++++++++ open_ess/victron_modbus/client.py | 25 +++++++-- open_ess/victron_modbus/service.py | 9 +-- 10 files changed, 131 insertions(+), 79 deletions(-) create mode 100644 open_ess/pricing/util.py diff --git a/open_ess/battery_system/battery_system.py b/open_ess/battery_system/battery_system.py index b545c65..4a9f5df 100644 --- a/open_ess/battery_system/battery_system.py +++ b/open_ess/battery_system/battery_system.py @@ -28,6 +28,9 @@ def id(self) -> str | None: ... @abstractmethod def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: ... + @abstractmethod + def get_soc(self) -> float | None: ... + class VictronBatterySystem(BatterySystem): def __init__(self, config: BatterySystemConfig, control: VictronClient): @@ -44,3 +47,6 @@ def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: until = datetime.now(tz=UTC) + timedelta(hours=1) logger.info(f"{self.name}: Set setpoint to {power} W") self._victron_client.set_ess_setpoint(power, until) + + def get_soc(self) -> float | None: + return self._victron_client.current_soc diff --git a/open_ess/frontend/app.py b/open_ess/frontend/app.py index fb4721a..67b8600 100644 --- a/open_ess/frontend/app.py +++ b/open_ess/frontend/app.py @@ -21,13 +21,13 @@ def create_app( config: "Config", battery_systems: list[BatterySystem], - timeseries: TimeseriesBackend | None = None, + mql_client: TimeseriesBackend | None = None, ) -> FastAPI: @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: _app.state.price_config = config.prices _app.state.battery_systems = battery_systems - _app.state.timeseries = timeseries + _app.state.mql_client = mql_client yield _app.state.database.close() @@ -41,11 +41,11 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: app.include_router(api_router, prefix="/api") # Mount timeseries query endpoints - if timeseries is not None: - if isinstance(timeseries, MetricSQLiteBackend): + if mql_client is not None: + if isinstance(mql_client, MetricSQLiteBackend): from metricsqlite.fastapi import create_router as create_metricsqlite_router - app.include_router(create_metricsqlite_router(timeseries._client), prefix="/api/v1") + app.include_router(create_metricsqlite_router(mql_client._client), prefix="/api/v1") else: app.include_router(timeseries_router, prefix="/api/v1") diff --git a/open_ess/frontend/routes/timeseries.py b/open_ess/frontend/routes/timeseries.py index 47d98bd..80a7fb0 100644 --- a/open_ess/frontend/routes/timeseries.py +++ b/open_ess/frontend/routes/timeseries.py @@ -28,7 +28,7 @@ def _parse_timestamp(value: float | str) -> datetime: @router.get("/query") -async def query( +async def get_query( timeseries: TimeseriesDep, query: str, time: float | str | None = None, @@ -68,7 +68,7 @@ async def query( @router.get("/query_range") -async def query_range( +async def get_query_range( timeseries: TimeseriesDep, query: str, start: float | str, diff --git a/open_ess/main.py b/open_ess/main.py index 98677c3..78a160b 100644 --- a/open_ess/main.py +++ b/open_ess/main.py @@ -1,11 +1,17 @@ import logging import signal +import uvicorn + +from open_ess.battery_system import BatterySystem, VictronBatterySystem from open_ess.config import Config +from open_ess.frontend import create_app +from open_ess.optimizer import OptimizerService from open_ess.pricing import EntsoeService from open_ess.service import ServiceManager from open_ess.timeseries import TimeseriesBackend, create_backend -from open_ess.util import parse_args, setup_logging +from open_ess.util import EndpointFilter, parse_args, setup_logging +from open_ess.victron_modbus import VictronService setup_logging() logger = logging.getLogger(__name__) @@ -15,34 +21,30 @@ def main() -> None: args = parse_args("Open Energy Storage System - optimize charging based on day-ahead prices") config = Config.from_file(args.config) - # Create timeseries backend - timeseries: TimeseriesBackend = create_backend(config.timeseries) - logger.info(f"Using timeseries backend: {config.timeseries.backend}") + # Create MetricsQL client (either MetricSQLite or VictoriaMetrics/Prometheus). + mql_client: TimeseriesBackend = create_backend(config.timeseries) + logger.info(f"Using mql_client backend: {config.timeseries.backend}") # Create services service_manager = ServiceManager() - service_manager.register_service(EntsoeService(timeseries, config.prices)) - # battery_systems: list[BatterySystem] = [] - # for battery_config in config.battery_systems: - # if battery_config.is_victron: - # victron_service = VictronService(timeseries, battery_config, timeseries) - # service_manager.register_service(victron_service) - # battery_system = VictronBatterySystem(battery_config, victron_service.client) - # battery_systems.append(battery_system) - # service_manager.register_service( - # OptimizerService( - # timeseries, - # battery_system=battery_system, - # price_config=config.prices, - # ), - # requires=victron_service, - # ) + service_manager.register_service(EntsoeService(mql_client, config.prices)) + battery_systems: list[BatterySystem] = [] + for battery_config in config.battery_systems: + if battery_config.is_victron: + victron_service = VictronService(battery_config, mql_client) + service_manager.register_service(victron_service) + battery_system = VictronBatterySystem(battery_config, victron_service.client) + battery_systems.append(battery_system) + service_manager.register_service( + OptimizerService(battery_system, config.prices, mql_client), + requires=victron_service, + ) # Shutdown handler def shutdown(signum: int, frame: object) -> None: logger.info("Shutting down...") service_manager.stop() - timeseries.close() + mql_client.close() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) @@ -50,18 +52,18 @@ def shutdown(signum: int, frame: object) -> None: service_manager.start() # Frontend - # if config.frontend.enable: - # logger.info(f"Starting web server on http://{config.frontend.host}:{config.frontend.port}") - # - # logging.getLogger("uvicorn.access").addFilter(EndpointFilter(["/api/power-flow"])) - # - # app = create_app(timeseries, config, battery_systems) - # uvicorn.run( - # app, - # host=config.frontend.host, - # port=config.frontend.port, - # log_level="info", - # ) + if config.frontend.enable: + logger.info(f"Starting web server on http://{config.frontend.host}:{config.frontend.port}") + + logging.getLogger("uvicorn.access").addFilter(EndpointFilter(["/api/power-flow"])) + + app = create_app(config, battery_systems, mql_client) + uvicorn.run( + app, + host=config.frontend.host, + port=config.frontend.port, + log_level="info", + ) service_manager.wait_for_stop() logger.info("Shutdown complete") diff --git a/open_ess/optimizer/optimizer.py b/open_ess/optimizer/optimizer.py index d0b0b3d..fc70270 100644 --- a/open_ess/optimizer/optimizer.py +++ b/open_ess/optimizer/optimizer.py @@ -8,8 +8,8 @@ import pyomo.environ as pyo from pyomo.opt import SolverFactory -from open_ess.battery_system import BatterySystemConfig -from open_ess.pricing import PriceConfig +from open_ess.battery_system import BatterySystem, BatterySystemConfig +from open_ess.pricing import PriceConfig, get_prices_from_mql from open_ess.timeseries import TimeseriesBackend logger = logging.getLogger(__name__) @@ -26,15 +26,15 @@ class Optimizer: the need for a separate binary variable. """ - def __init__(self, mql_client: TimeseriesBackend, price_config: PriceConfig, battery_config: BatterySystemConfig): - self._mql_client = mql_client + def __init__(self, price_config: PriceConfig, battery_system: BatterySystem, mql_client: TimeseriesBackend): self._price_config = price_config - self._battery_config = battery_config + self._battery_system = battery_system + self._mql_client = mql_client # TODO: check for cbc @property def battery_config(self) -> BatterySystemConfig: - return self._battery_config + return self._battery_system.config def optimize(self) -> list[tuple[datetime, datetime, int, float]]: """Generate optimal charge schedule using mixed-integer linear programming. @@ -48,20 +48,23 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: # Get hourly prices for the planning horizon now = datetime.now(UTC) start_hour = now.replace(minute=0, second=0, microsecond=0) - prices = self._database.get_prices( + prices = get_prices_from_mql( + self._mql_client, self._price_config.area, start=start_hour - timedelta(weeks=6), - aggregate_minutes=self._price_config.aggregate_minutes, + end=start_hour + timedelta(days=2), + hourly=self._price_config.hourly_average, ) # Get current SOC - current_soc = self._database.get_current_soc() + current_soc = self._battery_system.get_soc() + logger.info(current_soc) if current_soc is None: logger.error("No SoC data available") return [] - current_soc = min(max(current_soc, self._battery_config.min_soc), self._battery_config.max_soc) + current_soc = min(max(current_soc, self.battery_config.min_soc), self.battery_config.max_soc) # ^ current_soc may be outside the allowed boundaries. Clamp it between the bounds or pyomo will fail. - # TODO: actually fix the issue by allowing soc to be out of bound but don't allow it to go out of bounds. + # TODO: actually fix the issue by allowing soc to BE out of bound but don't allow it to GO out of bounds. if not prices: logger.warning("No price data available") @@ -80,13 +83,13 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: return [] # Build piecewise linear breakpoints for loss functions - assert self._battery_config.max_charge_power_kw is not None - assert self._battery_config.max_invert_power_kw is not None + assert self.battery_config.max_charge_power_kw is not None + assert self.battery_config.max_invert_power_kw is not None charger_bp, charger_loss_vals = build_piecewise_loss_points( - self._battery_config.max_charge_power_kw, charger_loss + self.battery_config.max_charge_power_kw, charger_loss ) inverter_bp, inverter_loss_vals = build_piecewise_loss_points( - self._battery_config.max_invert_power_kw, inverter_loss + self.battery_config.max_invert_power_kw, inverter_loss ) model = pyo.ConcreteModel() @@ -99,9 +102,9 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: model.market_price = pyo.Param(model.T, initialize=price_dict) # Decision variables - model.charge_power = pyo.Var(model.T, bounds=(0, self._battery_config.max_charge_power_kw)) - model.discharge_power = pyo.Var(model.T, bounds=(0, self._battery_config.max_invert_power_kw)) - model.soc = pyo.Var(model.T, bounds=(self._battery_config.min_soc, self._battery_config.max_soc)) + model.charge_power = pyo.Var(model.T, bounds=(0, self.battery_config.max_charge_power_kw)) + model.discharge_power = pyo.Var(model.T, bounds=(0, self.battery_config.max_invert_power_kw)) + model.soc = pyo.Var(model.T, bounds=(self.battery_config.min_soc, self.battery_config.max_soc)) # Auxiliary variables for piecewise linear losses max_charger_loss = charger_loss_vals[-1] @@ -140,12 +143,11 @@ def soc_balance_rule(model: pyomo.core.Model, t: int) -> Any: * self._price_config.aggregate_minutes / 60 ) - soc_change = 100 * net_energy / self._battery_config.capacity_kwh + soc_change = 100 * net_energy / self.battery_config.capacity_kwh return model.soc[t] == prev_soc + soc_change model.soc_balance = pyo.Constraint(model.T, rule=soc_balance_rule) model.final_soc = pyo.Constraint(expr=model.soc[model_length - 1] == current_soc) - # ^ Final SoC should equal starting SOC (energy neutral over horizon) # Objective: minimize cost (buy cost - sell revenue) diff --git a/open_ess/optimizer/service.py b/open_ess/optimizer/service.py index 6c65a95..5b34535 100644 --- a/open_ess/optimizer/service.py +++ b/open_ess/optimizer/service.py @@ -14,21 +14,19 @@ class OptimizerService(Service): def __init__( self, - mql_client: TimeseriesBackend, battery_system: BatterySystem, price_config: PriceConfig, + mql_client: TimeseriesBackend, ): super().__init__("OptimizerService") - self._mql_client = mql_client self._battery_system = battery_system self._price_config = price_config + self._mql_client = mql_client self._optimizer: Optimizer | None = None def on_start(self) -> None: - self._optimizer = Optimizer( - self._mql_client, price_config=self._price_config, battery_config=self._battery_system.config - ) + self._optimizer = Optimizer(self._price_config, self._battery_system, self._mql_client) def tick(self) -> None: if self._optimizer is None: @@ -39,7 +37,7 @@ def tick(self) -> None: if schedule: _, _, power, _ = schedule[0] self._battery_system.set_ess_setpoint(power) - self._db_conn.set_schedule(self._battery_system.id, schedule) # type: ignore[arg-type] + # TODO self._db_conn.set_schedule(self._battery_system.id, schedule) # type: ignore[arg-type] logger.debug(f"Updated schedule with {len(schedule)} entries") else: logger.warning("Optimizer returned empty schedule") diff --git a/open_ess/pricing/__init__.py b/open_ess/pricing/__init__.py index 921e29e..9cffc18 100644 --- a/open_ess/pricing/__init__.py +++ b/open_ess/pricing/__init__.py @@ -1,5 +1,6 @@ from .client import EntsoeClient from .config import PriceConfig from .service import EntsoeService +from .util import get_prices_from_mql -__all__ = ["EntsoeClient", "EntsoeService", "PriceConfig"] +__all__ = ["EntsoeClient", "EntsoeService", "get_prices_from_mql", "PriceConfig"] diff --git a/open_ess/pricing/util.py b/open_ess/pricing/util.py new file mode 100644 index 0000000..7fc2fa7 --- /dev/null +++ b/open_ess/pricing/util.py @@ -0,0 +1,33 @@ +from datetime import datetime +from typing import Literal + +from open_ess.timeseries import TimeseriesBackend + +from .areas import AREAS + +PRICE_TYPES = {"market", "buy", "sell"} +PriceType = Literal["market", "buy", "sell"] + + +def get_prices_from_mql( + mql_client: TimeseriesBackend, area: str, start: datetime, end: datetime, hourly=False, price: PriceType = "market" +) -> list[tuple[datetime, float]]: + """Prices are returned in currency per Kwh (usually €/kWh).""" + + # Validate area and price to prevent MetricsQL injection. + if area not in AREAS: + raise ValueError(f"Unknown area code: '{area}'") + if price not in PRICE_TYPES: + raise ValueError(f"Unknown price type: '{price}'") + + if hourly: + query = f'avg_over_time(openess_prices{{area="{area}", price="{price}"}}[1h])' + step = "1h" + else: + query = f'openess_prices{{area="{area}", price="{price}"}}' + step = "15m" + result = mql_client.query_range(query, start, end, step) + + if not result.series: + return [] + return list(result.series[0].values) diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index 601548e..5eb31b6 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -30,17 +30,18 @@ class VictronClient: def __init__( self, config: "BatterySystemConfig", - timeseries: TimeseriesBackend | None = None, + mql_client: TimeseriesBackend | None = None, ): if not isinstance(config.control, VictronConfig): raise TypeError(f"VictronClient requires VictronConfig, got {type(config.control).__name__}") self._config = config self._control: VictronConfig = config.control self._client = VictronModbusClient(self._control) - self._timeseries = timeseries + self._mql_client = mql_client self._serial: str | None = None + self._current_soc: float | None = None self._setpoint: float = 0.0 # In Watt self._setpoint_expiration: datetime | None = None @@ -58,6 +59,8 @@ def initialize(self) -> bool: if not self._config.monitor_only: self.write(self.system_id, System.ESS_MODE, 3) + self._current_soc = self.read(self.system_id, System.BATTERY_SOC) + return True @property @@ -92,6 +95,10 @@ def pvinverter_id(self) -> int | None: def need_mode_3(self) -> bool: return not self._config.monitor_only + @property + def current_soc(self) -> float | None: + return self._current_soc + def set_ess_setpoint(self, power: float, until: datetime) -> None: with self._lock: self._setpoint = power @@ -115,7 +122,7 @@ def write_setpoints(self) -> None: return idle_threshold = self._config.idle_threshold_w / 1000 - if (self._db_conn.get_current_soc() or 0) >= 99 and self._setpoint >= -idle_threshold: + if (self._current_soc or 0) >= 99 and self._setpoint >= -idle_threshold: # Keep putting power into the battery to allow balancing of the cells by the BMS. # TODO: implement balancing limits? self.write(self.vebus_id, VEBus.ESS_SETPOINT_L1, int((self._config.max_charge_power_kw or 0) * 1000)) @@ -132,8 +139,8 @@ def write_setpoints(self) -> None: if self._control.disable_inverter_when_idle: self.write(self.vebus_id, VEBus.ESS_DISABLE_FEEDBACK, 1) - def collect_and_store_measurements(self) -> None: - if self._timeseries is None: + def scrape_metrics(self) -> None: + if self._mql_client is None: return timestamp = datetime.now(UTC) samples: list[Sample] = [] @@ -227,6 +234,7 @@ def add(metric: str, value: float | None, labels: dict[str, str]) -> None: add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY), {"from": "ac_out", "to": "system"}) # BMS (direct battery measurements) + bms_soc = None if self.battery_id is not None: bms_values = self.read_many( self.battery_id, @@ -244,10 +252,15 @@ def add(metric: str, value: float | None, labels: dict[str, str]) -> None: if samples: try: - self._timeseries.write(samples) + self._mql_client.write(samples) except Exception as e: logger.exception(f"Failed to write samples to timeseries backend: {e}") + if bms_soc is not None: + self._current_soc = bms_soc + elif vebus_soc is not None: + self._current_soc = vebus_soc + # --------------------------------# # VictronModbusClient bindings # # --------------------------------# diff --git a/open_ess/victron_modbus/service.py b/open_ess/victron_modbus/service.py index 19c7af4..f3e78f8 100644 --- a/open_ess/victron_modbus/service.py +++ b/open_ess/victron_modbus/service.py @@ -14,16 +14,13 @@ class VictronService(Service): - """Collects measurements from Victron GX every second.""" - def __init__( self, config: "BatterySystemConfig", - timeseries: TimeseriesBackend | None = None, + mql_client: TimeseriesBackend | None = None, ): super().__init__("VictronService") - self._config = config - self._client = VictronClient(config, timeseries) + self._client = VictronClient(config, mql_client) @property def client(self) -> VictronClient: @@ -36,7 +33,7 @@ def on_start(self) -> None: def tick(self) -> None: self._client.write_setpoints() - self._client.collect_and_store_measurements() + self._client.scrape_metrics() def wait_until_next(self) -> None: # Sleep until the start of the next second From ce645788ede4e7733d69a72c5f213e6cacb150c4 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 5 May 2026 13:20:24 +0200 Subject: [PATCH 08/18] frontend: power graph uses /range_query --- docs/settings.md | 96 ----- open_ess/battery_system/battery_system.py | 10 + open_ess/battery_system/config.py | 2 +- open_ess/frontend/app.py | 6 +- open_ess/frontend/dependencies.py | 6 +- open_ess/frontend/routes/api.py | 418 +++++++++----------- open_ess/frontend/routes/timeseries.py | 6 +- open_ess/frontend/static/metrics.js | 164 ++++++-- open_ess/frontend/templates/metrics.html | 8 +- open_ess/optimizer/optimizer.py | 5 +- open_ess/pricing/__init__.py | 4 +- open_ess/pricing/util.py | 33 -- open_ess/timeseries/base.py | 38 +- open_ess/timeseries/metricsqlite/backend.py | 4 + 14 files changed, 405 insertions(+), 395 deletions(-) delete mode 100644 docs/settings.md delete mode 100644 open_ess/pricing/util.py diff --git a/docs/settings.md b/docs/settings.md deleted file mode 100644 index ece9120..0000000 --- a/docs/settings.md +++ /dev/null @@ -1,96 +0,0 @@ -# Configuration - -OpenESS is configured via a YAML file - -### Example Configuration - -```yaml -database: - path: /var/lib/open-ess/data.db - -prices: - area: NL - entsoe_api_key_file: /var/lib/open-ess/entsoe_api_key - buy_formula: "price" - sell_formula: "price" - -victron_gx: - host: 192.168.1.100 - port: 502 - system_id: 100 - -battery: - control: - type: victron - vebus_id: 228 - capacity_kwh: 10.0 - max_charge_power_kw: 3.0 - max_discharge_power_kw: 3.0 - min_soc: 10 - max_soc: 100 -``` - -### prices - -```yaml -prices: - area: - entsoe_api_key_file: /var/lib/open-ess/entsoe_api_key - buy_formula: "price" - sell_formula: "price" -``` - -The `buy_formula` and `sell_formula` allow you to transform the market price into your actual buy/sell price. Use `price` or `p` as the market price variable (EUR/kWh). - -Allowed operations: `+`, `-`, `*`, `/`, `**`, parentheses - -Examples: -- `"price"` - use market price directly -- `"(price + 0.05) * 1.21"` - add 0.05 EUR/kWh markup and 21% VAT -- `"price * 0.9"` - sell at 90% of market price - -### victron_gx - -Settings for connecting to your Victron GX device via Modbus TCP. - -| Setting | Required | Default | Description | -|---------|----------|---------|-------------| -| `host` | Yes | - | IP address of the GX device | -| `port` | No | `502` | Modbus TCP port | -| `system_id` | Yes | - | Modbus unit ID for system data (usually 100) | -| `grid_id` | No | - | Modbus unit ID for grid meter | -| `pvinverter_id` | No | - | Modbus unit ID for PV inverter | - -To find the Modbus unit IDs, go to your GX device: **Settings > Services > Modbus TCP > Available services** - -### battery - -Configuration for your battery system. Can be a single battery or a list of batteries (multi-battery support is planned). - -| Setting | Required | Default | Description | -|---------|----------|---------|-------------| -| `capacity_kwh` | Yes | - | Total battery capacity in kWh | -| `max_charge_power_kw` | Yes | - | Maximum charge power in kW (AC side) | -| `max_discharge_power_kw` | Yes | - | Maximum discharge power in kW (AC side) | -| `min_soc` | No | `10` | Minimum state of charge (%) | -| `max_soc` | No | `100` | Maximum state of charge (%) | -| `control` | Yes | - | Control configuration (see below) | - -#### battery.control (Victron) - -| Setting | Required | Default | Description | -|---------|----------|---------|-------------| -| `type` | Yes | - | Must be `victron` | -| `vebus_id` | Yes | - | Modbus unit ID of the MultiPlus/Quattro | -| `battery_id` | No | - | Modbus unit ID of the BMS (if available) | -| `monitor_only` | No | `false` | Only collect metrics, don't control the battery | -| `disable_charger_when_idle` | No | `false` | Disable charger when not charging (saves power) | -| `disable_inverter_when_idle` | No | `false` | Disable inverter when not discharging (saves power) | - -#### battery.control (MQTT) - Planned - -| Setting | Required | Default | Description | -|---------|----------|---------|-------------| -| `type` | Yes | - | Must be `mqtt` | -| `topic` | Yes | - | MQTT topic prefix for this battery | -| `monitor_only` | No | `false` | Only collect metrics, don't control the battery | diff --git a/open_ess/battery_system/battery_system.py b/open_ess/battery_system/battery_system.py index 4a9f5df..3c9c247 100644 --- a/open_ess/battery_system/battery_system.py +++ b/open_ess/battery_system/battery_system.py @@ -25,6 +25,12 @@ def name(self) -> str | None: @abstractmethod def id(self) -> str | None: ... + @property + @abstractmethod + def device_serial(self) -> str | None: + """Device serial number used for metrics labeling.""" + ... + @abstractmethod def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: ... @@ -42,6 +48,10 @@ def __init__(self, config: BatterySystemConfig, control: VictronClient): def id(self) -> str | None: return self._victron_client.serial + @property + def device_serial(self) -> str | None: + return self._victron_client.serial + def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: if until is None: until = datetime.now(tz=UTC) + timedelta(hours=1) diff --git a/open_ess/battery_system/config.py b/open_ess/battery_system/config.py index 6b508d1..5fbc0f7 100644 --- a/open_ess/battery_system/config.py +++ b/open_ess/battery_system/config.py @@ -19,7 +19,7 @@ class QueriesConfig(BaseModel): or openess_soc_ratio{device=~"$device", node="battery", unit="vebus"} ) * 100 - """ # + """ voltage: str = """ ( openess_voltage_volts{device=~"$device", node="battery", unit="battery"} diff --git a/open_ess/frontend/app.py b/open_ess/frontend/app.py index 67b8600..07ba8ff 100644 --- a/open_ess/frontend/app.py +++ b/open_ess/frontend/app.py @@ -29,7 +29,7 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: _app.state.battery_systems = battery_systems _app.state.mql_client = mql_client yield - _app.state.database.close() + # _app.state.mql_client.close() app = FastAPI( title="OpenESS", @@ -43,9 +43,7 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: # Mount timeseries query endpoints if mql_client is not None: if isinstance(mql_client, MetricSQLiteBackend): - from metricsqlite.fastapi import create_router as create_metricsqlite_router - - app.include_router(create_metricsqlite_router(mql_client._client), prefix="/api/v1") + app.include_router(mql_client.create_fastapi_router(), prefix="/api/v1") else: app.include_router(timeseries_router, prefix="/api/v1") diff --git a/open_ess/frontend/dependencies.py b/open_ess/frontend/dependencies.py index 6a7084d..3a66eae 100644 --- a/open_ess/frontend/dependencies.py +++ b/open_ess/frontend/dependencies.py @@ -15,11 +15,11 @@ def get_battery_systems(request: Request) -> list[BatterySystem]: return request.app.state.battery_systems # type: ignore[no-any-return] -def get_timeseries(request: Request) -> TimeseriesBackend | None: - return request.app.state.timeseries # type: ignore[no-any-return] +def get_mql_client(request: Request) -> TimeseriesBackend: + return request.app.state.mql_client # type: ignore[no-any-return] # Type aliases for cleaner route signatures PriceConfigDep = Annotated[PriceConfig, Depends(get_price_config)] BatterySystemsDep = Annotated[list[BatterySystem], Depends(get_battery_systems)] -TimeseriesDep = Annotated[TimeseriesBackend | None, Depends(get_timeseries)] +MqlClientDep = Annotated[TimeseriesBackend, Depends(get_mql_client)] diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index ce1c5f5..7a197ea 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,9 +1,12 @@ import logging +from datetime import datetime from typing import TYPE_CHECKING from fastapi import APIRouter, HTTPException from pydantic import BaseModel +from open_ess.frontend.dependencies import BatterySystemsDep, MqlClientDep + from .util import TimeSeries if TYPE_CHECKING: @@ -250,38 +253,37 @@ async def health_check() -> HealthResponse: # except Exception as e: # logger.exception("Failed to get battery ids") # raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# # ------------------------ # -# # Metrics page endpoints # -# # ------------------------ # -# -# -# class BatteryEnergySeries(BaseModel): -# energy_to_charger: list[float | None] = [] -# energy_from_inverter: list[float | None] = [] -# energy_to_battery: list[float | None] = [] -# energy_from_battery: list[float | None] = [] -# energy_loss_to_battery: list[float | None] = [] -# energy_loss_from_battery: list[float | None] = [] -# -# -# class EnergyGraphResponse(BaseModel): -# timestamps: list[datetime] -# -# grid_import: dict[str, list[float | None]] -# grid_export: dict[str, list[float | None]] -# -# battery_systems: dict[str, BatteryEnergySeries] -# -# solar: list[float | None] = [] -# to_consumption: list[float | None] = [] -# from_consumption: list[float | None] = [] -# -# + + +# ------------------------ # +# Metrics page endpoints # +# ------------------------ # + + +class BatteryEnergySeries(BaseModel): + energy_to_charger: list[float | None] = [] + energy_from_inverter: list[float | None] = [] + energy_to_battery: list[float | None] = [] + energy_from_battery: list[float | None] = [] + energy_loss_to_battery: list[float | None] = [] + energy_loss_from_battery: list[float | None] = [] + + +class EnergyGraphResponse(BaseModel): + timestamps: list[datetime] + + grid_import: dict[str, list[float | None]] + grid_export: dict[str, list[float | None]] + + battery_systems: dict[str, BatteryEnergySeries] + + solar: list[float | None] = [] + to_consumption: list[float | None] = [] + from_consumption: list[float | None] = [] + + # @router.get("/energy-graph", response_model=EnergyGraphResponse) # async def get_energy_flow_endpoint( -# db: Database, # timeseries: TimeseriesDep, # battery_systems: BatterySystemsDep, # battery_id: str | None = Query(default=None), @@ -311,17 +313,14 @@ async def health_check() -> HealthResponse: # if end is None: # end = now # -# if timeseries is not None: -# return await _get_energy_graph_timeseries(timeseries, battery_system, start, end, bucket_minutes) -# -# return await _get_energy_graph_legacy(db, battery_system, start, end, bucket_minutes) +# return await _get_energy_graph_timeseries(timeseries, battery_system, start, end, bucket_minutes) # except Exception as e: # logger.exception("Failed to get energy flow") # raise HTTPException(status_code=500, detail=str(e)) from e # # # async def _get_energy_graph_timeseries( -# timeseries: "TimeseriesBackend", +# mql_client: "TimeseriesBackend", # battery_system, # start: datetime, # end: datetime, @@ -339,10 +338,10 @@ async def health_check() -> HealthResponse: # from_mp_query = queries.energy_from_battery.replace("$device", device) # # # Use increase() to get energy delta per bucket -# grid_import_result = timeseries.query_range(f"increase({grid_import_query}[{step}])", start, end, step) -# grid_export_result = timeseries.query_range(f"increase({grid_export_query}[{step}])", start, end, step) -# to_mp_result = timeseries.query_range(f"increase({to_mp_query}[{step}])", start, end, step) -# from_mp_result = timeseries.query_range(f"increase({from_mp_query}[{step}])", start, end, step) +# grid_import_result = mql_client.query_range(f"increase({grid_import_query}[{step}])", start, end, step) +# grid_export_result = mql_client.query_range(f"increase({grid_export_query}[{step}])", start, end, step) +# to_mp_result = mql_client.query_range(f"increase({to_mp_query}[{step}])", start, end, step) +# from_mp_result = mql_client.query_range(f"increase({from_mp_query}[{step}])", start, end, step) # # # Convert to dict for easier lookup # def result_to_dict(result: "QueryResult") -> dict[datetime, float]: @@ -389,189 +388,163 @@ async def health_check() -> HealthResponse: # grid_import=grid_imports, # battery_systems={battery_system.config.name: battery_stats}, # ) -# -# -# async def _get_energy_graph_legacy( -# db: "DatabaseConnection", -# battery_system, -# start: datetime, -# end: datetime, -# bucket_minutes: int, -# ) -> EnergyGraphResponse: -# """Get energy graph data from legacy database.""" -# series = { -# "grid_import": db.get_energy_aggregated( -# "grid/energy/import/total", bucket_minutes * 60, start, end, center_buckets=True -# ), -# "grid_export": db.get_energy_aggregated( -# "grid/energy/export/total", bucket_minutes * 60, start, end, center_buckets=True -# ), -# "vebus_228_import": db.get_energy_aggregated( -# battery_system.config.metrics.energy_to_system, bucket_minutes * 60, start, end, center_buckets=True -# ), -# "vebus_228_export": db.get_energy_aggregated( -# battery_system.config.metrics.energy_from_system, bucket_minutes * 60, start, end, center_buckets=True -# ), -# } -# -# timestamps: set[datetime] = set() -# series_as_dict: dict[str, dict[datetime, float]] = {name: {} for name in series} -# for name, s in series.items(): -# for ts, v in s: -# timestamps.add(ts) -# series_as_dict[name][ts] = v -# sorted_timestamps = sorted(timestamps) -# -# grid_exports: dict[str, list[float | None]] = {"From MP": []} -# grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} -# battery_stats = BatteryEnergySeries() -# -# for ts in sorted_timestamps: -# from_mp = series_as_dict["vebus_228_export"].get(ts) -# grid_exports["From MP"].append(from_mp) -# unaccounted_export = series_as_dict["grid_export"].get(ts, 0) - (from_mp or 0) -# -# to_mp = series_as_dict["vebus_228_import"].get(ts) -# grid_imports["To MP"].append(to_mp) -# grid_import = series_as_dict["grid_import"].get(ts) -# if grid_import is not None: -# grid_import -= (to_mp or 0) - unaccounted_export -# grid_imports["Consumption"].append(grid_import) -# -# battery_stats.energy_to_charger.append(to_mp) -# battery_stats.energy_from_inverter.append(from_mp) -# -# return EnergyGraphResponse( -# timestamps=sorted_timestamps, -# grid_export=grid_exports, -# grid_import=grid_imports, -# battery_systems={"MultiPlus": battery_stats}, -# ) -# -# -# def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: -# """Calculate query step from aggregate_minutes or time range.""" -# if aggregate_minutes > 1: -# return f"{aggregate_minutes}m" -# # Auto-calculate based on range -# duration = (end - start).total_seconds() -# if duration <= 3600: # 1 hour -# return "1m" -# if duration <= 6 * 3600: # 6 hours -# return "5m" -# if duration <= 24 * 3600: # 24 hours -# return "15m" -# return "1h" -# -# -# @router.get("/power-graph", response_model=PowerResponse) -# async def get_power_graph( -# db: Database, -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# battery_id: str | None = Query(default=None), -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# aggregate_minutes: int = Query(default=1), -# ) -> PowerResponse: -# try: -# battery_system = None -# if battery_id: -# for bs in battery_systems: -# if bs.id == battery_id: -# battery_system = bs -# break -# elif len(battery_systems) == 1: -# battery_system = battery_systems[0] -# -# if battery_system is None: -# if battery_id: -# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") -# else: -# raise HTTPException(status_code=400, detail="Please provide a battery_id") -# -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(hours=24) -# if end is None: -# end = now -# -# # Schedule always comes from database -# schedule_series: list[tuple[datetime, float]] = [] -# for ts_start, ts_end, v, _ in db.get_schedule(battery_system.config.id, start): -# schedule_series.extend([(ts_start, v), (ts_end, v)]) -# -# if timeseries is not None: -# device = battery_system.id -# step = _calculate_step(start, end, aggregate_minutes) -# -# series: dict[str, TimeSeries] = {} -# -# # Grid power per phase -# for phase in ("L1", "L2", "L3"): -# query = f'openess_power_watts{{from="grid", phase="{phase}", device="{device}"}}' -# result = timeseries.query_range(query, start, end, step) -# series[f"Grid {phase}"] = query_result_to_timeseries(result) -# -# # AC power (to/from MultiPlus) -# ac_query = battery_system.config.queries.power_ac_in.replace("$device", device) -# ac_result = timeseries.query_range(ac_query, start, end, step) -# series["To MP"] = query_result_to_timeseries(ac_result) -# -# # Battery DC power -# battery_query = battery_system.config.queries.power_battery.replace("$device", device) -# battery_result = timeseries.query_range(battery_query, start, end, step) -# series["Battery"] = query_result_to_timeseries(battery_result) -# -# # Solar power (negated for display) -# solar_query = battery_system.config.queries.power_pv.replace("$device", device) -# solar_result = timeseries.query_range(solar_query, start, end, step) -# solar_ts = query_result_to_timeseries(solar_result) -# series["Solar"] = TimeSeries( -# timestamps=solar_ts.timestamps, -# values=[-v for v in solar_ts.values], -# ) -# -# series["Schedule"] = data_to_timeseries(schedule_series) -# return PowerResponse(series=series) -# -# # Legacy database queries -# bucket_seconds = aggregate_minutes * 60 -# legacy_series: dict[str, list[tuple[datetime, float]]] = { -# f"Grid L{i}": db.get_power(f"grid/power/l{i}", start, end, bucket_seconds) for i in (1, 2, 3) -# } -# legacy_series["To MP"] = db.get_power(battery_system.config.metrics.power_to_system, start, end, bucket_seconds) -# legacy_series["Battery"] = db.get_power( -# battery_system.config.metrics.power_to_battery, start, end, bucket_seconds -# ) -# legacy_series["Solar"] = [ -# (t, -p) for t, p in db.get_power("victron/pvinverter/31/power/l1", start, end, bucket_seconds) -# ] -# legacy_series["Schedule"] = schedule_series -# -# return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) -# except Exception as e: -# logger.exception("Failed to get power data") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# class PricePoint(BaseModel): -# time: datetime -# market: float | None -# buy: float | None -# sell: float | None -# -# -# class PricesResponse(BaseModel): -# area: str -# aggregate_minutes: int -# unit: str = "€/kWh" # TODO: based on area -# timeseries: list[PricePoint] -# -# + + +def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: + """Calculate query step from aggregate_minutes or time range.""" + if aggregate_minutes > 1: + return f"{aggregate_minutes}m" + # Auto-calculate based on range + duration = (end - start).total_seconds() + if duration <= 3600: # 1 hour + return "1m" + if duration <= 6 * 3600: # 6 hours + return "5m" + if duration <= 24 * 3600: # 24 hours + return "15m" + return "1h" + + +class PowerQueryDef(BaseModel): + label: str + query: str + is_total: bool | None = None + + +class ChartsPowerResponse(BaseModel): + queries: list[PowerQueryDef] + phases: list[str] + + +@router.get("/charts/power-queries", response_model=ChartsPowerResponse) +async def get_power_queries( + timeseries: MqlClientDep, + battery_systems: BatterySystemsDep, +) -> ChartsPowerResponse: + queries: list[PowerQueryDef] = [] + + phases: list[str] = [] + if timeseries is not None: + result = timeseries.query('openess_power_watts{from="grid"}') + phase_set: set[str] = set() + if hasattr(result, "series"): + for series in result.series: + phase_label = series.metric.get("phase") + if phase_label: + phase_set.add(phase_label) + phases = sorted(phase_set) + + # Grid power queries + if len(phases) > 1: + queries.append( + PowerQueryDef( + query='sum(avg_over_time(openess_power_watts{from="grid"}[$step]))', + label="Grid", + is_total=True, + ) + ) + for phase in phases: + queries.append( + PowerQueryDef( + query=f'avg_over_time(openess_power_watts{{from="grid", phase="{phase}"}}[$step])', + label=f"Grid {phase}", + is_total=False, + ) + ) + else: + queries.append( + PowerQueryDef( + query='avg_over_time(openess_power_watts{from="grid"}[$step])', + label="Grid", + ) + ) + + # Battery system queries + for bs in battery_systems: + device = bs.device_serial or "unknown" + bs_name = bs.config.name or bs.id + + # Discover phases for this battery system + bs_phases: list[str] = [] + if timeseries is not None: + result = timeseries.query(f'openess_power_watts{{to="system", device="{device}"}}') + phase_set: set[str] = set() + if hasattr(result, "series"): + for series in result.series: + phase_label = series.metric.get("phase") + if phase_label: + phase_set.add(phase_label) + bs_phases = sorted(phase_set) + + if len(bs_phases) > 1: + queries.append( + PowerQueryDef( + query=f""" + sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) + - on(device) + sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) + """, + label=f"{bs_name} AC", + is_total=True, + ) + ) + for phase in bs_phases: + queries.append( + PowerQueryDef( + query=f""" + sum by (device, phase) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}", phase="{phase}"}}[$step])) + - on(device, phase) + sum by (device, phase) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}", phase="{phase}"}}[$step])) + """, + label=f"{bs_name} AC {phase}", + is_total=False, + ) + ) + else: + queries.append( + PowerQueryDef( + query=f""" + sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) + - on(device) + sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) + """, + label=f"{bs_name} AC", + ) + ) + + queries.append( + PowerQueryDef( + query=f""" + avg_over_time(openess_power_watts{{from="system", to="battery", unit="battery", device="{device}"}}[$step]) + or + avg_over_time(openess_power_watts{{from="system", to="battery", unit="vebus", device="{device}"}}[$step]) + """, + label=f"{bs_name} Battery", + ) + ) + + return ChartsPowerResponse( + queries=queries, + phases=phases, + ) + + +class PricePoint(BaseModel): + time: datetime + market: float | None + buy: float | None + sell: float | None + + +class PricesResponse(BaseModel): + area: str + aggregate_minutes: int + unit: str = "€/kWh" # TODO: based on area + timeseries: list[PricePoint] + + # @router.get("/prices", response_model=PricesResponse) # async def get_price_data( -# db: Database, # price_config: PriceConfigDep, # area: str | None = Query(default=None), # start: datetime | None = Query(default=None), @@ -618,7 +591,6 @@ async def health_check() -> HealthResponse: # # @router.get("/battery-graph", response_model=dict[str, BatteryGraphResponse]) # async def get_battery_graph( -# db: Database, # timeseries: TimeseriesDep, # battery_systems: BatterySystemsDep, # battery_id: str | None = Query(default=None), @@ -668,8 +640,8 @@ async def health_check() -> HealthResponse: # except Exception as e: # logger.exception("Failed to get battery SOC") # raise HTTPException(status_code=500, detail=str(e)) from e -# -# + + # # ---------------# # # Cycles page # # # ---------------# diff --git a/open_ess/frontend/routes/timeseries.py b/open_ess/frontend/routes/timeseries.py index 80a7fb0..b04f8fa 100644 --- a/open_ess/frontend/routes/timeseries.py +++ b/open_ess/frontend/routes/timeseries.py @@ -10,7 +10,7 @@ from open_ess.timeseries import ScalarResult, VectorResult -from ..dependencies import TimeseriesDep +from ..dependencies import MqlClientDep router = APIRouter() @@ -29,7 +29,7 @@ def _parse_timestamp(value: float | str) -> datetime: @router.get("/query") async def get_query( - timeseries: TimeseriesDep, + timeseries: MqlClientDep, query: str, time: float | str | None = None, ) -> dict: @@ -69,7 +69,7 @@ async def get_query( @router.get("/query_range") async def get_query_range( - timeseries: TimeseriesDep, + timeseries: MqlClientDep, query: str, start: float | str, end: float | str, diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 4b18def..5a51e57 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -5,7 +5,9 @@ var dashboardStart = null; var dashboardEnd = null; var currentFoR = 'multiplus'; + var currentPowerMode = 'total'; // 'total' or 'phases' var cachedEnergyData = null; + var cachedPowerConfig = null; // Cached power chart config var rangeOffset = 0; var isRelayoutInProgress = false; @@ -173,52 +175,144 @@ } } + /** + * Fetch power chart configuration from backend. + * @returns {Promise} Chart config with queries, phases, has_phase_toggle + */ + async function fetchPowerChartConfig() { + if (cachedPowerConfig) { + return cachedPowerConfig; + } + var response = await fetch('/api/charts/power-queries'); + if (!response.ok) { + throw new Error('Failed to fetch power chart config: HTTP ' + response.status); + } + cachedPowerConfig = await response.json(); + + // Show/hide phase toggle button based on phases + var toggleContainer = document.getElementById('power-phase-buttons'); + if (toggleContainer) { + toggleContainer.style.display = cachedPowerConfig.phases.length > 1 ? '' : 'none'; + } + + return cachedPowerConfig; + } + + /** + * Filter queries based on current phase mode. + * @param {Array} queries - All query definitions + * @param {string} mode - 'total' or 'phases' + * @returns {Array} Filtered queries + */ + function filterQueriesByMode(queries, mode) { + return queries.filter(function(q) { + // null means show in both modes + if (q.is_total === null) { + return true; + } + if (mode === 'total') { + return q.is_total === true; + } else { + return q.is_total === false; + } + }); + } + + /** + * Execute a single MetricsQL query and return Plotly trace data. + * @param {Object} queryDef - Query definition {query, label, group, variant} + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} step - Step value like '5m' + * @returns {Promise} Plotly trace or null if no data + */ + async function executeQuery(queryDef, start, end, step) { + // Replace $step placeholder + var query = queryDef.query.replace(/\$step/g, step); + + var params = new URLSearchParams({ + query: query, + start: (start.getTime() / 1000).toString(), + end: (end.getTime() / 1000).toString(), + step: step, + }); + + var response = await fetch('/api/v1/query_range?' + params); + if (!response.ok) { + console.error('Query failed for', queryDef.label, ':', response.status); + return null; + } + + var result = await response.json(); + if (!result.data || !result.data.result || result.data.result.length === 0) { + return null; + } + + // Take the first series (most queries return single series) + var series = result.data.result[0]; + return { + x: series.values.map(function(v) { return new Date(v[0] * 1000); }), + y: series.values.map(function(v) { return parseFloat(v[1]); }), + type: 'scatter', + mode: 'lines', + name: queryDef.label, + line: { width: 1.5 }, + connectgaps: false, + }; + } + async function loadPowerChart(elementId, start, end, aggregateMinutes) { aggregateMinutes = aggregateMinutes || 5; Utils.showLoading(elementId); try { - var data = await Api.powerGraph({ - start: Utils.formatDate(start), - end: Utils.formatDate(end), - aggregate_minutes: aggregateMinutes, + var config = await fetchPowerChartConfig(); + var step = aggregateMinutes + 'm'; + + // Filter queries based on current mode + var activeQueries = filterQueriesByMode(config.queries, currentPowerMode); + + // Execute all queries in parallel + var tracePromises = activeQueries.map(function(q) { + return executeQuery(q, start, end, step); }); + var traces = await Promise.all(tracePromises); + // Filter out null results and apply settings var settings = Settings.load(); var useKw = settings.powerUnit === 'kw'; var unit = useKw ? 'kW' : 'W'; + var divisor = useKw ? 1000 : 1; - var traces = []; - var series = data.series || {}; - var sortedKeys = Object.keys(series).sort(); - - for (var i = 0; i < sortedKeys.length; i++) { - var key = sortedKeys[i]; - var s = series[key]; - if (!s.timestamps || !s.values) continue; - - traces.push({ - x: s.timestamps.map(function(t) { return new Date(t); }), - y: s.values, - type: 'scatter', - mode: 'lines', - name: key, - line: { width: 1.5 }, - connectgaps: false, - hovertemplate: '%{y:.1f} ' + unit + '' + key + '', - }); - } + var validTraces = traces.filter(function(t) { return t !== null; }); + validTraces.forEach(function(trace) { + if (useKw) { + trace.y = trace.y.map(function(v) { return v / divisor; }); + } + trace.hovertemplate = '%{y:.1f} ' + unit + '' + trace.name + ''; + }); var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, end); Utils.layoutAddNowLine(layout, start, end); - Utils.makePlot(elementId, traces, layout); + Utils.makePlot(elementId, validTraces, layout); } catch (error) { console.error('Error loading power data:', error); Utils.showError(elementId, 'Failed to load power data'); } } + /** + * Re-render power chart with current settings (called when toggle changes). + */ + function reloadPowerChart() { + if (dashboardStart && dashboardEnd) { + var hours = parseInt(document.getElementById('range-select').value); + var aggregateMinutes = getAggregateMinutes(hours); + loadPowerChart('power-chart', dashboardStart, dashboardEnd, aggregateMinutes); + } + } + async function loadPriceChart(elementId, start, end) { Utils.showLoading(elementId); @@ -486,15 +580,24 @@ document.addEventListener('DOMContentLoaded', function() { var savedRange = Settings.loadPagePref('dashboard', 'range', '24'); var savedFoR = Settings.loadPagePref('dashboard', 'for', 'multiplus'); + var savedPowerMode = Settings.loadPagePref('dashboard', 'powerMode', 'total'); document.getElementById('range-select').value = savedRange; currentFoR = savedFoR; + currentPowerMode = savedPowerMode; + // Energy frame-of-reference buttons var forButtons = document.querySelectorAll('#for-buttons .btn-toggle'); forButtons.forEach(function(btn) { btn.classList.toggle('active', btn.dataset.value === savedFoR); }); + // Power phase toggle buttons + var powerPhaseButtons = document.querySelectorAll('#power-phase-buttons .btn-toggle'); + powerPhaseButtons.forEach(function(btn) { + btn.classList.toggle('active', btn.dataset.value === savedPowerMode); + }); + document.getElementById('range-select').addEventListener('change', function(e) { Settings.savePagePref('dashboard', 'range', e.target.value); rangeOffset = 0; @@ -523,6 +626,17 @@ }); }); + // Power phase toggle handlers + powerPhaseButtons.forEach(function(btn) { + btn.addEventListener('click', function() { + document.querySelectorAll('#power-phase-buttons .btn-toggle').forEach(function(b) { b.classList.remove('active'); }); + btn.classList.add('active'); + currentPowerMode = btn.dataset.value || 'total'; + Settings.savePagePref('dashboard', 'powerMode', currentPowerMode); + reloadPowerChart(); + }); + }); + loadDashboard(); }); })(); diff --git a/open_ess/frontend/templates/metrics.html b/open_ess/frontend/templates/metrics.html index 0595b68..7e5490c 100644 --- a/open_ess/frontend/templates/metrics.html +++ b/open_ess/frontend/templates/metrics.html @@ -34,7 +34,13 @@

Energy

-

Power

+
+

Power

+ +
diff --git a/open_ess/optimizer/optimizer.py b/open_ess/optimizer/optimizer.py index fc70270..a167668 100644 --- a/open_ess/optimizer/optimizer.py +++ b/open_ess/optimizer/optimizer.py @@ -9,7 +9,7 @@ from pyomo.opt import SolverFactory from open_ess.battery_system import BatterySystem, BatterySystemConfig -from open_ess.pricing import PriceConfig, get_prices_from_mql +from open_ess.pricing import PriceConfig from open_ess.timeseries import TimeseriesBackend logger = logging.getLogger(__name__) @@ -48,8 +48,7 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: # Get hourly prices for the planning horizon now = datetime.now(UTC) start_hour = now.replace(minute=0, second=0, microsecond=0) - prices = get_prices_from_mql( - self._mql_client, + prices = self._mql_client.get_prices( self._price_config.area, start=start_hour - timedelta(weeks=6), end=start_hour + timedelta(days=2), diff --git a/open_ess/pricing/__init__.py b/open_ess/pricing/__init__.py index 9cffc18..d29b1aa 100644 --- a/open_ess/pricing/__init__.py +++ b/open_ess/pricing/__init__.py @@ -1,6 +1,6 @@ +from .areas import AREAS from .client import EntsoeClient from .config import PriceConfig from .service import EntsoeService -from .util import get_prices_from_mql -__all__ = ["EntsoeClient", "EntsoeService", "get_prices_from_mql", "PriceConfig"] +__all__ = ["AREAS", "EntsoeClient", "EntsoeService", "PriceConfig"] diff --git a/open_ess/pricing/util.py b/open_ess/pricing/util.py deleted file mode 100644 index 7fc2fa7..0000000 --- a/open_ess/pricing/util.py +++ /dev/null @@ -1,33 +0,0 @@ -from datetime import datetime -from typing import Literal - -from open_ess.timeseries import TimeseriesBackend - -from .areas import AREAS - -PRICE_TYPES = {"market", "buy", "sell"} -PriceType = Literal["market", "buy", "sell"] - - -def get_prices_from_mql( - mql_client: TimeseriesBackend, area: str, start: datetime, end: datetime, hourly=False, price: PriceType = "market" -) -> list[tuple[datetime, float]]: - """Prices are returned in currency per Kwh (usually €/kWh).""" - - # Validate area and price to prevent MetricsQL injection. - if area not in AREAS: - raise ValueError(f"Unknown area code: '{area}'") - if price not in PRICE_TYPES: - raise ValueError(f"Unknown price type: '{price}'") - - if hourly: - query = f'avg_over_time(openess_prices{{area="{area}", price="{price}"}}[1h])' - step = "1h" - else: - query = f'openess_prices{{area="{area}", price="{price}"}}' - step = "15m" - result = mql_client.query_range(query, start, end, step) - - if not result.series: - return [] - return list(result.series[0].values) diff --git a/open_ess/timeseries/base.py b/open_ess/timeseries/base.py index 6b73943..61aa636 100644 --- a/open_ess/timeseries/base.py +++ b/open_ess/timeseries/base.py @@ -3,7 +3,10 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime -from typing import Literal +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + from open_ess.battery_system import BatterySystem @dataclass @@ -120,3 +123,36 @@ def query_range( def close(self) -> None: """Close any connections.""" ... + + def get_prices( + self, + area: str, + start: datetime, + end: datetime, + hourly=False, + price: Literal["market", "buy", "sell"] = "market", + ) -> list[tuple[datetime, float]]: + """Prices are returned in currency per Kwh (usually €/kWh).""" + # Lazy import to avoid circular dependency + from open_ess.pricing import AREAS + + # Validate area and price to prevent MetricsQL injection. + if area not in AREAS: + raise ValueError(f"Unknown area code: '{area}'") + if price not in ("market", "buy", "sell"): + raise ValueError(f"Unknown price type: '{price}'") + + if hourly: + query = f'avg_over_time(openess_prices{{area="{area}", price="{price}"}}[1h])' + step = "1h" + else: + query = f'openess_prices{{area="{area}", price="{price}"}}' + step = "15m" + result = self.query_range(query, start, end, step) + + if not result.series: + return [] + return list(result.series[0].values) + + def get_schedule(self, battery_system: "BatterySystem"): + return [] diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py index 4d97701..07cae70 100644 --- a/open_ess/timeseries/metricsqlite/backend.py +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -3,6 +3,7 @@ from metricsqlite import MetricsQLiteClient from metricsqlite.engine import InstantVector, MatrixResult, RangeVectorResult, ScalarResult +from metricsqlite.fastapi import create_router from ..base import ( InstantQueryResult, @@ -105,6 +106,9 @@ def _convert_range_result(self, result: MatrixResult) -> RangeQueryResult: series.append(RangeSeries(metric=labels, values=values)) return RangeQueryResult(series=series) + def create_fastapi_router(self): + return create_router(self._client) + def close(self) -> None: """Close the database connection.""" self._client.close() From 787ac24b4431807c7fde1786db4ed58b4bf63cd9 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 5 May 2026 13:57:12 +0200 Subject: [PATCH 09/18] frontend: price graph uses /range_query --- open_ess/battery_system/config.py | 1 - open_ess/frontend/routes/api.py | 84 +++----- open_ess/frontend/static/api.js | 296 ++-------------------------- open_ess/frontend/static/metrics.js | 170 +++++++--------- 4 files changed, 122 insertions(+), 429 deletions(-) diff --git a/open_ess/battery_system/config.py b/open_ess/battery_system/config.py index 5fbc0f7..a6c46d4 100644 --- a/open_ess/battery_system/config.py +++ b/open_ess/battery_system/config.py @@ -58,7 +58,6 @@ class BatterySystemConfig(BaseModel): max_soc: int = 100 control: Annotated[VictronConfig | MqttControl, Field(discriminator="type")] - queries: QueriesConfig = QueriesConfig() @property def is_victron(self) -> bool: diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index 7a197ea..2243f61 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,17 +1,16 @@ import logging from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Query from pydantic import BaseModel -from open_ess.frontend.dependencies import BatterySystemsDep, MqlClientDep +from open_ess.frontend.dependencies import BatterySystemsDep, MqlClientDep, PriceConfigDep from .util import TimeSeries if TYPE_CHECKING: pass - logger = logging.getLogger(__name__) router = APIRouter(tags=["api"]) @@ -529,60 +528,37 @@ async def get_power_queries( ) -class PricePoint(BaseModel): - time: datetime - market: float | None - buy: float | None - sell: float | None +class PriceQueriesResponse(BaseModel): + market_query: str + buy_query: str + sell_query: str + step: Literal["15m", "1h"] + currency: str = "€" # TODO: based on area -class PricesResponse(BaseModel): - area: str - aggregate_minutes: int - unit: str = "€/kWh" # TODO: based on area - timeseries: list[PricePoint] +@router.get("/graph/price-queries", response_model=PriceQueriesResponse) +async def get_price_data( + price_config: PriceConfigDep, + area: str | None = Query(default=None), +) -> PriceQueriesResponse: + try: + if not area: + area = price_config.area + # TODO: validate area value + + step = "1h" if price_config.hourly_average else "15m" + + return PriceQueriesResponse( + market_query=f'avg_over_time(openess_prices{{area="{area}", price="market"}}[{step}])', + buy_query=f'avg_over_time(openess_prices{{area="{area}", price="buy"}}[{step}])', + sell_query=f'avg_over_time(openess_prices{{area="{area}", price="sell"}}[{step}])', + step=step, + ) + except Exception as e: + logger.exception("Failed to get prices") + raise HTTPException(status_code=500, detail=str(e)) from e -# @router.get("/prices", response_model=PricesResponse) -# async def get_price_data( -# price_config: PriceConfigDep, -# area: str | None = Query(default=None), -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# aggregate_minutes: int | None = Query(default=None), -# ) -> PricesResponse: -# try: -# if area is None: -# area = price_config.area -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(days=7) -# if end is None: -# end = now + timedelta(days=2) -# if aggregate_minutes is None: -# aggregate_minutes = price_config.aggregate_minutes -# -# timeseries = [] -# for timestamp, price in db.get_prices(area, start, end, aggregate_minutes=aggregate_minutes): -# timeseries.append( -# PricePoint( -# time=timestamp, -# market=round(price, 4), -# buy=round(price_config.buy_price(price), 4), -# sell=round(price_config.sell_price(price), 4), -# ) -# ) -# -# return PricesResponse( -# area=area, -# aggregate_minutes=aggregate_minutes, -# timeseries=timeseries, -# ) -# except Exception as e: -# logger.exception("Failed to get prices") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# # class BatteryGraphResponse(BaseModel): # soc: TimeSeries # schedule: TimeSeries # Scheduled (past and future) SoC diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index 28e0ce8..a81df1c 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -5,34 +5,12 @@ // === Types === // ============ -/** @typedef {"ok" | "warning" | "error"} Status */ - -/** @typedef {} StrEnum */ - /** * @typedef {Object} TimeSeries * @property {string[]} [timestamps] * @property {number[]} [values] */ -/** - * @typedef {Object} BatteryCycle - * @property {string} [start_time] - * @property {string} [end_time] - * @property {number} [duration_hours] - * @property {number} [min_soc] - * @property {(number | null)} [ac_energy_in] - * @property {(number | null)} [ac_energy_out] - * @property {number} [dc_energy_in] - * @property {number} [dc_energy_out] - * @property {(number | null)} [system_efficiency] - * @property {(number | null)} [battery_efficiency] - * @property {(number | null)} [charger_efficiency] - * @property {(number | null)} [inverter_efficiency] - * @property {(number | null)} [profit] - * @property {(number | null)} [scheduled_profit] - */ - /** * @typedef {Object} BatteryEnergySeries * @property {(number | null)[]} [energy_to_charger] @@ -44,35 +22,9 @@ */ /** - * @typedef {Object} BatteryGraphResponse - * @property {TimeSeries} [soc] - * @property {TimeSeries} [schedule] - * @property {TimeSeries} [voltage] - */ - -/** - * @typedef {Object} BatteryPowerValues - * @property {(number | null)} [charger] - * @property {(number | null)} [inverter] - * @property {(number | null)} [battery] - * @property {(number | null)} [losses] - */ - -/** - * @typedef {Object} BatterySystemInfo - * @property {string} [id] - * @property {string} [name] - */ - -/** - * @typedef {Object} EfficiencyScatterPoint - * @property {string} [time] - * @property {number} [battery_power] - * @property {number} [inverter_charger_power] - * @property {number} [losses] - * @property {(number | null)} [efficiency] - * @property {(number | null)} [soc] - * @property {string} [category] + * @typedef {Object} ChartsPowerResponse + * @property {PowerQueryDef[]} [queries] + * @property {string[]} [phases] */ /** @@ -99,11 +51,10 @@ */ /** - * @typedef {Object} PowerFlowData - * @property {Object.} [grid] - * @property {(number | null)} [solar] - * @property {Object.} [consumption] - * @property {Object.} [batteries] + * @typedef {Object} PowerQueryDef + * @property {string} [label] + * @property {string} [query] + * @property {(boolean | null)} is_total */ /** @@ -112,45 +63,12 @@ */ /** - * @typedef {Object} PricePoint - * @property {string} [time] - * @property {(number | null)} [market] - * @property {(number | null)} [buy] - * @property {(number | null)} [sell] - */ - -/** - * @typedef {Object} PricesResponse - * @property {string} [area] - * @property {number} [aggregate_minutes] - * @property {string} [unit] - * @property {PricePoint[]} [timeseries] - */ - -/** - * @typedef {Object} ServiceMessage - * @property {string} [timestamp] - * @property {Status} [status] - * @property {string} [message] - */ - -/** - * @typedef {Object} ServiceStatus - * @property {Status} [status] - * @property {ServiceMessage[]} [messages] - */ - -/** - * @typedef {Object} ServicesStatusResponse - * @property {(ServiceStatus | null)} [database] - * @property {(ServiceStatus | null)} [optimizer] - */ - -/** - * @typedef {Object} SystemLayoutData - * @property {number[]} [phases] - * @property {boolean} [has_solar] - * @property {BatterySystemInfo[]} [battery_systems] + * @typedef {Object} PriceQueriesResponse + * @property {string} [market_query] + * @property {string} [buy_query] + * @property {string} [sell_query] + * @property {Literal} [step] + * @property {string} [currency] */ /** @@ -179,85 +97,14 @@ }, /** - * @returns {Promise} - */ - systemLayout: async function() { - var response = await fetch('/api/system-layout'); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @returns {Promise} - */ - powerFlow: async function() { - var response = await fetch('/api/power-flow'); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @returns {Promise} - */ - servicesStatus: async function() { - var response = await fetch('/api/services-status'); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @returns {Promise} - */ - batteryIds: async function() { - var response = await fetch('/api/battery-ids'); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.bucket_minutes] - * @returns {Promise} + * @param {Annotated} params.timeseries + * @returns {Promise} */ - energyGraph: async function(params) { + chartsPowerQueries: async function(params) { var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.bucket_minutes !== undefined) searchParams.set('bucket_minutes', String(params.bucket_minutes)); + if (params.timeseries !== undefined) searchParams.set('timeseries', String(params.timeseries)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/energy-graph' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.aggregate_minutes] - * @returns {Promise} - */ - powerGraph: async function(params) { - var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/power-graph' + query); + var response = await fetch('/api/charts/power-queries' + query); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -266,114 +113,13 @@ /** * @param {(string | null)} [params.area] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {(number | null)} [params.aggregate_minutes] - * @returns {Promise} + * @returns {Promise} */ - prices: async function(params) { + graphPriceQueries: async function(params) { var searchParams = new URLSearchParams(); if (params.area !== undefined) searchParams.set('area', String(params.area)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/prices' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @returns {Promise>} - */ - batteryGraph: async function(params) { - var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/battery-graph' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {number} [params.limit] - * @param {number} [params.aggregate_minutes] - * @param {number} [params.idle_threshold] - * @returns {Promise} - */ - efficiencyScatter: async function(params) { - var searchParams = new URLSearchParams(); - if (params.limit !== undefined) searchParams.set('limit', String(params.limit)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); - if (params.idle_threshold !== undefined) searchParams.set('idle_threshold', String(params.idle_threshold)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/efficiency-scatter' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.min_soc_swing] - * @returns {Promise} - */ - cycles: async function(params) { - var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.min_soc_swing !== undefined) searchParams.set('min_soc_swing', String(params.min_soc_swing)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/cycles' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.aggregate_minutes] - * @returns {Promise} - */ - power: async function(params) { - var searchParams = new URLSearchParams(); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/power' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @returns {Promise} - */ - energy: async function(params) { - var searchParams = new URLSearchParams(); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/energy' + query); + var response = await fetch('/api/graph/price-queries' + query); if (!response.ok) { throw new Error('HTTP ' + response.status); } diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 5a51e57..21277f3 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -183,11 +183,7 @@ if (cachedPowerConfig) { return cachedPowerConfig; } - var response = await fetch('/api/charts/power-queries'); - if (!response.ok) { - throw new Error('Failed to fetch power chart config: HTTP ' + response.status); - } - cachedPowerConfig = await response.json(); + cachedPowerConfig = await Api.chartsPowerQueries({}); // Show/hide phase toggle button based on phases var toggleContainer = document.getElementById('power-phase-buttons'); @@ -218,49 +214,6 @@ }); } - /** - * Execute a single MetricsQL query and return Plotly trace data. - * @param {Object} queryDef - Query definition {query, label, group, variant} - * @param {Date} start - Start time - * @param {Date} end - End time - * @param {string} step - Step value like '5m' - * @returns {Promise} Plotly trace or null if no data - */ - async function executeQuery(queryDef, start, end, step) { - // Replace $step placeholder - var query = queryDef.query.replace(/\$step/g, step); - - var params = new URLSearchParams({ - query: query, - start: (start.getTime() / 1000).toString(), - end: (end.getTime() / 1000).toString(), - step: step, - }); - - var response = await fetch('/api/v1/query_range?' + params); - if (!response.ok) { - console.error('Query failed for', queryDef.label, ':', response.status); - return null; - } - - var result = await response.json(); - if (!result.data || !result.data.result || result.data.result.length === 0) { - return null; - } - - // Take the first series (most queries return single series) - var series = result.data.result[0]; - return { - x: series.values.map(function(v) { return new Date(v[0] * 1000); }), - y: series.values.map(function(v) { return parseFloat(v[1]); }), - type: 'scatter', - mode: 'lines', - name: queryDef.label, - line: { width: 1.5 }, - connectgaps: false, - }; - } - async function loadPowerChart(elementId, start, end, aggregateMinutes) { aggregateMinutes = aggregateMinutes || 5; Utils.showLoading(elementId); @@ -272,9 +225,17 @@ // Filter queries based on current mode var activeQueries = filterQueriesByMode(config.queries, currentPowerMode); - // Execute all queries in parallel - var tracePromises = activeQueries.map(function(q) { - return executeQuery(q, start, end, step); + // Execute all queries in parallel using Timeseries helper + var tracePromises = activeQueries.map(async function(q) { + var query = q.query.replace(/\$step/g, step); + try { + var result = await Timeseries.queryRangeRaw(query, start, end, step); + var traces = Timeseries.toPlotlyTraces(result, { name: q.label }); + return traces[0] || null; + } catch (e) { + console.error('Query failed for', q.label, ':', e); + return null; + } }); var traces = await Promise.all(tracePromises); @@ -313,67 +274,78 @@ } } + /** + * Extend trace data with one extra point for step-function display. + * @param {Object} trace - Plotly trace with x and y arrays + * @param {string} step - Step string like '1h' or '15m' + */ + function extendTraceForStepFunction(trace, step) { + if (trace.x.length > 0) { + var lastTime = trace.x[trace.x.length - 1]; + var stepMs = step === '1h' ? 3600000 : 900000; + trace.x.push(new Date(lastTime.getTime() + stepMs)); + trace.y.push(trace.y[trace.y.length - 1]); + } + } + async function loadPriceChart(elementId, start, end) { Utils.showLoading(elementId); + // Extend end to show future prices var extendedEnd = new Date(end.getTime() + 2 * 24 * 60 * 60 * 1000); try { - var data = await Api.prices({ - start: Utils.formatDate(start), - end: Utils.formatDate(extendedEnd), - }); + // Fetch query definitions from backend + var config = await Api.graphPriceQueries({}); + + // Execute all price queries in parallel + var [marketResult, buyResult, sellResult] = await Promise.all([ + Timeseries.queryRangeRaw(config.market_query, start, extendedEnd, config.step).catch(function() { return null; }), + Timeseries.queryRangeRaw(config.buy_query, start, extendedEnd, config.step).catch(function() { return null; }), + Timeseries.queryRangeRaw(config.sell_query, start, extendedEnd, config.step).catch(function() { return null; }), + ]); + + var marketTraces = Timeseries.toPlotlyTraces(marketResult, { name: 'Market' }); + var buyTraces = Timeseries.toPlotlyTraces(buyResult, { name: 'Buy' }); + var sellTraces = Timeseries.toPlotlyTraces(sellResult, { name: 'Sell' }); - if (!data.timeseries || data.timeseries.length === 0) { + if (marketTraces.length === 0 && buyTraces.length === 0 && sellTraces.length === 0) { Utils.showError(elementId, 'No price data available'); return; } var settings = Settings.load(); var priceMultiplier = settings.priceUnit === 'cent' ? 100 : 1; - var priceLabel = settings.priceUnit === 'cent' ? 'ct/kWh' : (data.unit || '€/kWh'); - - var timestamps = data.timeseries.map(function(d) { return new Date(d.time); }); - var marketPrices = data.timeseries.map(function(d) { return (d.market || 0) * priceMultiplier; }); - var buyPrices = data.timeseries.map(function(d) { return (d.buy || 0) * priceMultiplier; }); - var sellPrices = data.timeseries.map(function(d) { return (d.sell || 0) * priceMultiplier; }); - - var lastTime = timestamps[timestamps.length - 1]; - var extendedTime = new Date(lastTime.getTime() + (data.aggregate_minutes || 60) * 60 * 1000); - timestamps.push(extendedTime); - marketPrices.push(marketPrices[marketPrices.length - 1]); - buyPrices.push(buyPrices[buyPrices.length - 1]); - sellPrices.push(sellPrices[sellPrices.length - 1]); - - var traces = [ - { - name: 'Market', - x: timestamps, - y: marketPrices, - type: 'scatter', - mode: 'lines', - line: { shape: 'hv', color: '#95a5a6', width: 1 }, - hovertemplate: 'Market: %{y:.2f} ' + priceLabel + '', - }, - { - name: 'Buy', - x: timestamps, - y: buyPrices, - type: 'scatter', - mode: 'lines', - line: { shape: 'hv', color: '#e74c3c', width: 1.5 }, - hovertemplate: 'Buy: %{y:.2f} ' + priceLabel + '', - }, - { - name: 'Sell', - x: timestamps, - y: sellPrices, - type: 'scatter', - mode: 'lines', - line: { shape: 'hv', color: '#2ecc71', width: 1.5 }, - hovertemplate: 'Sell: %{y:.2f} ' + priceLabel + '', - }, - ]; + var priceLabel = settings.priceUnit === 'cent' ? 'ct/kWh' : (config.currency + '/kWh'); + + var traces = []; + + if (marketTraces[0]) { + var marketTrace = marketTraces[0]; + marketTrace.y = marketTrace.y.map(function(v) { return v * priceMultiplier; }); + marketTrace.line = { shape: 'hv', color: '#95a5a6', width: 1 }; + marketTrace.hovertemplate = 'Market: %{y:.2f} ' + priceLabel + ''; + extendTraceForStepFunction(marketTrace, config.step); + traces.push(marketTrace); + } + + if (buyTraces[0]) { + var buyTrace = buyTraces[0]; + buyTrace.y = buyTrace.y.map(function(v) { return v * priceMultiplier; }); + buyTrace.line = { shape: 'hv', color: '#e74c3c', width: 1.5 }; + buyTrace.hovertemplate = 'Buy: %{y:.2f} ' + priceLabel + ''; + extendTraceForStepFunction(buyTrace, config.step); + traces.push(buyTrace); + } + + if (sellTraces[0]) { + var sellTrace = sellTraces[0]; + sellTrace.y = sellTrace.y.map(function(v) { return v * priceMultiplier; }); + sellTrace.line = { shape: 'hv', color: '#2ecc71', width: 1.5 }; + sellTrace.hovertemplate = 'Sell: %{y:.2f} ' + priceLabel + ''; + extendTraceForStepFunction(sellTrace, config.step); + traces.push(sellTrace); + } var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, end); From 8b257c63aa46bc5b091e3028b1c8224aea0ca7f9 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 5 May 2026 15:06:13 +0200 Subject: [PATCH 10/18] frontend: battery graph uses /range_query --- open_ess/frontend/routes/api.py | 97 ++++++--------- open_ess/frontend/static/api.js | 22 +++- open_ess/frontend/static/metrics.js | 176 ++++++++++------------------ open_ess/optimizer/service.py | 37 +++++- 4 files changed, 155 insertions(+), 177 deletions(-) diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index 2243f61..f7dbe3e 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -417,14 +417,14 @@ class ChartsPowerResponse(BaseModel): @router.get("/charts/power-queries", response_model=ChartsPowerResponse) async def get_power_queries( - timeseries: MqlClientDep, + mql_client: MqlClientDep, battery_systems: BatterySystemsDep, ) -> ChartsPowerResponse: queries: list[PowerQueryDef] = [] phases: list[str] = [] - if timeseries is not None: - result = timeseries.query('openess_power_watts{from="grid"}') + if mql_client is not None: + result = mql_client.query('openess_power_watts{from="grid"}') phase_set: set[str] = set() if hasattr(result, "series"): for series in result.series: @@ -465,8 +465,8 @@ async def get_power_queries( # Discover phases for this battery system bs_phases: list[str] = [] - if timeseries is not None: - result = timeseries.query(f'openess_power_watts{{to="system", device="{device}"}}') + if mql_client is not None: + result = mql_client.query(f'openess_power_watts{{to="system", device="{device}"}}') phase_set: set[str] = set() if hasattr(result, "series"): for series in result.series: @@ -559,63 +559,36 @@ async def get_price_data( raise HTTPException(status_code=500, detail=str(e)) from e -# class BatteryGraphResponse(BaseModel): -# soc: TimeSeries -# schedule: TimeSeries # Scheduled (past and future) SoC -# voltage: TimeSeries -# -# -# @router.get("/battery-graph", response_model=dict[str, BatteryGraphResponse]) -# async def get_battery_graph( -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# battery_id: str | None = Query(default=None), -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# ) -> dict[str, BatteryGraphResponse]: -# try: -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(hours=48) -# if end is None: -# end = now + timedelta(hours=24) -# -# result = {} -# for battery_system in battery_systems: -# if battery_id is not None and battery_system.config.id != battery_id: -# continue -# -# # Schedule still comes from database (not in timeseries) -# scheduled = [(t, soc) for _, t, _, soc in db.get_schedule(battery_system.config.id, start)] -# -# # SOC and voltage from timeseries backend -# if timeseries is not None: -# device = battery_system.id -# soc_query = battery_system.config.queries.soc.replace("$device", device) -# voltage_query = battery_system.config.queries.voltage.replace("$device", device) -# -# soc_result = timeseries.query_range(soc_query, start, end, step="1m") -# voltage_result = timeseries.query_range(voltage_query, start, end, step="1m") -# -# result[battery_system.config.name] = BatteryGraphResponse( -# soc=query_result_to_timeseries(soc_result, rounding=1), -# schedule=data_to_timeseries(scheduled, rounding=1), -# voltage=query_result_to_timeseries(voltage_result, rounding=2), -# ) -# else: -# # Fallback to legacy database queries -# soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) -# voltage = db.get_voltage(battery_system.config.metrics.battery_voltage, start, end, bucket_seconds=60) -# -# result[battery_system.config.name] = BatteryGraphResponse( -# soc=data_to_timeseries(soc, rounding=1), -# schedule=data_to_timeseries(scheduled, rounding=1), -# voltage=data_to_timeseries(voltage, rounding=2), -# ) -# return result -# except Exception as e: -# logger.exception("Failed to get battery SOC") -# raise HTTPException(status_code=500, detail=str(e)) from e +class BatteryQueriesResponse(BaseModel): + soc_query: str + schedule_soc_query: str + voltage_query: str + + +@router.get("/charts/battery-queries", response_model=dict[str, BatteryQueriesResponse]) +async def get_battery_graph( + battery_systems: BatterySystemsDep, +) -> dict[str, BatteryQueriesResponse]: + try: + result = {} + for battery_system in battery_systems: + result[battery_system.config.name] = BatteryQueriesResponse( + soc_query=f""" + openess_soc_ratio{{device="{battery_system.id}", node="battery", unit="battery"}} * 100 + or + openess_soc_ratio{{device="{battery_system.id}", node="battery", unit="vebus"}} * 100 + """, + schedule_soc_query=f'first_over_time(openess_scheduled_soc_ratio{{device="{battery_system.id}"}}) * 100', + voltage_query=f""" + openess_voltage_volts{{device="{battery_system.id}", node="battery", unit="battery"}} + or + openess_voltage_volts{{device="{battery_system.id}", node="battery", unit="vebus"}} + """, + ) + return result + except Exception as e: + logger.exception("Failed to get battery SOC") + raise HTTPException(status_code=500, detail=str(e)) from e # # ---------------# diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index a81df1c..ea58600 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -21,6 +21,13 @@ * @property {(number | null)[]} [energy_loss_from_battery] */ +/** + * @typedef {Object} BatteryQueriesResponse + * @property {string} [soc_query] + * @property {string} [schedule_soc_query] + * @property {string} [voltage_query] + */ + /** * @typedef {Object} ChartsPowerResponse * @property {PowerQueryDef[]} [queries] @@ -97,12 +104,12 @@ }, /** - * @param {Annotated} params.timeseries + * @param {Annotated} params.mql_client * @returns {Promise} */ chartsPowerQueries: async function(params) { var searchParams = new URLSearchParams(); - if (params.timeseries !== undefined) searchParams.set('timeseries', String(params.timeseries)); + if (params.mql_client !== undefined) searchParams.set('mql_client', String(params.mql_client)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; var response = await fetch('/api/charts/power-queries' + query); if (!response.ok) { @@ -124,6 +131,17 @@ throw new Error('HTTP ' + response.status); } return response.json(); + }, + + /** + * @returns {Promise>} + */ + chartsBatteryQueries: async function() { + var response = await fetch('/api/charts/battery-queries'); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); } }; diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 21277f3..2ae5b9e 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -230,7 +230,10 @@ var query = q.query.replace(/\$step/g, step); try { var result = await Timeseries.queryRangeRaw(query, start, end, step); - var traces = Timeseries.toPlotlyTraces(result, { name: q.label }); + var traces = Timeseries.toPlotlyTraces(result); + if (traces[0]) { + traces[0].name = q.label; + } return traces[0] || null; } catch (e) { console.error('Query failed for', q.label, ':', e); @@ -359,53 +362,76 @@ } } - // Toggle: set to true to use new Timeseries API, false for legacy API - var USE_TIMESERIES_FOR_SOC = false; - - async function loadSocChartLegacy(elementId, start, end) { + async function loadSocChart(elementId, start, end) { Utils.showLoading(elementId); try { - var data = await Api.batteryGraph({ - start: Utils.formatDate(start), - end: Utils.formatDate(end), - }); - - var keys = Object.keys(data); - var multipleSystems = keys.length > 1; + // Fetch query definitions from backend + var config = await Api.chartsBatteryQueries(); + var batteryNames = Object.keys(config); + var multipleSystems = batteryNames.length > 1; var traces = []; - for (var i = 0; i < keys.length; i++) { - var name = keys[i]; - var battery = data[name]; + // Query all battery systems in parallel + var queryPromises = batteryNames.map(async function(name) { + var queries = config[name]; + var [socResult, scheduleResult, voltageResult] = await Promise.all([ + Timeseries.queryRangeRaw(queries.soc_query, start, end).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.schedule_soc_query, start, end).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.voltage_query, start, end).catch(function() { return null; }), + ]); + return { name: name, socResult: socResult, scheduleResult: scheduleResult, voltageResult: voltageResult }; + }); - var socTrace = Utils.makeTrace('SoC', Utils.timeseriesExtendToNow(battery.soc || { timestamps: [], values: [] })); - socTrace.line = { color: '#3498db', width: 2 }; - socTrace.hovertemplate = '%{y}%SoC'; - if (multipleSystems) { - socTrace.legendgroup = name; - socTrace.legendgrouptitle = { text: name }; + var results = await Promise.all(queryPromises); + + for (var i = 0; i < results.length; i++) { + var result = results[i]; + var batteryName = result.name; + var prefix = multipleSystems ? batteryName + ' ' : ''; + + // SOC trace + var socTraces = Timeseries.toPlotlyTraces(result.socResult); + if (socTraces[0]) { + var socTrace = socTraces[0]; + socTrace.name = prefix + 'SoC'; + socTrace.line = { color: '#3498db', width: 2 }; + socTrace.hovertemplate = '%{y:.1f}%' + socTrace.name + ''; + if (multipleSystems) { + socTrace.legendgroup = batteryName; + socTrace.legendgrouptitle = { text: batteryName }; + } + traces.push(socTrace); } - traces.push(socTrace); - - var schedTrace = Utils.makeTrace('Scheduled', battery.schedule || { timestamps: [], values: [] }); - schedTrace.line = { color: '#2ecc71', width: 2, dash: 'dot' }; - schedTrace.hovertemplate = '%{y}%Scheduled'; - if (multipleSystems) { - schedTrace.legendgroup = name; - schedTrace.legendgrouptitle = { text: name }; + + // Scheduled SOC trace + var schedTraces = Timeseries.toPlotlyTraces(result.scheduleResult); + if (schedTraces[0]) { + var schedTrace = schedTraces[0]; + schedTrace.name = prefix + 'Scheduled'; + schedTrace.line = { color: '#2ecc71', width: 2, dash: 'dot' }; + schedTrace.hovertemplate = '%{y:.1f}%' + schedTrace.name + ''; + if (multipleSystems) { + schedTrace.legendgroup = batteryName; + schedTrace.legendgrouptitle = { text: batteryName }; + } + traces.push(schedTrace); } - traces.push(schedTrace); - - var voltTrace = Utils.makeTrace('Voltage', battery.voltage || { timestamps: [], values: [] }); - voltTrace.line = { color: '#ff7171', width: 2 }; - voltTrace.hovertemplate = '%{y}VVoltage'; - voltTrace.yaxis = 'y2'; - if (multipleSystems) { - voltTrace.legendgroup = name; - voltTrace.legendgrouptitle = { text: name }; + + // Voltage trace (secondary y-axis) + var voltTraces = Timeseries.toPlotlyTraces(result.voltageResult); + if (voltTraces[0]) { + var voltTrace = voltTraces[0]; + voltTrace.name = prefix + 'Voltage'; + voltTrace.line = { color: '#ff7171', width: 2 }; + voltTrace.hovertemplate = '%{y:.1f}V' + voltTrace.name + ''; + voltTrace.yaxis = 'y2'; + if (multipleSystems) { + voltTrace.legendgroup = batteryName; + voltTrace.legendgrouptitle = { text: batteryName }; + } + traces.push(voltTrace); } - traces.push(voltTrace); } var layout = Utils.getDefaultLayout(); @@ -431,78 +457,6 @@ } } - /** - * Load SOC chart using the new Timeseries API. - * This queries the timeseries backend directly via /api/v1/query_range. - */ - async function loadSocChartTimeseries(elementId, start, end, batteryId) { - Utils.showLoading(elementId); - - try { - // Initialize Timeseries with battery system ID - await Timeseries.init(batteryId); - - // Query SOC and voltage in parallel - var [socResult, voltageResult] = await Promise.all([ - Timeseries.queryRange('soc', start, end), - Timeseries.queryRange('voltage', start, end), - ]); - - var traces = []; - - // Convert SOC result to Plotly trace - var socTraces = Timeseries.toPlotlyTraces(socResult, { name: 'SoC' }); - socTraces.forEach(function(trace) { - trace.line = { color: '#3498db', width: 2 }; - trace.hovertemplate = '%{y:.1f}%SoC'; - traces.push(trace); - }); - - // Convert voltage result to Plotly trace (on secondary y-axis) - var voltTraces = Timeseries.toPlotlyTraces(voltageResult, { name: 'Voltage' }); - voltTraces.forEach(function(trace) { - trace.line = { color: '#ff7171', width: 2 }; - trace.hovertemplate = '%{y:.1f}VVoltage'; - trace.yaxis = 'y2'; - traces.push(trace); - }); - - var layout = Utils.getDefaultLayout(); - Utils.layoutSetXRange(layout, start, end); - Utils.layoutAddNowLine(layout, start, end); - layout.yaxis = layout.yaxis || {}; - layout.yaxis.side = 'left'; - layout.yaxis.range = [0, 100]; - layout.yaxis.title = { text: "SoC (%)" }; - layout.yaxis2 = { - overlaying: 'y', - side: 'right', - gridcolor: 'transparent', - title: { text: "Voltage (V)" }, - }; - Utils.makePlot(elementId, traces, layout); - } catch (error) { - console.error('Error loading SoC data via Timeseries:', error); - Utils.showError(elementId, 'Failed to load SoC data'); - } - } - - async function loadSocChart(elementId, start, end) { - if (USE_TIMESERIES_FOR_SOC) { - // Get battery system ID from system layout - var layout = await Api.systemLayout(); - var batteryId = layout.battery_systems && layout.battery_systems[0] - ? layout.battery_systems[0].id - : null; - - if (batteryId) { - return loadSocChartTimeseries(elementId, start, end, batteryId); - } - console.warn('No battery system found, falling back to legacy API'); - } - return loadSocChartLegacy(elementId, start, end); - } - async function loadAndCacheEnergyData(start, end, bucketMinutes) { try { cachedEnergyData = await Api.energyGraph({ diff --git a/open_ess/optimizer/service.py b/open_ess/optimizer/service.py index 5b34535..54281c3 100644 --- a/open_ess/optimizer/service.py +++ b/open_ess/optimizer/service.py @@ -4,7 +4,7 @@ from open_ess.battery_system import BatterySystem from open_ess.pricing import PriceConfig from open_ess.service import Service -from open_ess.timeseries import TimeseriesBackend +from open_ess.timeseries import Sample, TimeseriesBackend from .optimizer import Optimizer @@ -37,7 +37,7 @@ def tick(self) -> None: if schedule: _, _, power, _ = schedule[0] self._battery_system.set_ess_setpoint(power) - # TODO self._db_conn.set_schedule(self._battery_system.id, schedule) # type: ignore[arg-type] + self._upsert_schedule(schedule) logger.debug(f"Updated schedule with {len(schedule)} entries") else: logger.warning("Optimizer returned empty schedule") @@ -51,3 +51,36 @@ def wait_until_next(self) -> None: microsecond=0, ) + timedelta(minutes=self._price_config.aggregate_minutes) self.wait_seconds((next_run - now).total_seconds()) + + def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]]): + """ + Schedules are stored in a bit of an insane way in the timeseries backend... + The timestamp of each inserted sample is increased by a tiny bit, proportional to how far in + the future the sample is. This way, a first_over_time() query will return the most recently + generated sample for a given timestamp. + """ + + samples: list[Sample] = [] + + now = datetime.now(UTC) + for ts_start, ts_end, power, soc in schedule: + delta = timedelta(milliseconds=1 + (ts_start - now).total_seconds() // 60) + + samples.append( + Sample( + metric="openess_scheduled_power_watts", + timestamp=ts_start + delta, + value=power, + labels={"device": self._battery_system.id}, + ) + ) + samples.append( + Sample( + metric="openess_scheduled_soc_ratio", + timestamp=ts_end + delta, + value=soc / 100, + labels={"device": self._battery_system.id}, + ) + ) + + self._mql_client.write(samples) From 00fec10eb60e1d9ee8fd06a718a2a7fd3e71b6b4 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 5 May 2026 15:53:20 +0200 Subject: [PATCH 11/18] move queries to BatterySystem --- open_ess/battery_system/__init__.py | 18 +++- open_ess/battery_system/battery_system.py | 111 ++++++++++++++++++++++ open_ess/frontend/routes/api.py | 110 +++++---------------- 3 files changed, 153 insertions(+), 86 deletions(-) diff --git a/open_ess/battery_system/__init__.py b/open_ess/battery_system/__init__.py index 4132a0f..eb375bb 100644 --- a/open_ess/battery_system/__init__.py +++ b/open_ess/battery_system/__init__.py @@ -1,4 +1,18 @@ -from .battery_system import BatterySystem, VictronBatterySystem +from .battery_system import ( + BatteryQueries, + BatterySystem, + PowerQueries, + PowerQueryDef, + VictronBatterySystem, +) from .config import BatterySystemConfig, QueriesConfig -__all__ = ["BatterySystem", "BatterySystemConfig", "QueriesConfig", "VictronBatterySystem"] +__all__ = [ + "BatteryQueries", + "BatterySystem", + "BatterySystemConfig", + "PowerQueries", + "PowerQueryDef", + "QueriesConfig", + "VictronBatterySystem", +] diff --git a/open_ess/battery_system/battery_system.py b/open_ess/battery_system/battery_system.py index 3c9c247..134b175 100644 --- a/open_ess/battery_system/battery_system.py +++ b/open_ess/battery_system/battery_system.py @@ -1,5 +1,6 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from open_ess.victron_modbus import VictronClient @@ -9,6 +10,32 @@ logger = logging.getLogger(__name__) +@dataclass +class PowerQueryDef: + """Definition of a power chart query.""" + + query: str + label: str + is_total: bool | None = None # True=total only, False=phases only, None=both + + +@dataclass +class PowerQueries: + """Power chart queries for a battery system.""" + + queries: list[PowerQueryDef] = field(default_factory=list) + phases: list[str] = field(default_factory=list) + + +@dataclass +class BatteryQueries: + """Battery chart queries (SOC, voltage, schedule).""" + + soc_query: str + voltage_query: str + schedule_soc_query: str + + class BatterySystem(ABC): def __init__(self, config: BatterySystemConfig): self._config = config @@ -37,6 +64,16 @@ def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: @abstractmethod def get_soc(self) -> float | None: ... + @abstractmethod + def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries: + """Return power chart queries for this battery system.""" + ... + + @abstractmethod + def get_battery_queries(self) -> BatteryQueries: + """Return battery chart queries (SOC, voltage, schedule).""" + ... + class VictronBatterySystem(BatterySystem): def __init__(self, config: BatterySystemConfig, control: VictronClient): @@ -60,3 +97,77 @@ def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: def get_soc(self) -> float | None: return self._victron_client.current_soc + + def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries: + device = self.device_serial or "unknown" + bs_name = self.config.name or self.id + queries: list[PowerQueryDef] = [] + + if phases and len(phases) > 1: + # Multi-phase: add total and per-phase queries + queries.append( + PowerQueryDef( + query=f""" + sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) + - on(device) + sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) + """, + label=f"{bs_name} AC", + is_total=True, + ) + ) + for phase in phases: + queries.append( + PowerQueryDef( + query=f""" + sum by (device, phase) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}", phase="{phase}"}}[$step])) + - on(device, phase) + sum by (device, phase) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}", phase="{phase}"}}[$step])) + """, + label=f"{bs_name} AC {phase}", + is_total=False, + ) + ) + else: + # Single phase or unknown + queries.append( + PowerQueryDef( + query=f""" + sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) + - on(device) + sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) + """, + label=f"{bs_name} AC", + ) + ) + + # Battery DC power (works for both single and multi-phase) + queries.append( + PowerQueryDef( + query=f""" + avg_over_time(openess_power_watts{{from="system", to="battery", unit="battery", device="{device}"}}[$step]) + or + avg_over_time(openess_power_watts{{from="system", to="battery", unit="vebus", device="{device}"}}[$step]) + """, + label=f"{bs_name} Battery", + ) + ) + + return PowerQueries(queries=queries, phases=phases or []) + + def get_battery_queries(self) -> BatteryQueries: + device = self.device_serial or "unknown" + + return BatteryQueries( + soc_query=f""" + openess_soc_ratio{{device="{device}", node="battery", unit="battery"}} * 100 + or + openess_soc_ratio{{device="{device}", node="battery", unit="vebus"}} * 100 + """, + voltage_query=f""" + openess_voltage_volts{{device="{device}", node="battery", unit="battery"}} + or + openess_voltage_volts{{device="{device}", node="battery", unit="vebus"}} + """, + schedule_soc_query=f'first_over_time(openess_scheduled_soc_ratio{{device="{device}"}}) * 100', + ) diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index f7dbe3e..98295e4 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -415,6 +415,19 @@ class ChartsPowerResponse(BaseModel): phases: list[str] +def _discover_phases(mql_client, query: str) -> list[str]: + if mql_client is None: + return [] + result = mql_client.query(query) + phase_set: set[str] = set() + if hasattr(result, "series"): + for series in result.series: + phase_label = series.metric.get("phase") + if phase_label: + phase_set.add(phase_label) + return sorted(phase_set) + + @router.get("/charts/power-queries", response_model=ChartsPowerResponse) async def get_power_queries( mql_client: MqlClientDep, @@ -422,16 +435,8 @@ async def get_power_queries( ) -> ChartsPowerResponse: queries: list[PowerQueryDef] = [] - phases: list[str] = [] - if mql_client is not None: - result = mql_client.query('openess_power_watts{from="grid"}') - phase_set: set[str] = set() - if hasattr(result, "series"): - for series in result.series: - phase_label = series.metric.get("phase") - if phase_label: - phase_set.add(phase_label) - phases = sorted(phase_set) + # Discover grid phases + phases = _discover_phases(mql_client, 'openess_power_watts{from="grid"}') # Grid power queries if len(phases) > 1: @@ -461,66 +466,10 @@ async def get_power_queries( # Battery system queries for bs in battery_systems: device = bs.device_serial or "unknown" - bs_name = bs.config.name or bs.id - - # Discover phases for this battery system - bs_phases: list[str] = [] - if mql_client is not None: - result = mql_client.query(f'openess_power_watts{{to="system", device="{device}"}}') - phase_set: set[str] = set() - if hasattr(result, "series"): - for series in result.series: - phase_label = series.metric.get("phase") - if phase_label: - phase_set.add(phase_label) - bs_phases = sorted(phase_set) - - if len(bs_phases) > 1: - queries.append( - PowerQueryDef( - query=f""" - sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) - - on(device) - sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) - """, - label=f"{bs_name} AC", - is_total=True, - ) - ) - for phase in bs_phases: - queries.append( - PowerQueryDef( - query=f""" - sum by (device, phase) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}", phase="{phase}"}}[$step])) - - on(device, phase) - sum by (device, phase) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}", phase="{phase}"}}[$step])) - """, - label=f"{bs_name} AC {phase}", - is_total=False, - ) - ) - else: - queries.append( - PowerQueryDef( - query=f""" - sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) - - on(device) - sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) - """, - label=f"{bs_name} AC", - ) - ) - - queries.append( - PowerQueryDef( - query=f""" - avg_over_time(openess_power_watts{{from="system", to="battery", unit="battery", device="{device}"}}[$step]) - or - avg_over_time(openess_power_watts{{from="system", to="battery", unit="vebus", device="{device}"}}[$step]) - """, - label=f"{bs_name} Battery", - ) - ) + bs_phases = _discover_phases(mql_client, f'openess_power_watts{{to="system", device="{device}"}}') + bs_queries = bs.get_power_queries(bs_phases) + for q in bs_queries.queries: + queries.append(PowerQueryDef(query=q.query, label=q.label, is_total=q.is_total)) return ChartsPowerResponse( queries=queries, @@ -546,7 +495,7 @@ async def get_price_data( area = price_config.area # TODO: validate area value - step = "1h" if price_config.hourly_average else "15m" + step: Literal["15m", "1h"] = "1h" if price_config.hourly_average else "15m" return PriceQueriesResponse( market_query=f'avg_over_time(openess_prices{{area="{area}", price="market"}}[{step}])', @@ -571,19 +520,12 @@ async def get_battery_graph( ) -> dict[str, BatteryQueriesResponse]: try: result = {} - for battery_system in battery_systems: - result[battery_system.config.name] = BatteryQueriesResponse( - soc_query=f""" - openess_soc_ratio{{device="{battery_system.id}", node="battery", unit="battery"}} * 100 - or - openess_soc_ratio{{device="{battery_system.id}", node="battery", unit="vebus"}} * 100 - """, - schedule_soc_query=f'first_over_time(openess_scheduled_soc_ratio{{device="{battery_system.id}"}}) * 100', - voltage_query=f""" - openess_voltage_volts{{device="{battery_system.id}", node="battery", unit="battery"}} - or - openess_voltage_volts{{device="{battery_system.id}", node="battery", unit="vebus"}} - """, + for bs in battery_systems: + queries = bs.get_battery_queries() + result[bs.config.name] = BatteryQueriesResponse( + soc_query=queries.soc_query, + schedule_soc_query=queries.schedule_soc_query, + voltage_query=queries.voltage_query, ) return result except Exception as e: From 3505ced52d18f643d7dd0a916592bd501b2f7bfe Mon Sep 17 00:00:00 2001 From: david Date: Tue, 5 May 2026 23:05:08 +0200 Subject: [PATCH 12/18] frontend: energy graph uses /range_query --- open_ess/battery_system/__init__.py | 6 + open_ess/battery_system/battery_system.py | 58 +++++- open_ess/frontend/routes/api.py | 105 +++++++---- open_ess/frontend/static/api.js | 48 +++-- open_ess/frontend/static/metrics.js | 214 +++++++++++----------- 5 files changed, 265 insertions(+), 166 deletions(-) diff --git a/open_ess/battery_system/__init__.py b/open_ess/battery_system/__init__.py index eb375bb..e0a8b59 100644 --- a/open_ess/battery_system/__init__.py +++ b/open_ess/battery_system/__init__.py @@ -1,6 +1,9 @@ from .battery_system import ( BatteryQueries, BatterySystem, + EnergyQueries, + EnergyQueryDef, + EnergyQuerySet, PowerQueries, PowerQueryDef, VictronBatterySystem, @@ -11,6 +14,9 @@ "BatteryQueries", "BatterySystem", "BatterySystemConfig", + "EnergyQueries", + "EnergyQueryDef", + "EnergyQuerySet", "PowerQueries", "PowerQueryDef", "QueriesConfig", diff --git a/open_ess/battery_system/battery_system.py b/open_ess/battery_system/battery_system.py index 134b175..0758b49 100644 --- a/open_ess/battery_system/battery_system.py +++ b/open_ess/battery_system/battery_system.py @@ -11,9 +11,30 @@ @dataclass -class PowerQueryDef: - """Definition of a power chart query.""" +class EnergyQueryDef: + query: str + label: str + + +@dataclass +class EnergyQueries: + queries: list[EnergyQueryDef] = field(default_factory=list) + +@dataclass +class EnergyQuerySet: + """Structured energy queries for a battery system.""" + + energy_to_charger: str # AC input to charger + energy_from_inverter: str # AC output from inverter + energy_to_battery: str # DC energy into battery + energy_from_battery: str # DC energy from battery + energy_loss_to_battery: str # Charge losses (AC - DC) + energy_loss_from_battery: str # Discharge losses (DC - AC) + + +@dataclass +class PowerQueryDef: query: str label: str is_total: bool | None = None # True=total only, False=phases only, None=both @@ -21,16 +42,12 @@ class PowerQueryDef: @dataclass class PowerQueries: - """Power chart queries for a battery system.""" - queries: list[PowerQueryDef] = field(default_factory=list) phases: list[str] = field(default_factory=list) @dataclass class BatteryQueries: - """Battery chart queries (SOC, voltage, schedule).""" - soc_query: str voltage_query: str schedule_soc_query: str @@ -64,6 +81,11 @@ def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: @abstractmethod def get_soc(self) -> float | None: ... + @abstractmethod + def get_energy_queries(self) -> EnergyQuerySet: + """Return structured energy queries for this battery system.""" + ... + @abstractmethod def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries: """Return power chart queries for this battery system.""" @@ -98,6 +120,30 @@ def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: def get_soc(self) -> float | None: return self._victron_client.current_soc + def get_energy_queries(self) -> EnergyQuerySet: + device = self.device_serial + + # AC side energy (charger input / inverter output) + energy_to_charger = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[$step])' + energy_from_inverter = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[$step])' + + # DC side energy (battery charge/discharge) + energy_to_battery = f'increase(openess_energy_kwh{{from="system", to="battery", device="{device}"}}[$step])' + energy_from_battery = f'increase(openess_energy_kwh{{from="battery", to="system", device="{device}"}}[$step])' + + # Losses = AC - DC (positive means energy lost as heat) + energy_loss_to_battery = f"({energy_to_charger}) - ({energy_to_battery})" + energy_loss_from_battery = f"({energy_from_battery}) - ({energy_from_inverter})" + + return EnergyQuerySet( + energy_to_charger=energy_to_charger, + energy_from_inverter=energy_from_inverter, + energy_to_battery=energy_to_battery, + energy_from_battery=energy_from_battery, + energy_loss_to_battery=energy_loss_to_battery, + energy_loss_from_battery=energy_loss_from_battery, + ) + def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries: device = self.device_serial or "unknown" bs_name = self.config.name or self.id diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index 98295e4..848c44c 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime from typing import TYPE_CHECKING, Literal from fastapi import APIRouter, HTTPException, Query @@ -259,26 +258,63 @@ async def health_check() -> HealthResponse: # ------------------------ # -class BatteryEnergySeries(BaseModel): - energy_to_charger: list[float | None] = [] - energy_from_inverter: list[float | None] = [] - energy_to_battery: list[float | None] = [] - energy_from_battery: list[float | None] = [] - energy_loss_to_battery: list[float | None] = [] - energy_loss_from_battery: list[float | None] = [] +class BatterySystemQueries(BaseModel): + energy_to_charger: str + energy_from_inverter: str + energy_to_battery: str + energy_from_battery: str + energy_loss_to_battery: str + energy_loss_from_battery: str -class EnergyGraphResponse(BaseModel): - timestamps: list[datetime] +class EnergyQueriesResponse(BaseModel): + grid_import_query: str + grid_export_query: str - grid_import: dict[str, list[float | None]] - grid_export: dict[str, list[float | None]] + battery_systems: dict[str, BatterySystemQueries] + solar_query: str - battery_systems: dict[str, BatteryEnergySeries] - solar: list[float | None] = [] - to_consumption: list[float | None] = [] - from_consumption: list[float | None] = [] +@router.get("/charts/energy-queries", response_model=EnergyQueriesResponse) +async def get_energy_queries(battery_systems: BatterySystemsDep) -> EnergyQueriesResponse: + try: + battery_system_queries: dict[str, BatterySystemQueries] = {} + grid_import_parts: list[str] = [] + grid_export_parts: list[str] = [] + + for battery_system in battery_systems: + device = battery_system.device_serial + queries = battery_system.get_energy_queries() + + battery_system_queries[battery_system.id] = BatterySystemQueries( + energy_to_charger=queries.energy_to_charger, + energy_from_inverter=queries.energy_from_inverter, + energy_to_battery=queries.energy_to_battery, + energy_from_battery=queries.energy_from_battery, + energy_loss_to_battery=queries.energy_loss_to_battery, + energy_loss_from_battery=queries.energy_loss_from_battery, + ) + + # Collect grid queries per device (will sum across all battery systems) + grid_import_parts.append( + f'(increase(openess_energy_kwh{{from="grid", to="system", device="{device}"}}[$step]))' + ) + grid_export_parts.append( + f'(increase(openess_energy_kwh{{from="system", to="grid", device="{device}"}}[$step]))' + ) + + grid_import_query = " + ".join(grid_import_parts) + grid_export_query = " + ".join(grid_export_parts) + + return EnergyQueriesResponse( + grid_import_query=grid_import_query, + grid_export_query=grid_export_query, + battery_systems=battery_system_queries, + solar_query="", # TODO: add solar query when solar support is implemented + ) + except Exception as e: + logger.exception("Failed to get energy queries") + raise HTTPException(status_code=500, detail=str(e)) from e # @router.get("/energy-graph", response_model=EnergyGraphResponse) @@ -389,21 +425,6 @@ class EnergyGraphResponse(BaseModel): # ) -def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: - """Calculate query step from aggregate_minutes or time range.""" - if aggregate_minutes > 1: - return f"{aggregate_minutes}m" - # Auto-calculate based on range - duration = (end - start).total_seconds() - if duration <= 3600: # 1 hour - return "1m" - if duration <= 6 * 3600: # 6 hours - return "5m" - if duration <= 24 * 3600: # 24 hours - return "15m" - return "1h" - - class PowerQueryDef(BaseModel): label: str query: str @@ -485,8 +506,8 @@ class PriceQueriesResponse(BaseModel): currency: str = "€" # TODO: based on area -@router.get("/graph/price-queries", response_model=PriceQueriesResponse) -async def get_price_data( +@router.get("/charts/price-queries", response_model=PriceQueriesResponse) +async def get_price_queries( price_config: PriceConfigDep, area: str | None = Query(default=None), ) -> PriceQueriesResponse: @@ -515,7 +536,7 @@ class BatteryQueriesResponse(BaseModel): @router.get("/charts/battery-queries", response_model=dict[str, BatteryQueriesResponse]) -async def get_battery_graph( +async def get_battery_queries( battery_systems: BatterySystemsDep, ) -> dict[str, BatteryQueriesResponse]: try: @@ -529,7 +550,7 @@ async def get_battery_graph( ) return result except Exception as e: - logger.exception("Failed to get battery SOC") + logger.exception("Failed to get battery queries") raise HTTPException(status_code=500, detail=str(e)) from e @@ -880,6 +901,20 @@ async def get_battery_graph( # # Generalized endpoints # # # -------------------------# # + +# def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: +# """Calculate query step from aggregate_minutes or time range.""" +# if aggregate_minutes > 1: +# return f"{aggregate_minutes}m" +# # Auto-calculate based on range +# duration = (end - start).total_seconds() +# if duration <= 3600: # 1 hour +# return "1m" +# if duration <= 6 * 3600: # 6 hours +# return "5m" +# if duration <= 24 * 3600: # 24 hours +# return "15m" +# return "1h" # # # TODO: add parameter to select subset of series # @router.get("/power", response_model=PowerResponse) diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index ea58600..151f736 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -11,16 +11,6 @@ * @property {number[]} [values] */ -/** - * @typedef {Object} BatteryEnergySeries - * @property {(number | null)[]} [energy_to_charger] - * @property {(number | null)[]} [energy_from_inverter] - * @property {(number | null)[]} [energy_to_battery] - * @property {(number | null)[]} [energy_from_battery] - * @property {(number | null)[]} [energy_loss_to_battery] - * @property {(number | null)[]} [energy_loss_from_battery] - */ - /** * @typedef {Object} BatteryQueriesResponse * @property {string} [soc_query] @@ -28,6 +18,16 @@ * @property {string} [voltage_query] */ +/** + * @typedef {Object} BatterySystemQueries + * @property {string} [energy_to_charger] + * @property {string} [energy_from_inverter] + * @property {string} [energy_to_battery] + * @property {string} [energy_from_battery] + * @property {string} [energy_loss_to_battery] + * @property {string} [energy_loss_from_battery] + */ + /** * @typedef {Object} ChartsPowerResponse * @property {PowerQueryDef[]} [queries] @@ -35,14 +35,11 @@ */ /** - * @typedef {Object} EnergyGraphResponse - * @property {string[]} [timestamps] - * @property {Object.} [grid_import] - * @property {Object.} [grid_export] - * @property {Object.} [battery_systems] - * @property {(number | null)[]} [solar] - * @property {(number | null)[]} [to_consumption] - * @property {(number | null)[]} [from_consumption] + * @typedef {Object} EnergyQueriesResponse + * @property {string} [grid_import_query] + * @property {string} [grid_export_query] + * @property {Object.} [battery_systems] + * @property {string} [solar_query] */ /** @@ -103,6 +100,17 @@ return response.json(); }, + /** + * @returns {Promise} + */ + chartsEnergyQueries: async function() { + var response = await fetch('/api/charts/energy-queries'); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); + }, + /** * @param {Annotated} params.mql_client * @returns {Promise} @@ -122,11 +130,11 @@ * @param {(string | null)} [params.area] * @returns {Promise} */ - graphPriceQueries: async function(params) { + chartsPriceQueries: async function(params) { var searchParams = new URLSearchParams(); if (params.area !== undefined) searchParams.set('area', String(params.area)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/graph/price-queries' + query); + var response = await fetch('/api/charts/price-queries' + query); if (!response.ok) { throw new Error('HTTP ' + response.status); } diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 2ae5b9e..39c988f 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -6,7 +6,6 @@ var dashboardEnd = null; var currentFoR = 'multiplus'; var currentPowerMode = 'total'; // 'total' or 'phases' - var cachedEnergyData = null; var cachedPowerConfig = null; // Cached power chart config var rangeOffset = 0; var isRelayoutInProgress = false; @@ -90,88 +89,110 @@ }); } - function renderGridEnergyChart(elementId, data, start, end) { - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var toDisplay = useKw ? function(wh) { return wh ? wh / 1000 : 0; } : function(wh) { return wh || 0; }; - - var timestamps = (data.timestamps || []).map(function(t) { return new Date(t); }); - - var gridExport = data.grid_export || {}; - var gridImport = data.grid_import || {}; - - var traces = [ - { - x: timestamps, - y: (gridExport["From MP"] || []).map(function(v) { return toDisplay(v); }), - type: 'bar', - name: 'From MP', - marker: { color: '#278e60' }, - textposition: 'none', - }, - { - x: timestamps, - y: (gridImport["Consumption"] || []).map(function(v) { return -toDisplay(v); }), - type: 'bar', - name: 'Consumption', - marker: { color: '#3498db' }, - textposition: 'none', - }, - { - x: timestamps, - y: (gridImport["To MP"] || []).map(function(v) { return -toDisplay(v); }), - type: 'bar', - name: 'To MP', - marker: { color: '#3498ab' }, - textposition: 'none', - }, - ]; - - var layout = Utils.getDefaultLayout(); - Utils.layoutSetXRange(layout, start, end); - Utils.layoutAddNowLine(layout, start, end); - Utils.makePlot(elementId, traces, layout); - } + var cachedEnergyConfig = null; - function renderBatteryEnergyChart(elementId, data, start, end) { - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var toDisplay = useKw ? function(wh) { return wh ? wh / 1000 : 0; } : function(wh) { return wh || 0; }; - - var timestamps = (data.timestamps || []).map(function(t) { return new Date(t); }); - var mpData = (data.battery_systems || {})["MultiPlus"] || {}; - - var traces = [ - { - x: timestamps, - y: (mpData.energy_from_inverter || []).map(function(v) { return toDisplay(v); }), - type: 'bar', - name: 'Inverter Output', - marker: { color: '#f39c12' }, - textposition: 'none', - }, - { - x: timestamps, - y: (mpData.energy_to_charger || []).map(function(v) { return -toDisplay(v); }), - type: 'bar', - name: 'Charger Input', - marker: { color: '#3498db' }, - textposition: 'none', - }, - ]; - - var layout = Utils.getDefaultLayout(); - Utils.layoutSetXRange(layout, start, end); - Utils.layoutAddNowLine(layout, start, end); - Utils.makePlot(elementId, traces, layout); + /** + * Execute a MetricsQL query and return a Plotly trace. + * @param {string} query - MetricsQL query with $step placeholder + * @param {string} label - Trace label + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} step - Step string like '60m' + * @param {Object} opts - Trace options (color, negate) + * @returns {Promise} Plotly trace or null + */ + async function executeEnergyQuery(query, label, start, end, step, opts) { + if (!query) return null; + opts = opts || {}; + + try { + var resolvedQuery = query.replace(/\$step/g, step); + var result = await Timeseries.queryRangeRaw(resolvedQuery, start, end, step); + var plotlyTraces = Timeseries.toPlotlyTraces(result); + + if (plotlyTraces[0]) { + var trace = plotlyTraces[0]; + trace.name = label; + trace.type = 'bar'; + delete trace.mode; + delete trace.line; + trace.marker = { color: opts.color || '#95a5a6' }; + + if (opts.negate) { + trace.y = trace.y.map(function(v) { return -v; }); + } + + return trace; + } + } catch (e) { + console.error('Query failed for', label, ':', e); + } + return null; } - function renderEnergyFlowChart(elementId, data, start, end, frameOfReference) { - frameOfReference = frameOfReference || 'multiplus'; - if (frameOfReference === 'grid') { - renderGridEnergyChart(elementId, data, start, end); - } else { - renderBatteryEnergyChart(elementId, data, start, end); + async function loadEnergyChart(elementId, start, end, bucketMinutes) { + Utils.showLoading(elementId); + + try { + // Fetch query definitions (cached after first call) + if (!cachedEnergyConfig) { + cachedEnergyConfig = await Api.chartsEnergyQueries(); + } + + var step = bucketMinutes + 'm'; + var promises = []; + + // Grid queries (system-wide) + promises.push(executeEnergyQuery( + cachedEnergyConfig.grid_import_query, 'Grid Import', + start, end, step, { color: '#e74c3c', negate: true } + )); + promises.push(executeEnergyQuery( + cachedEnergyConfig.grid_export_query, 'Grid Export', + start, end, step, { color: '#2ecc71' } + )); + + // Solar query (if available) + if (cachedEnergyConfig.solar_query) { + promises.push(executeEnergyQuery( + cachedEnergyConfig.solar_query, 'Solar', + start, end, step, { color: '#f1c40f' } + )); + } + + // Per-battery-system queries + var batteryIds = Object.keys(cachedEnergyConfig.battery_systems || {}); + var multipleSystems = batteryIds.length > 1; + + for (var i = 0; i < batteryIds.length; i++) { + var bsId = batteryIds[i]; + var bs = cachedEnergyConfig.battery_systems[bsId]; + var prefix = multipleSystems ? bsId + ' ' : ''; + + // AC side: charger input (charge) and inverter output (discharge) + promises.push(executeEnergyQuery( + bs.energy_to_charger, prefix + 'Charge', + start, end, step, { color: '#3498db', negate: true } + )); + promises.push(executeEnergyQuery( + bs.energy_from_inverter, prefix + 'Discharge', + start, end, step, { color: '#f39c12' } + )); + } + + var results = await Promise.all(promises); + var traces = results.filter(function(t) { return t !== null; }); + + var layout = Utils.getDefaultLayout(); + layout.barmode = 'relative'; + Utils.layoutSetXRange(layout, start, end); + Utils.layoutAddNowLine(layout, start, end); + layout.yaxis = layout.yaxis || {}; + layout.yaxis.title = { text: 'kWh' }; + Utils.makePlot(elementId, traces, layout); + } catch (error) { + console.error('Error loading energy data:', error); + Utils.showError(elementId, 'Failed to load energy data'); } } @@ -299,7 +320,7 @@ try { // Fetch query definitions from backend - var config = await Api.graphPriceQueries({}); + var config = await Api.chartsPriceQueries({}); // Execute all price queries in parallel var [marketResult, buyResult, sellResult] = await Promise.all([ @@ -457,19 +478,6 @@ } } - async function loadAndCacheEnergyData(start, end, bucketMinutes) { - try { - cachedEnergyData = await Api.energyGraph({ - start: Utils.formatDate(start), - end: Utils.formatDate(end), - bucket_minutes: bucketMinutes, - }); - } catch (error) { - console.error('Error fetching energy data:', error); - cachedEnergyData = null; - } - } - async function loadDashboard() { var hours = parseInt(document.getElementById('range-select').value); var aggregateMinutes = getAggregateMinutes(hours); @@ -481,25 +489,21 @@ updateRangeLabel(); - cachedEnergyData = null; - await Promise.all([ - loadAndCacheEnergyData(dashboardStart, dashboardEnd, bucketMinutes), + loadEnergyChart('energy-chart', dashboardStart, dashboardEnd, bucketMinutes), loadPowerChart('power-chart', dashboardStart, dashboardEnd, aggregateMinutes), loadPriceChart('prices-chart', dashboardStart, dashboardEnd), loadSocChart('soc-chart', dashboardStart, dashboardEnd) ]); - if (cachedEnergyData) { - renderEnergyFlowChart('energy-chart', cachedEnergyData, dashboardStart, dashboardEnd, currentFoR); - } - setupZoomSync(); } - function renderEnergyOnly() { - if (dashboardStart && dashboardEnd && cachedEnergyData) { - renderEnergyFlowChart('energy-chart', cachedEnergyData, dashboardStart, dashboardEnd, currentFoR); + function reloadEnergyChart() { + if (dashboardStart && dashboardEnd) { + var hours = parseInt(document.getElementById('range-select').value); + var bucketMinutes = getBucketMinutes(hours); + loadEnergyChart('energy-chart', dashboardStart, dashboardEnd, bucketMinutes); } } @@ -548,7 +552,7 @@ btn.classList.add('active'); currentFoR = btn.dataset.value || 'multiplus'; Settings.savePagePref('dashboard', 'for', currentFoR); - renderEnergyOnly(); + reloadEnergyChart(); }); }); From 0bd3549f07df1fa4630dd5429650e6d172f4fb8d Mon Sep 17 00:00:00 2001 From: david Date: Thu, 7 May 2026 22:13:22 +0200 Subject: [PATCH 13/18] frontend: all pages uses /range_query --- open_ess/battery_system/__init__.py | 3 +- open_ess/battery_system/config.py | 36 - open_ess/frontend/routes/api.py | 1459 +++++++++------------- open_ess/frontend/static/api.js | 103 +- open_ess/frontend/static/cycles.js | 214 ++-- open_ess/frontend/static/dashboard.js | 7 +- open_ess/frontend/static/debug.js | 141 ++- open_ess/frontend/static/metrics.js | 144 ++- open_ess/frontend/static/timeseries.js | 61 + open_ess/frontend/templates/cycles.html | 4 + open_ess/frontend/templates/metrics.html | 4 +- 11 files changed, 1066 insertions(+), 1110 deletions(-) diff --git a/open_ess/battery_system/__init__.py b/open_ess/battery_system/__init__.py index e0a8b59..1ce04ef 100644 --- a/open_ess/battery_system/__init__.py +++ b/open_ess/battery_system/__init__.py @@ -8,7 +8,7 @@ PowerQueryDef, VictronBatterySystem, ) -from .config import BatterySystemConfig, QueriesConfig +from .config import BatterySystemConfig __all__ = [ "BatteryQueries", @@ -19,6 +19,5 @@ "EnergyQuerySet", "PowerQueries", "PowerQueryDef", - "QueriesConfig", "VictronBatterySystem", ] diff --git a/open_ess/battery_system/config.py b/open_ess/battery_system/config.py index a6c46d4..9f45038 100644 --- a/open_ess/battery_system/config.py +++ b/open_ess/battery_system/config.py @@ -10,42 +10,6 @@ class MqttControl(BaseModel): topic: str -class QueriesConfig(BaseModel): - # Battery state - # Readings from battery (BMS) have priority over readings from vebus (MultiPlus). - soc: str = """ - ( - openess_soc_ratio{device=~"$device", node="battery", unit="battery"} - or - openess_soc_ratio{device=~"$device", node="battery", unit="vebus"} - ) * 100 - """ - voltage: str = """ - ( - openess_voltage_volts{device=~"$device", node="battery", unit="battery"} - or - openess_voltage_volts{device=~"$device", node="battery", unit="vebus"} - ) * 100 - """ - - # Power - system_power: str = """ - openess_power_watts{device="$device", phase=~"$phase", from="ac_in", to="system"} - - - openess_power_watts{device="$device", phase=~"$phase", from="system", to="ac_out"} - """ - battery_power: str = """ - openess_power_watts{device="$device", from="system", to="battery", unit="battery"} - or - openess_power_watts{device="$device", from="system", to="battery", unit="vebus"} - """ - - # energy_to_system: str | list[str] | None = None - # energy_from_system: str | list[str] | None = None - # energy_to_battery: str | list[str] | None = None - # energy_from_battery: str | list[str] | None = None - - class BatterySystemConfig(BaseModel): name: str | None = None monitor_only: bool = False diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index 848c44c..4bda32d 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,10 +1,13 @@ import logging +from datetime import UTC, datetime, timedelta +from enum import StrEnum from typing import TYPE_CHECKING, Literal from fastapi import APIRouter, HTTPException, Query from pydantic import BaseModel from open_ess.frontend.dependencies import BatterySystemsDep, MqlClientDep, PriceConfigDep +from open_ess.timeseries import TimeseriesBackend from .util import TimeSeries @@ -41,216 +44,192 @@ async def health_check() -> HealthResponse: raise HTTPException(status_code=500, detail=str(e)) from e -# # ---------------------------- # -# # Power overview (Dashboard) # -# # ---------------------------- # -# -# -# class BatterySystemInfo(BaseModel): -# id: str -# name: str -# -# -# class SystemLayoutData(BaseModel): -# phases: list[int] -# # TODO: grid_labels: list[str] # ["L1", "L2", "L3"] -# has_solar: bool -# battery_systems: list[BatterySystemInfo] -# -# -# @router.get("/system-layout", response_model=SystemLayoutData) -# async def get_system_layout(battery_systems: BatterySystemsDep) -> SystemLayoutData: -# return SystemLayoutData( -# phases=[1, 2, 3], -# # grid_labels=["L1", "L2", "L3"], -# has_solar=True, # TODO -# battery_systems=[BatterySystemInfo(id=b.id, name=b.name) for b in battery_systems], -# ) -# -# -# class BatteryPowerValues(BaseModel): -# charger: float | None -# inverter: float | None -# battery: float | None -# losses: float | None -# -# -# class PowerFlowData(BaseModel): -# grid: dict[str, float | None] -# solar: float | None -# consumption: dict[str, float] # e.g. {"L1": 800, "L2": 300, "L3": 200} -# batteries: dict[str, BatteryPowerValues] -# -# -# def _get_instant_value(result: "QueryResult") -> float | None: -# """Extract the latest value from an instant query result.""" -# if result.series and result.series[0].values: -# return result.series[0].values[-1][1] -# return None -# -# -# @router.get("/power-flow", response_model=PowerFlowData) -# async def get_power_flow( -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# ) -> PowerFlowData: -# if timeseries is not None: -# return await _get_power_flow_timeseries(timeseries, battery_systems) -# return await _get_power_flow_legacy(battery_systems) -# -# -# async def _get_power_flow_timeseries( -# timeseries: "TimeseriesBackend", -# battery_systems: list, -# ) -> PowerFlowData: -# """Get power flow data from timeseries backend.""" -# now = datetime.now(UTC) -# -# # Grid power per phase -# grid_power: dict[str, float | None] = {} -# for phase in ("L1", "L2", "L3"): -# query = f'openess_power_watts{{from="grid", phase="{phase}"}}' -# result = timeseries.query(query, now) -# grid_power[phase] = _get_instant_value(result) -# -# # Solar power -# solar_query = 'openess_power_watts{from="pvinverter"}' -# solar_result = timeseries.query(solar_query, now) -# solar_power = _get_instant_value(solar_result) -# -# # Battery power for each system -# batteries: dict[str, BatteryPowerValues] = {} -# for battery_system in battery_systems: -# device = battery_system.id -# -# # AC power (charger/inverter) -# ac_in_query = battery_system.config.queries.power_ac_in.replace("$device", device) -# ac_in_result = timeseries.query(ac_in_query, now) -# system = _get_instant_value(ac_in_result) or 0 -# -# charger = -system if system < 0 else 0 -# inverter = system if system > 0 else 0 -# -# # DC battery power -# battery_query = battery_system.config.queries.power_battery.replace("$device", device) -# battery_result = timeseries.query(battery_query, now) -# battery = _get_instant_value(battery_result) or 0 -# -# losses = battery - system -# -# batteries[battery_system.id] = BatteryPowerValues( -# charger=charger, -# inverter=inverter, -# battery=battery, -# losses=losses, -# ) -# -# return PowerFlowData( -# grid=grid_power, -# solar=solar_power, -# consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, -# batteries=batteries, -# ) -# -# -# async def _get_power_flow_legacy(battery_systems: list) -> PowerFlowData: -# """Get power flow data from legacy database.""" -# start = datetime.now(UTC) - timedelta(seconds=10) -# -# grid_power: dict[str, float | None] = {} -# for i in (1, 2, 3): -# power = None -# result = db.get_power(f"grid/power/l{i}", start=start, bucket_seconds=None) -# if result: -# _, power = result[-1] -# grid_power[f"L{i}"] = power -# -# solar_power = None -# result = db.get_power("victron/pvinverter/31/power/l1", start=start, bucket_seconds=None) -# if result: -# _, solar_power = result[-1] -# -# batteries: dict[str, BatteryPowerValues] = {} -# for battery_system in battery_systems: -# charger = 0 -# inverter = 0 -# battery = 0 -# losses = 0 -# system = 0 -# result = db.get_power(battery_system.config.metrics.power_to_system, start=start, bucket_seconds=None) -# if result: -# _, system = result[-1] -# if system < 0: -# charger = -system -# if system > 0: -# inverter = system -# -# result = db.get_power(battery_system.config.metrics.power_to_battery, start=start, bucket_seconds=None) -# if result: -# _, battery = result[-1] -# losses = battery - system -# -# batteries[battery_system.id] = BatteryPowerValues( -# charger=charger, -# inverter=inverter, -# battery=battery, -# losses=losses, -# ) -# -# return PowerFlowData( -# grid=grid_power, -# solar=solar_power, -# consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, -# batteries=batteries, -# ) -# -# -# # ------------------------------- # -# # Services overview (Dashboard) # -# # ------------------------------- # -# -# -# class Status(StrEnum): -# OK = "ok" -# WARNING = "warning" -# ERROR = "error" -# -# -# class ServiceMessage(BaseModel): -# timestamp: datetime -# status: Status -# message: str -# -# -# class ServiceStatus(BaseModel): -# status: Status -# messages: list[ServiceMessage] -# -# -# class ServicesStatusResponse(BaseModel): -# database: ServiceStatus | None -# optimizer: ServiceStatus | None -# -# -# @router.get("/services-status", response_model=ServicesStatusResponse) -# async def services_status() -> ServicesStatusResponse: -# try: -# return ServicesStatusResponse( -# database=ServiceStatus(status=Status.OK, messages=[]), -# optimizer=ServiceStatus(status=Status.OK, messages=[]), -# ) -# except Exception as e: -# logger.exception("Health check failed") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# @router.get("/battery-ids", response_model=list[str]) -# async def get_battery_ids(battery_systems: BatterySystemsDep) -> list[str]: -# try: -# return [s.id for s in battery_systems] -# except Exception as e: -# logger.exception("Failed to get battery ids") -# raise HTTPException(status_code=500, detail=str(e)) from e +# ---------------------------- # +# Power overview (Dashboard) # +# ---------------------------- # + + +class BatterySystemInfo(BaseModel): + id: str + name: str + + +class SystemLayoutData(BaseModel): + phases: list[int] + has_solar: bool + battery_systems: list[BatterySystemInfo] + + +@router.get("/system-layout", response_model=SystemLayoutData) +async def get_system_layout( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, +) -> SystemLayoutData: + # Discover phases from grid power metrics + phases: list[int] = [] + if mql_client: + result = mql_client.query('openess_power_watts{from="grid"}') + phase_set: set[int] = set() + if hasattr(result, "series"): + for series in result.series: + phase_label = series.metric.get("phase", "") + if phase_label.startswith("L"): + phase_set.add(int(phase_label[1:])) + phases = sorted(phase_set) if phase_set else [1, 2, 3] + else: + phases = [1, 2, 3] + + # TODO: detect solar from metrics + has_solar = False + + return SystemLayoutData( + phases=phases, + has_solar=has_solar, + battery_systems=[BatterySystemInfo(id=b.id, name=b.name or b.id) for b in battery_systems], + ) + + +class BatteryPowerValues(BaseModel): + charger: float | None + inverter: float | None + battery: float | None + losses: float | None + soc: float | None + + +class PowerFlowData(BaseModel): + grid: dict[str, float | None] + solar: float | None + consumption: dict[str, float] + batteries: dict[str, BatteryPowerValues] + + +def _get_instant_value(result) -> float | None: + """Extract the value from an instant query result.""" + if hasattr(result, "series") and result.series: + return result.series[0].value + return None + + +def _get_instant_values_by_label(result, label_key: str) -> dict[str, float]: + """Extract values from an instant query result, keyed by a label.""" + values: dict[str, float] = {} + if hasattr(result, "series"): + for series in result.series: + key = series.metric.get(label_key, "") + if key: + values[key] = series.value + return values + + +@router.get("/power-flow", response_model=PowerFlowData) +async def get_power_flow( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, +) -> PowerFlowData: + if mql_client is None: + raise HTTPException(503, "Timeseries backend not configured") + + # Grid power per phase + grid_result = mql_client.query('openess_power_watts{from="grid"}') + grid_power = _get_instant_values_by_label(grid_result, "phase") + + # Solar power (sum all PV inverter phases) + solar_result = mql_client.query('sum(openess_power_watts{from="pvinverter"})') + solar_power = _get_instant_value(solar_result) + + # Battery power for each system + batteries: dict[str, BatteryPowerValues] = {} + for battery_system in battery_systems: + device = battery_system.device_serial + + # AC power: charger input - inverter output + ac_in_result = mql_client.query(f'sum(openess_power_watts{{from="ac_in", to="system", device="{device}"}})') + ac_out_result = mql_client.query(f'sum(openess_power_watts{{from="system", to="ac_out", device="{device}"}})') + ac_in = _get_instant_value(ac_in_result) or 0 + ac_out = _get_instant_value(ac_out_result) or 0 + + charger = ac_in + inverter = ac_out + + # DC battery power + battery_result = mql_client.query(f'openess_power_watts{{from="system", to="battery", device="{device}"}}') + battery_power = _get_instant_value(battery_result) or 0 + + # SOC + soc_result = mql_client.query(f'openess_soc_ratio{{device="{device}", node="battery"}} * 100') + soc = _get_instant_value(soc_result) + + # Losses = AC net - DC + ac_net = ac_in - ac_out + losses = ac_net - battery_power + + batteries[battery_system.id] = BatteryPowerValues( + charger=charger, + inverter=inverter, + battery=battery_power, + losses=losses, + soc=soc, + ) + + # Consumption = grid + solar + battery discharge - battery charge + # For now, return zeros (would need more complex calculation) + consumption: dict[str, float] = {} + for phase, grid_val in grid_power.items(): + consumption[phase] = grid_val or 0.0 + + return PowerFlowData( + grid=grid_power, + solar=solar_power, + consumption=consumption, + batteries=batteries, + ) + + +# ------------------------------- # +# Services overview (Dashboard) # +# ------------------------------- # + + +class Status(StrEnum): + OK = "ok" + WARNING = "warning" + ERROR = "error" + + +class ServiceMessage(BaseModel): + message: str + + +class ServiceStatus(BaseModel): + status: Status + messages: list[ServiceMessage] + + +class ServicesStatusResponse(BaseModel): + database: ServiceStatus | None + optimizer: ServiceStatus | None + + +@router.get("/services-status", response_model=ServicesStatusResponse) +async def services_status() -> ServicesStatusResponse: + try: + return ServicesStatusResponse( + database=ServiceStatus(status=Status.OK, messages=[]), + optimizer=ServiceStatus(status=Status.OK, messages=[]), + ) + except Exception as e: + logger.exception("Services status check failed") + raise HTTPException(status_code=500, detail=str(e)) from e + + +@router.get("/battery-ids", response_model=list[str]) +async def get_battery_ids(battery_systems: BatterySystemsDep) -> list[str]: + try: + return [s.id for s in battery_systems] + except Exception as e: + logger.exception("Failed to get battery ids") + raise HTTPException(status_code=500, detail=str(e)) from e # ------------------------ # @@ -258,173 +237,118 @@ async def health_check() -> HealthResponse: # ------------------------ # -class BatterySystemQueries(BaseModel): - energy_to_charger: str - energy_from_inverter: str - energy_to_battery: str - energy_from_battery: str - energy_loss_to_battery: str - energy_loss_from_battery: str +class EnergyQueryDef(BaseModel): + query: str + label: str + color: str + negate: bool = False + + +class EnergyViewConfig(BaseModel): + """Configuration for a single energy chart view.""" + + id: str + name: str + queries: list[EnergyQueryDef] class EnergyQueriesResponse(BaseModel): - grid_import_query: str - grid_export_query: str + """Response containing all energy chart view configurations.""" - battery_systems: dict[str, BatterySystemQueries] - solar_query: str + views: list[EnergyViewConfig] @router.get("/charts/energy-queries", response_model=EnergyQueriesResponse) async def get_energy_queries(battery_systems: BatterySystemsDep) -> EnergyQueriesResponse: try: - battery_system_queries: dict[str, BatterySystemQueries] = {} + views: list[EnergyViewConfig] = [] + + # Collect grid queries across all battery systems grid_import_parts: list[str] = [] grid_export_parts: list[str] = [] + consumption_parts: list[str] = [] for battery_system in battery_systems: device = battery_system.device_serial + bs_name = battery_system.name or battery_system.id queries = battery_system.get_energy_queries() - battery_system_queries[battery_system.id] = BatterySystemQueries( - energy_to_charger=queries.energy_to_charger, - energy_from_inverter=queries.energy_from_inverter, - energy_to_battery=queries.energy_to_battery, - energy_from_battery=queries.energy_from_battery, - energy_loss_to_battery=queries.energy_loss_to_battery, - energy_loss_from_battery=queries.energy_loss_from_battery, + # Battery system view: shows charge/discharge from battery's perspective + views.append( + EnergyViewConfig( + id=battery_system.id, + name=bs_name, + queries=[ + EnergyQueryDef( + query=queries.energy_to_charger, + label="Charge", + color="#3498db", + negate=True, + ), + EnergyQueryDef( + query=queries.energy_from_inverter, + label="Discharge", + color="#f39c12", + ), + EnergyQueryDef( + query=queries.energy_loss_to_battery, + label="Charge Losses", + color="#e74c3c", + negate=True, + ), + EnergyQueryDef( + query=queries.energy_loss_from_battery, + label="Discharge Losses", + color="#c0392b", + negate=True, + ), + ], + ) ) - # Collect grid queries per device (will sum across all battery systems) + # Collect grid queries per device grid_import_parts.append( - f'(increase(openess_energy_kwh{{from="grid", to="system", device="{device}"}}[$step]))' + f'increase(openess_energy_kwh{{from="grid", to="system", device="{device}"}}[$step])' ) grid_export_parts.append( - f'(increase(openess_energy_kwh{{from="system", to="grid", device="{device}"}}[$step]))' + f'increase(openess_energy_kwh{{from="system", to="grid", device="{device}"}}[$step])' ) - grid_import_query = " + ".join(grid_import_parts) - grid_export_query = " + ".join(grid_export_parts) + # Consumption = AC out from inverter + consumption_parts.append( + f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[$step])' + ) - return EnergyQueriesResponse( - grid_import_query=grid_import_query, - grid_export_query=grid_export_query, - battery_systems=battery_system_queries, - solar_query="", # TODO: add solar query when solar support is implemented - ) + # Grid view: shows grid import/export + grid_import_query = " + ".join(f"({q})" for q in grid_import_parts) if grid_import_parts else "" + grid_export_query = " + ".join(f"({q})" for q in grid_export_parts) if grid_export_parts else "" + + grid_queries: list[EnergyQueryDef] = [] + if grid_import_query: + grid_queries.append(EnergyQueryDef(query=grid_import_query, label="Import", color="#e74c3c", negate=True)) + if grid_export_query: + grid_queries.append(EnergyQueryDef(query=grid_export_query, label="Export", color="#2ecc71")) + # TODO: Add solar query when solar support is implemented + + views.append(EnergyViewConfig(id="grid", name="Grid", queries=grid_queries)) + + # Consumption view: shows energy consumed (AC out) + consumption_query = " + ".join(f"({q})" for q in consumption_parts) if consumption_parts else "" + consumption_queries: list[EnergyQueryDef] = [] + if consumption_query: + consumption_queries.append(EnergyQueryDef(query=consumption_query, label="Consumption", color="#9b59b6")) + if grid_import_query: + consumption_queries.append(EnergyQueryDef(query=grid_import_query, label="From Grid", color="#e74c3c")) + # TODO: Add solar contribution + + views.append(EnergyViewConfig(id="consumption", name="Consumption", queries=consumption_queries)) + + return EnergyQueriesResponse(views=views) except Exception as e: logger.exception("Failed to get energy queries") raise HTTPException(status_code=500, detail=str(e)) from e -# @router.get("/energy-graph", response_model=EnergyGraphResponse) -# async def get_energy_flow_endpoint( -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# battery_id: str | None = Query(default=None), -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# bucket_minutes: int = Query(default=60), -# ) -> EnergyGraphResponse: -# try: -# battery_system = None -# if battery_id: -# for bs in battery_systems: -# if bs.id == battery_id: -# battery_system = bs -# break -# elif len(battery_systems) == 1: -# battery_system = battery_systems[0] -# -# if battery_system is None: -# if battery_id: -# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") -# else: -# raise HTTPException(status_code=400, detail="Please provide a battery_id") -# -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(hours=24) -# if end is None: -# end = now -# -# return await _get_energy_graph_timeseries(timeseries, battery_system, start, end, bucket_minutes) -# except Exception as e: -# logger.exception("Failed to get energy flow") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# async def _get_energy_graph_timeseries( -# mql_client: "TimeseriesBackend", -# battery_system, -# start: datetime, -# end: datetime, -# bucket_minutes: int, -# ) -> EnergyGraphResponse: -# """Get energy graph data from timeseries backend.""" -# device = battery_system.id -# step = f"{bucket_minutes}m" -# -# # Query energy series using increase() to get per-bucket energy consumption -# queries = battery_system.config.queries -# grid_import_query = queries.energy_grid_import.replace("$device", device) -# grid_export_query = queries.energy_grid_export.replace("$device", device) -# to_mp_query = queries.energy_to_battery.replace("$device", device) -# from_mp_query = queries.energy_from_battery.replace("$device", device) -# -# # Use increase() to get energy delta per bucket -# grid_import_result = mql_client.query_range(f"increase({grid_import_query}[{step}])", start, end, step) -# grid_export_result = mql_client.query_range(f"increase({grid_export_query}[{step}])", start, end, step) -# to_mp_result = mql_client.query_range(f"increase({to_mp_query}[{step}])", start, end, step) -# from_mp_result = mql_client.query_range(f"increase({from_mp_query}[{step}])", start, end, step) -# -# # Convert to dict for easier lookup -# def result_to_dict(result: "QueryResult") -> dict[datetime, float]: -# if not result.series or not result.series[0].values: -# return {} -# return {ts: val for ts, val in result.series[0].values} -# -# grid_import_data = result_to_dict(grid_import_result) -# grid_export_data = result_to_dict(grid_export_result) -# to_mp_data = result_to_dict(to_mp_result) -# from_mp_data = result_to_dict(from_mp_result) -# -# # Collect all timestamps -# all_timestamps: set[datetime] = set() -# all_timestamps.update(grid_import_data.keys()) -# all_timestamps.update(grid_export_data.keys()) -# all_timestamps.update(to_mp_data.keys()) -# all_timestamps.update(from_mp_data.keys()) -# timestamps = sorted(all_timestamps) -# -# # Build response series -# grid_exports: dict[str, list[float | None]] = {"From MP": []} -# grid_imports: dict[str, list[float | None]] = {"Consumption": [], "To MP": []} -# battery_stats = BatteryEnergySeries() -# -# for ts in timestamps: -# from_mp = from_mp_data.get(ts) -# grid_exports["From MP"].append(round(from_mp, 3) if from_mp else None) -# unaccounted_export = grid_export_data.get(ts, 0) - (from_mp or 0) -# -# to_mp = to_mp_data.get(ts) -# grid_imports["To MP"].append(round(to_mp, 3) if to_mp else None) -# grid_import = grid_import_data.get(ts) -# if grid_import is not None: -# grid_import -= (to_mp or 0) - unaccounted_export -# grid_imports["Consumption"].append(round(grid_import, 3) if grid_import else None) -# -# battery_stats.energy_to_charger.append(round(to_mp, 3) if to_mp else None) -# battery_stats.energy_from_inverter.append(round(from_mp, 3) if from_mp else None) -# -# return EnergyGraphResponse( -# timestamps=timestamps, -# grid_export=grid_exports, -# grid_import=grid_imports, -# battery_systems={battery_system.config.name: battery_stats}, -# ) - - class PowerQueryDef(BaseModel): label: str query: str @@ -554,501 +478,344 @@ async def get_battery_queries( raise HTTPException(status_code=500, detail=str(e)) from e -# # ---------------# -# # Cycles page # -# # ---------------# -# -# -# class EfficiencyScatterPoint(BaseModel): -# time: datetime -# battery_power: float -# inverter_charger_power: float -# losses: float -# efficiency: float | None -# soc: int | None -# category: str -# -# -# @router.get("/efficiency-scatter", response_model=list[EfficiencyScatterPoint]) -# async def get_efficiency_scatter( -# db: Database, -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# battery_id: str | None = Query(default=None), -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# aggregate_minutes: int = Query(default=10), -# idle_threshold: int = Query(default=5), -# ) -> list[EfficiencyScatterPoint]: -# try: -# battery_system = None -# if battery_id: -# for bs in battery_systems: -# if bs.id == battery_id: -# battery_system = bs -# break -# elif len(battery_systems) == 1: -# battery_system = battery_systems[0] -# -# if battery_system is None: -# if battery_id: -# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") -# else: -# raise HTTPException(status_code=400, detail="Please provide a battery_id") -# -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(days=7) -# if end is None: -# end = now -# -# if timeseries is not None: -# return await _get_efficiency_scatter_timeseries( -# timeseries, battery_system, start, end, aggregate_minutes, idle_threshold -# ) -# -# return await _get_efficiency_scatter_legacy(db, battery_system, start, end, aggregate_minutes, idle_threshold) -# except Exception as e: -# logger.exception("Failed to get efficiency scatter data") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# async def _get_efficiency_scatter_timeseries( -# timeseries: "TimeseriesBackend", -# battery_system, -# start: datetime, -# end: datetime, -# aggregate_minutes: int, -# idle_threshold: int, -# ) -> list[EfficiencyScatterPoint]: -# """Get efficiency scatter data from timeseries backend.""" -# device = battery_system.id -# step = f"{aggregate_minutes}m" -# queries = battery_system.config.queries -# -# # Query AC in, AC out, and battery DC power -# ac_in_query = queries.power_ac_in.replace("$device", device) -# ac_out_query = queries.power_ac_out.replace("$device", device) -# dc_query = queries.power_battery.replace("$device", device) -# -# ac_in_result = timeseries.query_range(ac_in_query, start, end, step) -# ac_out_result = timeseries.query_range(ac_out_query, start, end, step) -# dc_result = timeseries.query_range(dc_query, start, end, step) -# -# # Convert to dicts -# def result_to_dict(result: "QueryResult") -> dict[datetime, float]: -# if not result.series or not result.series[0].values: -# return {} -# return {ts: val for ts, val in result.series[0].values} -# -# ac_in_data = result_to_dict(ac_in_result) -# ac_out_data = result_to_dict(ac_out_result) -# dc_data = result_to_dict(dc_result) -# -# # Merge data by timestamp -# all_timestamps = set(ac_in_data.keys()) & set(ac_out_data.keys()) & set(dc_data.keys()) -# -# points = [] -# for ts in sorted(all_timestamps): -# ac = ac_in_data[ts] - ac_out_data[ts] -# dc = dc_data[ts] -# -# if abs(dc) < idle_threshold: -# category = "idling" -# elif dc > 0: -# category = "charging" -# else: -# category = "discharging" -# -# losses = ac - dc -# efficiency = None -# if category == "charging" and ac > 0: -# efficiency = (dc / ac) * 100 -# elif category == "discharging" and dc < 0: -# efficiency = (ac / dc) * 100 -# -# points.append( -# EfficiencyScatterPoint( -# time=ts, -# battery_power=round(abs(dc), 1), -# inverter_charger_power=round(ac, 1), -# losses=round(losses, 1), -# efficiency=round(efficiency, 1) if efficiency is not None else None, -# soc=None, -# category=category, -# ) -# ) -# -# return points -# -# -# async def _get_efficiency_scatter_legacy( -# db: "DatabaseConnection", -# battery_system, -# start: datetime, -# end: datetime, -# aggregate_minutes: int, -# idle_threshold: int, -# ) -> list[EfficiencyScatterPoint]: -# """Get efficiency scatter data from legacy database.""" -# metrics = battery_system.config.metrics -# bucket_seconds = aggregate_minutes * 60 -# -# # Use configured metrics paths -# ac_in_path = metrics.power_to_system -# if isinstance(ac_in_path, list): -# ac_in_path = ac_in_path[0] -# -# # AC out is typically the same vebus but ac_out instead of ac_in -# ac_out_path = ac_in_path.replace("ac_in", "ac_out") if ac_in_path else None -# -# dc_path = metrics.power_to_battery -# if isinstance(dc_path, list): -# dc_path = dc_path[0] -# -# ac_in = db.get_power(ac_in_path, start, end, bucket_seconds=bucket_seconds) if ac_in_path else [] -# ac_out = db.get_power(ac_out_path, start, end, bucket_seconds=bucket_seconds) if ac_out_path else [] -# dc = db.get_power(dc_path, start, end, bucket_seconds=bucket_seconds) if dc_path else [] -# -# data: dict[datetime, list[float | None]] = { -# ts: [v_in - v_out, None] for (ts, v_in), (_, v_out) in zip(ac_in, ac_out, strict=False) -# } -# for ts, v in dc: -# if ts in data: -# data[ts][1] = v -# -# points = [] -# for ts, (ac, dc_val) in data.items(): -# if ac is None or dc_val is None: -# continue -# -# if abs(dc_val) < idle_threshold: -# category = "idling" -# elif dc_val > 0: -# category = "charging" -# else: -# category = "discharging" -# -# losses = ac - dc_val -# efficiency = None -# if category == "charging" and ac > 0: -# efficiency = (dc_val / ac) * 100 -# elif category == "discharging" and dc_val < 0: -# efficiency = (ac / dc_val) * 100 -# -# points.append( -# EfficiencyScatterPoint( -# time=ts, -# battery_power=round(abs(dc_val), 1), -# inverter_charger_power=round(ac, 1), -# losses=round(losses, 1), -# efficiency=round(efficiency, 1) if efficiency is not None else None, -# soc=None, -# category=category, -# ) -# ) -# -# return points -# -# -# class BatteryCycle(BaseModel): -# start_time: datetime -# end_time: datetime -# duration_hours: float -# min_soc: float -# ac_energy_in: float | None -# ac_energy_out: float | None -# dc_energy_in: float -# dc_energy_out: float -# system_efficiency: float | None -# battery_efficiency: float | None -# charger_efficiency: float | None -# inverter_efficiency: float | None -# profit: float | None -# scheduled_profit: float | None -# -# -# @router.get("/cycles", response_model=list[BatteryCycle]) -# async def get_battery_cycles( -# db: Database, -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# price_config: PriceConfigDep, -# battery_id: str | None = Query(default=None), -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# min_soc_swing: int = Query(default=10), -# ) -> list[BatteryCycle]: -# try: -# battery_system = None -# if battery_id: -# for bs in battery_systems: -# if bs.id == battery_id: -# battery_system = bs -# break -# elif len(battery_systems) == 1: -# battery_system = battery_systems[0] -# -# if battery_system is None: -# if battery_id: -# raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") -# else: -# raise HTTPException(status_code=400, detail="Please provide a battery_id") -# -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(days=30) -# if end is None: -# end = now -# -# # Get SOC data from timeseries or legacy database -# if timeseries is not None: -# device = battery_system.id -# soc_query = battery_system.config.queries.soc.replace("$device", device) -# soc_result = timeseries.query_range(soc_query, start, end, step="1m") -# battery_soc = [(ts, val) for ts, val in soc_result.series[0].values] if soc_result.series else [] -# else: -# battery_soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) -# -# raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) -# -# cycles = [] -# for cycle_start, cycle_end, min_soc in raw_cycles: -# duration = (cycle_end - cycle_start).total_seconds() / 3600.0 -# -# # TODO: this is cursed AF and should be fixed -# dc_energy_in = 0.0 -# dc_energy_out = 0.0 -# for _, p in db.get_power(battery_system.config.metrics.power_to_battery[0], cycle_start, cycle_end): -# # for _, p in db.get_power("vebus_228_battery", cycle_start, cycle_end): -# if p > 0: -# dc_energy_in += p -# else: -# dc_energy_out += -p -# dc_energy_in /= 60000 -# dc_energy_out /= 60000 -# -# ac_in_import = db.get_energy( -# battery_system.config.metrics.energy_to_system, cycle_start, cycle_end, normalize=True -# ) -# # ac_out_import = db.get_energy("vebus_228_ac_out_import", cycle_start, cycle_end, normalize=True) -# ac_in_export = db.get_energy( -# battery_system.config.metrics.energy_from_system, cycle_start, cycle_end, normalize=True -# ) -# # ac_out_export = db.get_energy("vebus_228_ac_out_export", cycle_start, cycle_end, normalize=True) -# -# ac_energy_in = 0.0 -# if ac_in_import: -# ac_energy_in += ac_in_import[-1][1] -# # if ac_out_import: -# # ac_energy_in += ac_out_import[-1][1] -# ac_energy_out = 0.0 -# if ac_in_export: -# ac_energy_out += ac_in_export[-1][1] -# # if ac_out_export: -# # ac_energy_out += ac_out_export[-1][1] -# -# profit = 0.0 -# scheduled_profit = 0.0 -# e_in = { -# ts: v -# for ts, v in db.get_energy_aggregated( -# battery_system.config.metrics.energy_to_system, 3600, cycle_start, cycle_end -# ) -# } -# e_out = { -# ts: v -# for ts, v in db.get_energy_aggregated( -# battery_system.config.metrics.energy_from_system, 3600, cycle_start, cycle_end -# ) -# } -# scheduled = {ts: v for ts, _, v, _ in db.get_schedule(battery_system.config.id, cycle_start)} -# for ts, v in db.get_prices(price_config.area, cycle_start, cycle_end, aggregate_minutes=60): -# profit -= price_config.buy_price(v) * e_in.get(ts, 0) -# profit += price_config.sell_price(v) * e_out.get(ts, 0) -# scheduled_power = scheduled.get(ts, 0) -# if scheduled_power > 0: -# scheduled_profit -= price_config.buy_price(v) * scheduled_power / 1000 -# if scheduled_power < 0: -# scheduled_profit += price_config.sell_price(v) * -scheduled_power / 1000 -# -# cycles.append( -# BatteryCycle( -# start_time=cycle_start, -# end_time=cycle_end, -# duration_hours=round(duration, 2), -# min_soc=round(min_soc, 1), -# ac_energy_in=round(ac_energy_in, 2) if ac_energy_in else None, -# ac_energy_out=round(ac_energy_out, 2) if ac_energy_out else None, -# dc_energy_in=round(dc_energy_in, 2), -# dc_energy_out=round(dc_energy_out, 2), -# system_efficiency=round(ac_energy_out / ac_energy_in * 100, 1) if ac_energy_in else None, -# battery_efficiency=round(dc_energy_out / dc_energy_in * 100, 1) if dc_energy_in else None, -# charger_efficiency=round(dc_energy_in / ac_energy_in * 100, 1) if ac_energy_in else None, -# inverter_efficiency=round(ac_energy_out / dc_energy_out * 100, 1) if dc_energy_out else None, -# profit=round(profit, 2), -# scheduled_profit=round(scheduled_profit, 2), -# ) -# ) -# -# return cycles -# except Exception as e: -# logger.exception("Failed to get battery cycles") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# # -------------------------# -# # Generalized endpoints # -# # -------------------------# -# - -# def _calculate_step(start: datetime, end: datetime, aggregate_minutes: int) -> str: -# """Calculate query step from aggregate_minutes or time range.""" -# if aggregate_minutes > 1: -# return f"{aggregate_minutes}m" -# # Auto-calculate based on range -# duration = (end - start).total_seconds() -# if duration <= 3600: # 1 hour -# return "1m" -# if duration <= 6 * 3600: # 6 hours -# return "5m" -# if duration <= 24 * 3600: # 24 hours -# return "15m" -# return "1h" -# -# # TODO: add parameter to select subset of series -# @router.get("/power", response_model=PowerResponse) -# async def get_power( -# db: Database, -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# aggregate_minutes: int = Query(default=1), -# ) -> PowerResponse: -# try: -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(hours=24) -# if end is None: -# end = now -# -# if timeseries is not None and battery_systems: -# # Query all power metrics from timeseries -# step = _calculate_step(start, end, aggregate_minutes) -# series: dict[str, TimeSeries] = {} -# -# for battery_system in battery_systems: -# device = battery_system.id -# queries = battery_system.config.queries -# prefix = battery_system.config.name or device -# -# # Query each power metric -# power_queries = { -# "Grid": queries.power_grid_total, -# "Grid L1": f'openess_power_watts{{from="grid", phase="L1", device="{device}"}}', -# "Grid L2": f'openess_power_watts{{from="grid", phase="L2", device="{device}"}}', -# "Grid L3": f'openess_power_watts{{from="grid", phase="L3", device="{device}"}}', -# "PV": queries.power_pv, -# "Battery": queries.power_battery, -# "AC In": queries.power_ac_in, -# "AC Out": queries.power_ac_out, -# } -# -# for name, query in power_queries.items(): -# resolved_query = query.replace("$device", device) -# result = timeseries.query_range(resolved_query, start, end, step) -# label = f"{prefix}/{name}" if len(battery_systems) > 1 else name -# series[label] = query_result_to_timeseries(result) -# -# return PowerResponse(series=series) -# -# # Legacy database fallback -# legacy_series = db.get_all_power(start, end, aggregate_minutes * 60) -# return PowerResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) -# except Exception as e: -# logger.exception("Failed to get debug power flows") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# # TODO: add parameter to select subset of series -# # TODO: add normalize parameter -# @router.get("/energy", response_model=EnergyResponse) -# async def get_energy( -# db: Database, -# timeseries: TimeseriesDep, -# battery_systems: BatterySystemsDep, -# start: datetime | None = Query(default=None), -# end: datetime | None = Query(default=None), -# bucket_minutes: int = Query(default=60), -# ) -> EnergyResponse: -# try: -# now = datetime.now(UTC) -# if start is None: -# start = now - timedelta(hours=24) -# if end is None: -# end = now -# -# if timeseries is not None and battery_systems: -# # Query all energy metrics from timeseries -# step = f"{bucket_minutes}m" -# series: dict[str, TimeSeries] = {} -# -# for battery_system in battery_systems: -# device = battery_system.id -# queries = battery_system.config.queries -# prefix = battery_system.config.name or device -# -# energy_queries = { -# "Grid Import": queries.energy_grid_import, -# "Grid Export": queries.energy_grid_export, -# "To Battery": queries.energy_to_battery, -# "From Battery": queries.energy_from_battery, -# } -# -# for name, query in energy_queries.items(): -# resolved_query = query.replace("$device", device) -# # Use increase() to get energy delta per bucket -# result = timeseries.query_range(f"increase({resolved_query}[{step}])", start, end, step) -# label = f"{prefix}/{name}" if len(battery_systems) > 1 else name -# series[label] = query_result_to_timeseries(result, rounding=3) -# -# return EnergyResponse(series=series) -# -# # Legacy database fallback -# legacy_series = db.get_all_energy(start, end, normalize=True) -# -# # Get integrated power flows -# for label in db.get_power_labels(start, end): -# legacy_series[f"{label} [integrated]"] = db.integrate_power(label, start, end) -# -# return EnergyResponse(series={k: data_to_timeseries(v) for k, v in legacy_series.items()}) -# except Exception as e: -# logger.exception("Failed to get debug energy flows") -# raise HTTPException(status_code=500, detail=str(e)) from e -# -# -# # -------------------------- # -# # Timeseries query helpers # -# # -------------------------- # -# -# -# @router.get("/queries/{battery_id}") -# async def get_queries( -# battery_id: str, -# battery_systems: BatterySystemsDep, -# ) -> dict[str, str]: -# """Return resolved MetricsQL queries for a battery system. -# -# The queries are templated in BatterySystemConfig.queries and resolved -# with the device serial number. Frontend can use these to query the -# timeseries backend directly via /api/v1/query_range. -# """ -# battery = next((b for b in battery_systems if b.id == battery_id), None) -# if not battery: -# raise HTTPException(404, f"Battery system {battery_id} not found") -# -# device = battery.id -# queries = battery.config.queries -# -# return {field: getattr(queries, field).replace("$device", device) for field in queries.model_fields} +# ---------------# +# Cycles page # +# ---------------# + + +class EfficiencyScatterPoint(BaseModel): + time: datetime + battery_power: float + inverter_charger_power: float + losses: float + efficiency: float | None + soc: int | None + category: str + + +@router.get("/efficiency-scatter", response_model=list[EfficiencyScatterPoint]) +async def get_efficiency_scatter( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, + battery_id: str | None = Query(default=None), + start: datetime | None = Query(default=None), + end: datetime | None = Query(default=None), + aggregate_minutes: int = Query(default=10), + idle_threshold: int = Query(default=5), + limit: int = Query(default=2000), +) -> list[EfficiencyScatterPoint]: + try: + if mql_client is None: + raise HTTPException(503, "Timeseries backend not configured") + + battery_system = None + if battery_id: + for bs in battery_systems: + if bs.id == battery_id: + battery_system = bs + break + elif len(battery_systems) == 1: + battery_system = battery_systems[0] + + if battery_system is None: + if battery_id: + raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") + else: + raise HTTPException(status_code=400, detail="Please provide a battery_id") + + now = datetime.now(UTC) + if start is None: + start = now - timedelta(days=7) + if end is None: + end = now + + return _get_efficiency_scatter(mql_client, battery_system, start, end, aggregate_minutes, idle_threshold, limit) + except HTTPException: + raise + except Exception as e: + logger.exception("Failed to get efficiency scatter data") + raise HTTPException(status_code=500, detail=str(e)) from e + + +def _get_efficiency_scatter( + mql_client: TimeseriesBackend, + battery_system, + start: datetime, + end: datetime, + aggregate_minutes: int, + idle_threshold: int, + limit: int, +) -> list[EfficiencyScatterPoint]: + """Get efficiency scatter data from timeseries backend.""" + device = battery_system.device_serial + step = f"{aggregate_minutes}m" + + # Query AC in, AC out, and battery DC power + ac_in_query = f'sum(avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[{step}]))' + ac_out_query = f'sum(avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[{step}]))' + dc_query = f'avg_over_time(openess_power_watts{{from="system", to="battery", device="{device}"}}[{step}])' + + ac_in_result = mql_client.query_range(ac_in_query, start, end, step) + ac_out_result = mql_client.query_range(ac_out_query, start, end, step) + dc_result = mql_client.query_range(dc_query, start, end, step) + + # Convert to dicts + def result_to_dict(result) -> dict[datetime, float]: + if not result.series or not result.series[0].values: + return {} + return {ts: val for ts, val in result.series[0].values} + + ac_in_data = result_to_dict(ac_in_result) + ac_out_data = result_to_dict(ac_out_result) + dc_data = result_to_dict(dc_result) + + # Merge data by timestamp + all_timestamps = set(ac_in_data.keys()) & set(dc_data.keys()) + if ac_out_data: + all_timestamps &= set(ac_out_data.keys()) + + points = [] + for ts in sorted(all_timestamps): + ac_in = ac_in_data.get(ts, 0) + ac_out = ac_out_data.get(ts, 0) + ac = ac_in - ac_out # Net AC power (positive = charging) + dc = dc_data[ts] + + if abs(dc) < idle_threshold: + category = "idling" + elif dc > 0: + category = "charging" + else: + category = "discharging" + + losses = abs(ac) - abs(dc) + efficiency = None + if category == "charging" and ac > 0: + efficiency = (dc / ac) * 100 + elif category == "discharging" and dc < 0 and ac_out > 0: + efficiency = (ac_out / abs(dc)) * 100 + + points.append( + EfficiencyScatterPoint( + time=ts, + battery_power=round(abs(dc), 1), + inverter_charger_power=round(ac, 1), + losses=round(losses, 1), + efficiency=round(efficiency, 1) if efficiency is not None else None, + soc=None, + category=category, + ) + ) + + if len(points) >= limit: + break + + return points + + +class BatteryCycle(BaseModel): + start_time: datetime + end_time: datetime + duration_hours: float + min_soc: float + ac_energy_in: float | None + ac_energy_out: float | None + dc_energy_in: float | None + dc_energy_out: float | None + system_efficiency: float | None + battery_efficiency: float | None + charger_efficiency: float | None + inverter_efficiency: float | None + profit: float | None + scheduled_profit: float | None + + +@router.get("/cycles", response_model=list[BatteryCycle]) +async def get_battery_cycles( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, + price_config: PriceConfigDep, + battery_id: str | None = Query(default=None), + start: datetime | None = Query(default=None), + end: datetime | None = Query(default=None), + min_soc_swing: int = Query(default=10), +) -> list[BatteryCycle]: + try: + if mql_client is None: + raise HTTPException(503, "Timeseries backend not configured") + + battery_system = None + if battery_id: + for bs in battery_systems: + if bs.id == battery_id: + battery_system = bs + break + elif len(battery_systems) == 1: + battery_system = battery_systems[0] + + if battery_system is None: + if battery_id: + raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") + else: + raise HTTPException(status_code=400, detail="Please provide a battery_id") + + now = datetime.now(UTC) + if start is None: + start = now - timedelta(days=30) + if end is None: + end = now + + return _get_battery_cycles(mql_client, battery_system, price_config, start, end, min_soc_swing) + except HTTPException: + raise + except Exception as e: + logger.exception("Failed to get battery cycles") + raise HTTPException(status_code=500, detail=str(e)) from e + + +def _get_battery_cycles( + mql_client: TimeseriesBackend, + battery_system, + price_config, + start: datetime, + end: datetime, + min_soc_swing: int, +) -> list[BatteryCycle]: + """Get battery cycles from timeseries backend.""" + from .util import find_full_battery_cycles + + device = battery_system.device_serial + + # Get SOC data at 1-minute resolution for cycle detection + soc_query = f'openess_soc_ratio{{device="{device}", node="battery"}} * 100' + soc_result = mql_client.query_range(soc_query, start, end, step="1m") + + if not soc_result.series or not soc_result.series[0].values: + return [] + + battery_soc = [(ts, val) for ts, val in soc_result.series[0].values] + + # Find cycles using the existing algorithm + raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) + + cycles = [] + for cycle_start, cycle_end, min_soc in raw_cycles: + duration = (cycle_end - cycle_start).total_seconds() / 3600.0 + + # Query energy for this cycle + # AC energy in (charger input) + ac_in_query = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[1h])' + ac_in_result = mql_client.query_range(ac_in_query, cycle_start, cycle_end, step="1h") + ac_energy_in = _sum_series_values(ac_in_result) + + # AC energy out (inverter output) + ac_out_query = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[1h])' + ac_out_result = mql_client.query_range(ac_out_query, cycle_start, cycle_end, step="1h") + ac_energy_out = _sum_series_values(ac_out_result) + + # DC energy in (battery charge) + dc_in_query = f'increase(openess_energy_kwh{{from="system", to="battery", device="{device}"}}[1h])' + dc_in_result = mql_client.query_range(dc_in_query, cycle_start, cycle_end, step="1h") + dc_energy_in = _sum_series_values(dc_in_result) + + # DC energy out (battery discharge) + dc_out_query = f'increase(openess_energy_kwh{{from="battery", to="system", device="{device}"}}[1h])' + dc_out_result = mql_client.query_range(dc_out_query, cycle_start, cycle_end, step="1h") + dc_energy_out = _sum_series_values(dc_out_result) + + # Calculate efficiencies + system_eff = None + if ac_energy_in and ac_energy_in > 0: + system_eff = round((ac_energy_out or 0) / ac_energy_in * 100, 1) + + battery_eff = None + if dc_energy_in and dc_energy_in > 0: + battery_eff = round((dc_energy_out or 0) / dc_energy_in * 100, 1) + + charger_eff = None + if ac_energy_in and ac_energy_in > 0: + charger_eff = round((dc_energy_in or 0) / ac_energy_in * 100, 1) + + inverter_eff = None + if dc_energy_out and dc_energy_out > 0: + inverter_eff = round((ac_energy_out or 0) / dc_energy_out * 100, 1) + + # Calculate profit using hourly prices + profit = _calculate_cycle_profit(mql_client, price_config, device, cycle_start, cycle_end) + + cycles.append( + BatteryCycle( + start_time=cycle_start, + end_time=cycle_end, + duration_hours=round(duration, 2), + min_soc=round(min_soc, 1), + ac_energy_in=round(ac_energy_in, 3) if ac_energy_in else None, + ac_energy_out=round(ac_energy_out, 3) if ac_energy_out else None, + dc_energy_in=round(dc_energy_in, 3) if dc_energy_in else None, + dc_energy_out=round(dc_energy_out, 3) if dc_energy_out else None, + system_efficiency=system_eff, + battery_efficiency=battery_eff, + charger_efficiency=charger_eff, + inverter_efficiency=inverter_eff, + profit=profit, + scheduled_profit=None, # TODO: implement scheduled profit + ) + ) + + return cycles + + +def _sum_series_values(result) -> float | None: + """Sum all values in a range query result.""" + if not result.series or not result.series[0].values: + return None + total = sum(val for _, val in result.series[0].values) + return total if total > 0 else None + + +def _calculate_cycle_profit( + mql_client: TimeseriesBackend, + price_config, + device: str, + cycle_start: datetime, + cycle_end: datetime, +) -> float | None: + """Calculate profit for a battery cycle based on hourly prices.""" + area = price_config.area + + # Get hourly prices + price_query = f'avg_over_time(openess_prices{{area="{area}", price="market"}}[1h])' + price_result = mql_client.query_range(price_query, cycle_start, cycle_end, step="1h") + + if not price_result.series or not price_result.series[0].values: + return None + + # Get hourly energy in/out + ac_in_query = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[1h])' + ac_out_query = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[1h])' + + ac_in_result = mql_client.query_range(ac_in_query, cycle_start, cycle_end, step="1h") + ac_out_result = mql_client.query_range(ac_out_query, cycle_start, cycle_end, step="1h") + + # Build timestamp -> value dicts + prices = {ts: val for ts, val in price_result.series[0].values} + energy_in = {} + energy_out = {} + + if ac_in_result.series and ac_in_result.series[0].values: + energy_in = {ts: val for ts, val in ac_in_result.series[0].values} + if ac_out_result.series and ac_out_result.series[0].values: + energy_out = {ts: val for ts, val in ac_out_result.series[0].values} + + profit = 0.0 + for ts, market_price in prices.items(): + e_in = energy_in.get(ts, 0) + e_out = energy_out.get(ts, 0) + + buy_price = price_config.buy_price(market_price) + sell_price = price_config.sell_price(market_price) + + profit -= buy_price * e_in + profit += sell_price * e_out + + return round(profit, 2) if profit != 0 else None diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index 151f736..2e75a23 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -19,27 +19,29 @@ */ /** - * @typedef {Object} BatterySystemQueries - * @property {string} [energy_to_charger] - * @property {string} [energy_from_inverter] - * @property {string} [energy_to_battery] - * @property {string} [energy_from_battery] - * @property {string} [energy_loss_to_battery] - * @property {string} [energy_loss_from_battery] + * @typedef {Object} EnergyQueryDef + * @property {string} query + * @property {string} label + * @property {string} color + * @property {boolean} [negate] */ /** - * @typedef {Object} ChartsPowerResponse - * @property {PowerQueryDef[]} [queries] - * @property {string[]} [phases] + * @typedef {Object} EnergyViewConfig + * @property {string} id + * @property {string} name + * @property {EnergyQueryDef[]} queries */ /** * @typedef {Object} EnergyQueriesResponse - * @property {string} [grid_import_query] - * @property {string} [grid_export_query] - * @property {Object.} [battery_systems] - * @property {string} [solar_query] + * @property {EnergyViewConfig[]} views + */ + +/** + * @typedef {Object} ChartsPowerResponse + * @property {PowerQueryDef[]} [queries] + * @property {string[]} [phases] */ /** @@ -150,6 +152,79 @@ throw new Error('HTTP ' + response.status); } return response.json(); + }, + + /** + * @returns {Promise} + */ + systemLayout: async function() { + var response = await fetch('/api/system-layout'); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); + }, + + /** + * @returns {Promise} + */ + powerFlow: async function() { + var response = await fetch('/api/power-flow'); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); + }, + + /** + * @returns {Promise} + */ + servicesStatus: async function() { + var response = await fetch('/api/services-status'); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); + }, + + /** + * @param {Object} params + * @param {number} [params.aggregate_minutes] + * @param {number} [params.limit] + * @returns {Promise} + */ + efficiencyScatter: async function(params) { + params = params || {}; + var searchParams = new URLSearchParams(); + if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); + if (params.limit !== undefined) searchParams.set('limit', String(params.limit)); + var query = searchParams.toString() ? '?' + searchParams.toString() : ''; + var response = await fetch('/api/efficiency-scatter' + query); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); + }, + + /** + * @param {Object} params + * @param {string} [params.start] + * @param {string} [params.end] + * @param {number} [params.min_soc_swing] + * @returns {Promise} + */ + cycles: async function(params) { + params = params || {}; + var searchParams = new URLSearchParams(); + if (params.start !== undefined) searchParams.set('start', params.start); + if (params.end !== undefined) searchParams.set('end', params.end); + if (params.min_soc_swing !== undefined) searchParams.set('min_soc_swing', String(params.min_soc_swing)); + var query = searchParams.toString() ? '?' + searchParams.toString() : ''; + var response = await fetch('/api/cycles' + query); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); } }; diff --git a/open_ess/frontend/static/cycles.js b/open_ess/frontend/static/cycles.js index 86b3d77..947dedb 100644 --- a/open_ess/frontend/static/cycles.js +++ b/open_ess/frontend/static/cycles.js @@ -3,6 +3,8 @@ 'use strict'; var cyclesTable = null; + var currentScatterMode = 'losses'; // 'losses' or 'efficiency' + var cachedScatterData = null; function getEfficiencyClass(efficiency) { if (efficiency == null) return ''; @@ -34,102 +36,128 @@ limit: parseInt(limit), }); - if (data.length === 0) { - document.getElementById(elementId).innerHTML = '
No data available
'; - return; - } + cachedScatterData = data; + renderScatterChart(data); - var isDark = Utils.isDarkTheme(); - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var divisor = useKw ? 1000 : 1; - var powerUnit = useKw ? 'kW' : 'W'; - - var categories = { - charging: { data: [], color: 'rgba(52, 152, 219, 0.5)', name: 'Charging' }, - discharging: { data: [], color: 'rgba(231, 76, 60, 0.5)', name: 'Discharging' }, - idling: { data: [], color: 'rgba(149, 165, 166, 0.5)', name: 'Idling' }, - balancing: { data: [], color: 'rgba(155, 89, 182, 0.5)', name: 'Balancing' }, - }; + } catch (error) { + console.error('Error loading scatter chart:', error); + document.getElementById(elementId).innerHTML = '
Failed to load scatter chart
'; + } + } + + function renderScatterChart(data) { + var elementId = 'scatter-chart'; - for (var i = 0; i < data.length; i++) { - var d = data[i]; - if (d.category && categories[d.category]) { + if (!data || data.length === 0) { + document.getElementById(elementId).innerHTML = '
No data available
'; + return; + } + + var isDark = Utils.isDarkTheme(); + var settings = Settings.load(); + var useKw = settings.powerUnit === 'kw'; + var divisor = useKw ? 1000 : 1; + var powerUnit = useKw ? 'kW' : 'W'; + var isEfficiencyMode = currentScatterMode === 'efficiency'; + + var categories = { + charging: { data: [], color: 'rgba(52, 152, 219, 0.5)', name: 'Charging' }, + discharging: { data: [], color: 'rgba(231, 76, 60, 0.5)', name: 'Discharging' }, + idling: { data: [], color: 'rgba(149, 165, 166, 0.5)', name: 'Idling' }, + balancing: { data: [], color: 'rgba(155, 89, 182, 0.5)', name: 'Balancing' }, + }; + + for (var i = 0; i < data.length; i++) { + var d = data[i]; + if (d.category && categories[d.category]) { + // For efficiency mode, only include points with valid efficiency + if (!isEfficiencyMode || d.efficiency != null) { categories[d.category].data.push(d); } } + } - function fmtPower(w) { - return useKw ? (w / 1000).toFixed(2) : Math.round(w).toString(); - } + function fmtPower(w) { + return useKw ? (w / 1000).toFixed(2) : Math.round(w).toString(); + } - function buildHoverText(d) { - var eff = d.efficiency != null ? d.efficiency.toFixed(1) + '%' : 'N/A'; - var soc = d.soc != null ? d.soc + '%' : 'N/A'; - var time = formatScatterTime(d.time || ''); - - switch (d.category) { - case 'charging': - return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Charger: ' + fmtPower(d.inverter_charger_power || 0) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; - case 'discharging': - return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Inverter: ' + fmtPower(Math.abs(d.inverter_charger_power || 0)) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; - case 'balancing': - return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Balancing power'; - case 'idling': - return 'Time: ' + time + '
SOC: ' + soc + '
Idle consumption: ' + fmtPower(d.losses || 0) + ' ' + powerUnit; - default: - return 'Time: ' + time; - } + function buildHoverText(d) { + var eff = d.efficiency != null ? d.efficiency.toFixed(1) + '%' : 'N/A'; + var soc = d.soc != null ? d.soc + '%' : 'N/A'; + var time = formatScatterTime(d.time || ''); + + switch (d.category) { + case 'charging': + return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Charger: ' + fmtPower(d.inverter_charger_power || 0) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; + case 'discharging': + return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Inverter: ' + fmtPower(Math.abs(d.inverter_charger_power || 0)) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; + case 'balancing': + return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Balancing power'; + case 'idling': + return 'Time: ' + time + '
SOC: ' + soc + '
Idle consumption: ' + fmtPower(d.losses || 0) + ' ' + powerUnit; + default: + return 'Time: ' + time; } + } - var traces = Object.keys(categories).map(function(key) { - var cat = categories[key]; - return { - x: cat.data.map(function(d) { return (d.battery_power || 0) / divisor; }), - y: cat.data.map(function(d) { return (d.losses || 0) / divisor; }), - type: 'scatter', - mode: 'markers', - name: cat.name, - marker: { color: cat.color, size: 8 }, - text: cat.data.map(buildHoverText), - hoverinfo: 'text', - }; - }); - - var layout = { - margin: { t: 30, r: 30, b: 60, l: 60 }, - paper_bgcolor: 'transparent', - plot_bgcolor: 'transparent', - font: { - family: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', - color: isDark ? '#e4e4e4' : '#333333', - }, - xaxis: { - title: 'Battery Power (' + powerUnit + ')', - gridcolor: isDark ? '#2a2a4a' : '#eeeeee', - linecolor: isDark ? '#3a3a5a' : '#dddddd', - rangemode: 'tozero', - }, - yaxis: { - title: 'Losses (' + powerUnit + ')', - gridcolor: isDark ? '#2a2a4a' : '#eeeeee', - linecolor: isDark ? '#3a3a5a' : '#dddddd', - rangemode: 'tozero', - }, - legend: { - orientation: 'h', - y: -0.15, - font: { color: isDark ? '#e4e4e4' : '#333333' }, - }, - hovermode: 'closest', + var traces = Object.keys(categories).map(function(key) { + var cat = categories[key]; + return { + x: cat.data.map(function(d) { return (d.battery_power || 0) / divisor; }), + y: cat.data.map(function(d) { + if (isEfficiencyMode) { + return d.efficiency || 0; + } + return (d.losses || 0) / divisor; + }), + type: 'scatter', + mode: 'markers', + name: cat.name, + marker: { color: cat.color, size: 8 }, + text: cat.data.map(buildHoverText), + hoverinfo: 'text', }; + }); - document.getElementById(elementId).innerHTML = ''; - Plotly.newPlot(elementId, traces, layout, { responsive: true, displayModeBar: false }); + var yAxisTitle = isEfficiencyMode ? 'Efficiency (%)' : 'Losses (' + powerUnit + ')'; + var yAxisRange = isEfficiencyMode ? [50, 100] : undefined; - } catch (error) { - console.error('Error loading scatter chart:', error); - document.getElementById(elementId).innerHTML = '
Failed to load scatter chart
'; + var layout = { + margin: { t: 30, r: 30, b: 60, l: 60 }, + paper_bgcolor: 'transparent', + plot_bgcolor: 'transparent', + font: { + family: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', + color: isDark ? '#e4e4e4' : '#333333', + }, + xaxis: { + title: 'Battery Power (' + powerUnit + ')', + gridcolor: isDark ? '#2a2a4a' : '#eeeeee', + linecolor: isDark ? '#3a3a5a' : '#dddddd', + rangemode: 'tozero', + }, + yaxis: { + title: yAxisTitle, + gridcolor: isDark ? '#2a2a4a' : '#eeeeee', + linecolor: isDark ? '#3a3a5a' : '#dddddd', + rangemode: isEfficiencyMode ? undefined : 'tozero', + range: yAxisRange, + }, + legend: { + orientation: 'h', + y: -0.15, + font: { color: isDark ? '#e4e4e4' : '#333333' }, + }, + hovermode: 'closest', + }; + + document.getElementById(elementId).innerHTML = ''; + Plotly.newPlot(elementId, traces, layout, { responsive: true, displayModeBar: false }); + } + + function reloadScatterFromCache() { + if (cachedScatterData) { + renderScatterChart(cachedScatterData); } } @@ -242,6 +270,24 @@ document.getElementById('days-select').value = Settings.loadPagePref('cycles', 'days', '30'); document.getElementById('swing-select').value = Settings.loadPagePref('cycles', 'swing', '10'); + // Initialize scatter mode from saved preference + currentScatterMode = Settings.loadPagePref('cycles', 'scatterMode', 'losses'); + var scatterModeButtons = document.querySelectorAll('#scatter-mode-buttons .btn-toggle'); + scatterModeButtons.forEach(function(btn) { + btn.classList.toggle('active', btn.dataset.value === currentScatterMode); + }); + + // Scatter mode toggle handlers + scatterModeButtons.forEach(function(btn) { + btn.addEventListener('click', function() { + document.querySelectorAll('#scatter-mode-buttons .btn-toggle').forEach(function(b) { b.classList.remove('active'); }); + btn.classList.add('active'); + currentScatterMode = btn.dataset.value || 'losses'; + Settings.savePagePref('cycles', 'scatterMode', currentScatterMode); + reloadScatterFromCache(); + }); + }); + document.getElementById('scatter-aggregate-select').addEventListener('change', function(e) { Settings.savePagePref('cycles', 'aggregate', e.target.value); loadScatterChart(); diff --git a/open_ess/frontend/static/dashboard.js b/open_ess/frontend/static/dashboard.js index 0f4a158..3eeaab1 100644 --- a/open_ess/frontend/static/dashboard.js +++ b/open_ess/frontend/static/dashboard.js @@ -71,6 +71,7 @@ '' + '
' + battery.name + '
' + '
' + + '
SOC: --%
' + '
Charger:
' + '
Inverter:
' + '
Battery:
' + @@ -180,11 +181,15 @@ for (var k = 0; k < layout.battery_systems.length; k++) { var battery = layout.battery_systems[k]; - var battData = data.batteries[battery.id]; + var battData = data.batteries[battery.id] || {}; var chargerPwr = battData.charger || 0; var inverterPwr = battData.inverter || 0; var batteryPwr = battData.battery || 0; var lossesPwr = battData.losses || 0; + var soc = battData.soc; + + var socEl = document.getElementById(battery.id + '-soc'); + if (socEl) socEl.textContent = 'SOC: ' + (soc != null ? Math.round(soc) + '%' : '--%'); var chargerEl = document.getElementById(battery.id + '-charger-power'); if (chargerEl) chargerEl.textContent = 'Charger: ' + formatPower(chargerPwr); diff --git a/open_ess/frontend/static/debug.js b/open_ess/frontend/static/debug.js index 3c96896..34fbce3 100644 --- a/open_ess/frontend/static/debug.js +++ b/open_ess/frontend/static/debug.js @@ -1,4 +1,4 @@ -// Debug page - Power and Energy charts +// Debug page - Raw Power and Energy metrics (function() { 'use strict'; @@ -10,55 +10,81 @@ return document.getElementById('aggregate-select'); } + /** + * Execute a query and convert to Plotly traces. + * @param {string} query - MetricsQL query + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} step - Step string + * @returns {Promise} Array of Plotly traces + */ + async function queryToTraces(query, start, end, step) { + try { + var result = await Timeseries.queryRangeRaw(query, start, end, step); + return Timeseries.toPlotlyTraces(result); + } catch (e) { + console.error('Query failed:', query, e); + return []; + } + } + async function loadPowerChart() { var elementId = 'power-chart'; Utils.showLoading(elementId); var hours = parseInt(getHoursSelect().value); var aggregateMinutes = parseInt(getAggregateSelect().value); + var step = aggregateMinutes + 'm'; var now = new Date(); var start = new Date(now.getTime() - hours * 60 * 60 * 1000); try { - var data = await Api.power({ - start: Utils.formatDate(start), - end: Utils.formatDate(now), - aggregate_minutes: aggregateMinutes, - }); + // Query raw power metrics + var powerQuery = 'avg_over_time(openess_power_watts[' + step + '])'; + var scheduledQuery = 'avg_over_time(openess_scheduled_power_watts[' + step + '])'; - if (!data.series || Object.keys(data.series).length === 0) { - Utils.showError(elementId, 'No power flow data available'); + var [powerTraces, scheduledTraces] = await Promise.all([ + queryToTraces(powerQuery, start, now, step), + queryToTraces(scheduledQuery, start, now, step), + ]); + + var traces = powerTraces.concat(scheduledTraces); + + if (traces.length === 0) { + Utils.showError(elementId, 'No power data available'); return; } + // Style scheduled traces differently + traces.forEach(function(trace) { + if (trace.name && trace.name.indexOf('scheduled') !== -1) { + trace.line = { width: 1.5, dash: 'dot' }; + } else { + trace.line = { width: 1.5 }; + } + }); + var settings = Settings.load(); var useKw = settings.powerUnit === 'kw'; var powerUnit = useKw ? 'kW' : 'W'; + var divisor = useKw ? 1000 : 1; - var traces = []; - var sortedKeys = Object.keys(data.series).sort(); - - for (var i = 0; i < sortedKeys.length; i++) { - var key = sortedKeys[i]; - var series = data.series[key]; - if (!series.timestamps || !series.values) continue; - - traces.push({ - x: series.timestamps.map(function(t) { return new Date(t); }), - y: series.values, - type: 'scatter', - mode: 'lines', - name: key, - line: { width: 1.5 }, - connectgaps: false, - hovertemplate: '%{y:.1f} ' + powerUnit + '' + key + '', + if (useKw) { + traces.forEach(function(trace) { + trace.y = trace.y.map(function(v) { return v / divisor; }); }); } + traces.forEach(function(trace) { + trace.hovertemplate = '%{y:.1f} ' + powerUnit + '' + trace.name + ''; + }); + var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, now); layout.hovermode = 'x unified'; + layout.yaxis = layout.yaxis || {}; + layout.yaxis.title = { text: powerUnit }; Utils.makePlot(elementId, traces, layout); } catch (error) { console.error('Error loading power flows:', error); @@ -71,56 +97,49 @@ Utils.showLoading(elementId); var hours = parseInt(getHoursSelect().value); + var aggregateMinutes = parseInt(getAggregateSelect().value); + var step = aggregateMinutes + 'm'; var now = new Date(); var start = new Date(now.getTime() - hours * 60 * 60 * 1000); try { - var data = await Api.energy({ - start: Utils.formatDate(start), - end: Utils.formatDate(now), + // Query raw energy counter and integrated power + var energyQuery = 'openess_energy_kwh'; + var integratedQuery = 'integrate(openess_power_watts) / 3600000'; // Convert Ws to kWh + + var [energyTraces, integratedTraces] = await Promise.all([ + queryToTraces(energyQuery, start, now, step), + queryToTraces(integratedQuery, start, now, step), + ]); + + // Mark integrated traces + integratedTraces.forEach(function(trace) { + trace.name = trace.name + ' [integrated]'; + trace.line = { width: 1.5, dash: 'dot' }; }); - if (!data.series || Object.keys(data.series).length === 0) { - Utils.showError(elementId, 'No energy flow data available'); + var traces = energyTraces.concat(integratedTraces); + + if (traces.length === 0) { + Utils.showError(elementId, 'No energy data available'); return; } - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var energyUnit = useKw ? 'kWh' : 'Wh'; - - var traces = []; - var sortedKeys = Object.keys(data.series).sort(); - - for (var i = 0; i < sortedKeys.length; i++) { - var key = sortedKeys[i]; - var series = data.series[key]; - if (!series.timestamps || !series.values) continue; - - var timestamps = series.timestamps.map(function(t) { return new Date(t); }); - timestamps.push(new Date()); - var lastValue = series.values[series.values.length - 1]; - var values = series.values.concat([lastValue]); - - var isIntegrated = key.includes('[integrated]'); - traces.push({ - x: timestamps, - y: values, - type: 'scatter', - mode: 'lines', - name: key, - line: { - width: isIntegrated ? 1.5 : 2, - dash: isIntegrated ? 'dot' : 'solid', - }, - hovertemplate: '%{y:.2f} ' + energyUnit + '' + key + '', - }); - } + // Style energy traces + energyTraces.forEach(function(trace) { + trace.line = { width: 2 }; + }); + + traces.forEach(function(trace) { + trace.hovertemplate = '%{y:.3f} kWh' + trace.name + ''; + }); var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, now); layout.hovermode = 'x unified'; + layout.yaxis = layout.yaxis || {}; + layout.yaxis.title = { text: 'kWh' }; Utils.makePlot(elementId, traces, layout); } catch (error) { console.error('Error loading energy flows:', error); diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 39c988f..2cb80f5 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -4,9 +4,10 @@ var dashboardStart = null; var dashboardEnd = null; - var currentFoR = 'multiplus'; + var currentEnergyView = null; // Current energy view ID var currentPowerMode = 'total'; // 'total' or 'phases' var cachedPowerConfig = null; // Cached power chart config + var cachedEnergyConfig = null; // Cached energy chart config var rangeOffset = 0; var isRelayoutInProgress = false; @@ -89,8 +90,6 @@ }); } - var cachedEnergyConfig = null; - /** * Execute a MetricsQL query and return a Plotly trace. * @param {string} query - MetricsQL query with $step placeholder @@ -130,6 +129,61 @@ return null; } + /** + * Initialize energy view buttons based on config. + */ + function initEnergyViewButtons() { + if (!cachedEnergyConfig || !cachedEnergyConfig.views) return; + + var container = document.getElementById('for-buttons'); + if (!container) return; + + // Clear existing buttons + container.innerHTML = ''; + + // Create buttons for each view + cachedEnergyConfig.views.forEach(function(view, index) { + var btn = document.createElement('button'); + btn.className = 'btn-toggle'; + btn.dataset.value = view.id; + btn.textContent = view.name; + + // Set first button or saved preference as active + if (currentEnergyView === view.id || (!currentEnergyView && index === 0)) { + btn.classList.add('active'); + currentEnergyView = view.id; + } + + btn.addEventListener('click', function() { + container.querySelectorAll('.btn-toggle').forEach(function(b) { + b.classList.remove('active'); + }); + btn.classList.add('active'); + currentEnergyView = view.id; + Settings.savePagePref('dashboard', 'for', currentEnergyView); + reloadEnergyChart(); + }); + + container.appendChild(btn); + }); + } + + /** + * Get the current view config. + * @returns {Object|null} Current view config or null + */ + function getCurrentEnergyView() { + if (!cachedEnergyConfig || !cachedEnergyConfig.views) return null; + + for (var i = 0; i < cachedEnergyConfig.views.length; i++) { + if (cachedEnergyConfig.views[i].id === currentEnergyView) { + return cachedEnergyConfig.views[i]; + } + } + // Fallback to first view + return cachedEnergyConfig.views[0] || null; + } + async function loadEnergyChart(elementId, start, end, bucketMinutes) { Utils.showLoading(elementId); @@ -137,48 +191,27 @@ // Fetch query definitions (cached after first call) if (!cachedEnergyConfig) { cachedEnergyConfig = await Api.chartsEnergyQueries(); + initEnergyViewButtons(); } - var step = bucketMinutes + 'm'; - var promises = []; - - // Grid queries (system-wide) - promises.push(executeEnergyQuery( - cachedEnergyConfig.grid_import_query, 'Grid Import', - start, end, step, { color: '#e74c3c', negate: true } - )); - promises.push(executeEnergyQuery( - cachedEnergyConfig.grid_export_query, 'Grid Export', - start, end, step, { color: '#2ecc71' } - )); - - // Solar query (if available) - if (cachedEnergyConfig.solar_query) { - promises.push(executeEnergyQuery( - cachedEnergyConfig.solar_query, 'Solar', - start, end, step, { color: '#f1c40f' } - )); + var view = getCurrentEnergyView(); + if (!view || !view.queries) { + Utils.showError(elementId, 'No energy view configured'); + return; } - // Per-battery-system queries - var batteryIds = Object.keys(cachedEnergyConfig.battery_systems || {}); - var multipleSystems = batteryIds.length > 1; - - for (var i = 0; i < batteryIds.length; i++) { - var bsId = batteryIds[i]; - var bs = cachedEnergyConfig.battery_systems[bsId]; - var prefix = multipleSystems ? bsId + ' ' : ''; + var step = bucketMinutes + 'm'; + var promises = []; - // AC side: charger input (charge) and inverter output (discharge) - promises.push(executeEnergyQuery( - bs.energy_to_charger, prefix + 'Charge', - start, end, step, { color: '#3498db', negate: true } - )); + // Execute all queries for the current view + view.queries.forEach(function(queryDef) { promises.push(executeEnergyQuery( - bs.energy_from_inverter, prefix + 'Discharge', - start, end, step, { color: '#f39c12' } + queryDef.query, + queryDef.label, + start, end, step, + { color: queryDef.color, negate: queryDef.negate } )); - } + }); var results = await Promise.all(promises); var traces = results.filter(function(t) { return t !== null; }); @@ -312,7 +345,7 @@ } } - async function loadPriceChart(elementId, start, end) { + async function loadPriceChart(elementId, start, end, aggregateMinutes) { Utils.showLoading(elementId); // Extend end to show future prices @@ -383,11 +416,12 @@ } } - async function loadSocChart(elementId, start, end) { + async function loadSocChart(elementId, start, end, aggregateMinutes) { Utils.showLoading(elementId); try { // Fetch query definitions from backend + var step = aggregateMinutes + 'm'; var config = await Api.chartsBatteryQueries(); var batteryNames = Object.keys(config); var multipleSystems = batteryNames.length > 1; @@ -397,9 +431,9 @@ var queryPromises = batteryNames.map(async function(name) { var queries = config[name]; var [socResult, scheduleResult, voltageResult] = await Promise.all([ - Timeseries.queryRangeRaw(queries.soc_query, start, end).catch(function() { return null; }), - Timeseries.queryRangeRaw(queries.schedule_soc_query, start, end).catch(function() { return null; }), - Timeseries.queryRangeRaw(queries.voltage_query, start, end).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.soc_query, start, end, step).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.schedule_soc_query, start, end, step).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.voltage_query, start, end, step).catch(function() { return null; }), ]); return { name: name, socResult: socResult, scheduleResult: scheduleResult, voltageResult: voltageResult }; }); @@ -492,8 +526,8 @@ await Promise.all([ loadEnergyChart('energy-chart', dashboardStart, dashboardEnd, bucketMinutes), loadPowerChart('power-chart', dashboardStart, dashboardEnd, aggregateMinutes), - loadPriceChart('prices-chart', dashboardStart, dashboardEnd), - loadSocChart('soc-chart', dashboardStart, dashboardEnd) + loadPriceChart('prices-chart', dashboardStart, dashboardEnd, aggregateMinutes), + loadSocChart('soc-chart', dashboardStart, dashboardEnd, aggregateMinutes) ]); setupZoomSync(); @@ -509,19 +543,13 @@ document.addEventListener('DOMContentLoaded', function() { var savedRange = Settings.loadPagePref('dashboard', 'range', '24'); - var savedFoR = Settings.loadPagePref('dashboard', 'for', 'multiplus'); + var savedEnergyView = Settings.loadPagePref('dashboard', 'for', null); var savedPowerMode = Settings.loadPagePref('dashboard', 'powerMode', 'total'); document.getElementById('range-select').value = savedRange; - currentFoR = savedFoR; + currentEnergyView = savedEnergyView; // Will be validated when config loads currentPowerMode = savedPowerMode; - // Energy frame-of-reference buttons - var forButtons = document.querySelectorAll('#for-buttons .btn-toggle'); - forButtons.forEach(function(btn) { - btn.classList.toggle('active', btn.dataset.value === savedFoR); - }); - // Power phase toggle buttons var powerPhaseButtons = document.querySelectorAll('#power-phase-buttons .btn-toggle'); powerPhaseButtons.forEach(function(btn) { @@ -546,16 +574,6 @@ } }); - forButtons.forEach(function(btn) { - btn.addEventListener('click', function() { - document.querySelectorAll('#for-buttons .btn-toggle').forEach(function(b) { b.classList.remove('active'); }); - btn.classList.add('active'); - currentFoR = btn.dataset.value || 'multiplus'; - Settings.savePagePref('dashboard', 'for', currentFoR); - reloadEnergyChart(); - }); - }); - // Power phase toggle handlers powerPhaseButtons.forEach(function(btn) { btn.addEventListener('click', function() { diff --git a/open_ess/frontend/static/timeseries.js b/open_ess/frontend/static/timeseries.js index 58d2b16..09fafc0 100644 --- a/open_ess/frontend/static/timeseries.js +++ b/open_ess/frontend/static/timeseries.js @@ -108,6 +108,64 @@ var Timeseries = (function () { return response.json(); } + /** + * Execute an instant query against the timeseries backend. + * + * @param {string} query - MetricsQL query string + * @param {Date} [time] - Optional evaluation time (defaults to now) + * @returns {Promise} Query result in Prometheus/VM format + */ + async function queryInstant(query, time) { + var params = new URLSearchParams({ query: query }); + if (time) { + params.set("time", (time.getTime() / 1000).toString()); + } + + var response = await fetch("/api/v1/query?" + params); + if (!response.ok) { + throw new Error("Query failed: HTTP " + response.status); + } + return response.json(); + } + + /** + * Extract a single value from an instant query result. + * + * @param {Object} result - Query result from queryInstant + * @returns {number|null} The value or null if not found + */ + function getInstantValue(result) { + if (!result || !result.data || !result.data.result || result.data.result.length === 0) { + return null; + } + var first = result.data.result[0]; + if (first.value && first.value.length === 2) { + return parseFloat(first.value[1]); + } + return null; + } + + /** + * Extract all values from an instant query result as a map. + * + * @param {Object} result - Query result from queryInstant + * @param {string} labelKey - Label key to use as map key + * @returns {Object} Map of label value to metric value + */ + function getInstantValues(result, labelKey) { + var values = {}; + if (!result || !result.data || !result.data.result) { + return values; + } + result.data.result.forEach(function(series) { + var key = series.metric[labelKey] || "unknown"; + if (series.value && series.value.length === 2) { + values[key] = parseFloat(series.value[1]); + } + }); + return values; + } + /** * Format metric labels into a readable string. * @@ -192,6 +250,9 @@ var Timeseries = (function () { init: init, queryRange: queryRange, queryRangeRaw: queryRangeRaw, + queryInstant: queryInstant, + getInstantValue: getInstantValue, + getInstantValues: getInstantValues, toPlotlyTraces: toPlotlyTraces, calculateStep: calculateStep, formatLabels: formatLabels, diff --git a/open_ess/frontend/templates/cycles.html b/open_ess/frontend/templates/cycles.html index a68fdde..b99fc28 100644 --- a/open_ess/frontend/templates/cycles.html +++ b/open_ess/frontend/templates/cycles.html @@ -24,6 +24,10 @@

Power vs Losses

+
+ + +
diff --git a/open_ess/frontend/templates/metrics.html b/open_ess/frontend/templates/metrics.html index 7e5490c..983de55 100644 --- a/open_ess/frontend/templates/metrics.html +++ b/open_ess/frontend/templates/metrics.html @@ -25,9 +25,7 @@

Energy

- - - +
From 71fe162b66254d12977ee26f31c0c17e51a6e682 Mon Sep 17 00:00:00 2001 From: david Date: Thu, 7 May 2026 22:30:11 +0200 Subject: [PATCH 14/18] entsoe client: fix bug where prices aren't fetches anymore --- open_ess/pricing/client.py | 6 ++++-- open_ess/util.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/open_ess/pricing/client.py b/open_ess/pricing/client.py index 5e6e840..71df335 100644 --- a/open_ess/pricing/client.py +++ b/open_ess/pricing/client.py @@ -9,6 +9,7 @@ from pandas import DataFrame from open_ess.timeseries import Sample, TimeseriesBackend, VectorResult +from open_ess.util import ms_to_dt from .areas import AREAS from .config import PriceConfig @@ -89,13 +90,14 @@ def fetch_missing_prices(self, area: str) -> None: end_of_tomorrow = (now + timedelta(days=2)).replace(hour=0, minute=0, second=0, microsecond=0) result: VectorResult = self._mql_client.query( - f'last_over_time(openess_prices{{area="{area}"}}[8w])', time=end_of_tomorrow + f'timestamp(openess_prices{{area="{area}", price="market"}}[8w])', time=end_of_tomorrow ) if len(result.series) == 0: fetch_start = now.replace(hour=0, minute=0, second=0, microsecond=0) fetch_start -= timedelta(weeks=8) else: - latest = result.series[0].timestamp + latest_ms = int(result.series[0].value) * 1000 + latest = ms_to_dt(latest_ms) if latest >= end_of_tomorrow: return else: diff --git a/open_ess/util.py b/open_ess/util.py index 72b34f2..d463f26 100644 --- a/open_ess/util.py +++ b/open_ess/util.py @@ -1,5 +1,6 @@ import argparse import logging +from datetime import UTC, datetime from pathlib import Path from typing import ClassVar @@ -70,3 +71,13 @@ def parse_args(description: str) -> argparse.Namespace: help="Path to config file (YAML)", ) return parser.parse_args() + + +def dt_to_ms(dt: datetime) -> int: + """UTC datetime to Unix milliseconds.""" + return int(dt.timestamp() * 1000) + + +def ms_to_dt(ms: int) -> datetime: + """Unix milliseconds to UTC datetime.""" + return datetime.fromtimestamp(ms / 1000, tz=UTC) From 9f1ae8e48598665013dff4c22bb9edacaa300329 Mon Sep 17 00:00:00 2001 From: david Date: Sat, 9 May 2026 09:18:18 +0200 Subject: [PATCH 15/18] frontend: remove cli.py --- open_ess/frontend/README.md | 24 ++---------------------- open_ess/frontend/cli.py | 36 ------------------------------------ 2 files changed, 2 insertions(+), 58 deletions(-) delete mode 100644 open_ess/frontend/cli.py diff --git a/open_ess/frontend/README.md b/open_ess/frontend/README.md index 9893ef8..2906ffe 100644 --- a/open_ess/frontend/README.md +++ b/open_ess/frontend/README.md @@ -5,25 +5,5 @@ It's a bit of a mess now so if you read this and think you can do better, feel f contact me on GitHub or create a pull request! In short, the frontend uses FastAPI and Pydantic models to define the api endpoints. `main.py` -runs the FastAPI stuff with `uvicorn`. Typescript definitions for the api responses and -api endpoints are generated to `src/types.ts` and `npm` compiles the `.ts` files to `.js` -using `esbuild`. - -And here's the long version: - -`routes/api.py` defines the api using FastAPI and Pydantic. These can be used to generate -`src/types.ts` using the `generate-types` command. The script can be found at -`open_ess/scripts/generate_types.py`. - -The pages themselves are also written in typescript and can also be found in -`open_ess/frontend/src`. These are then compiled to javascript and stored in -`open_ess/frontend/static`. This can be done with either; - - `esbuild open_ess/frontend/src/*.ts --outdir=open_ess/frontend/static --bundle --minify` - -or - - `npm run build` - -The `datatables` library is pretty small and is bundled with the `.js` files. `plotly` is very -big (5MB) and is not bundled. `plotly` is in `static/vendor` and is loaded via `base.html`. +runs the FastAPI stuff with `uvicorn`. Javascript definitions for the api responses and +api endpoints are generated to `src/api.js` and `npm`. diff --git a/open_ess/frontend/cli.py b/open_ess/frontend/cli.py deleted file mode 100644 index 63e18a4..0000000 --- a/open_ess/frontend/cli.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging - -import uvicorn - -from open_ess.config import Config -from open_ess.database import Database -from open_ess.frontend.app import create_app -from open_ess.util import parse_args, setup_logging - -setup_logging() -logger = logging.getLogger(__name__) - - -def main() -> None: - args = parse_args("Open Energy Storage System web dashboard") - - config = Config.from_file(args.config) - if not config.frontend.enable: - logger.info("Frontend is not enabled. Exiting...") - return - - database = Database(config.database) - - logger.info(f"Starting web server on http://{config.frontend.host}:{config.frontend.port}") - - app = create_app(database, config, battery_systems=[]) - uvicorn.run( - app, - host=config.frontend.host, - port=config.frontend.port, - log_level="info", - ) - - -if __name__ == "__main__": - main() From 926284f704012f7fea59c1d03b83ba80b57516e3 Mon Sep 17 00:00:00 2001 From: david Date: Sat, 9 May 2026 09:25:54 +0200 Subject: [PATCH 16/18] fix mypy issues --- open_ess/optimizer/service.py | 11 ++++++++--- open_ess/pricing/client.py | 8 +++++--- open_ess/timeseries/base.py | 7 ++----- open_ess/timeseries/metricsqlite/backend.py | 6 ++++-- open_ess/victron_modbus/client.py | 6 ++++-- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/open_ess/optimizer/service.py b/open_ess/optimizer/service.py index 54281c3..bf49a5c 100644 --- a/open_ess/optimizer/service.py +++ b/open_ess/optimizer/service.py @@ -52,7 +52,7 @@ def wait_until_next(self) -> None: ) + timedelta(minutes=self._price_config.aggregate_minutes) self.wait_seconds((next_run - now).total_seconds()) - def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]]): + def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]]) -> None: """ Schedules are stored in a bit of an insane way in the timeseries backend... The timestamp of each inserted sample is increased by a tiny bit, proportional to how far in @@ -60,6 +60,11 @@ def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]] generated sample for a given timestamp. """ + device_id = self._battery_system.id + if device_id is None: + logger.warning("Cannot upsert schedule: battery system has no device ID") + return + samples: list[Sample] = [] now = datetime.now(UTC) @@ -71,7 +76,7 @@ def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]] metric="openess_scheduled_power_watts", timestamp=ts_start + delta, value=power, - labels={"device": self._battery_system.id}, + labels={"device": device_id}, ) ) samples.append( @@ -79,7 +84,7 @@ def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]] metric="openess_scheduled_soc_ratio", timestamp=ts_end + delta, value=soc / 100, - labels={"device": self._battery_system.id}, + labels={"device": device_id}, ) ) diff --git a/open_ess/pricing/client.py b/open_ess/pricing/client.py index 71df335..83b1e64 100644 --- a/open_ess/pricing/client.py +++ b/open_ess/pricing/client.py @@ -89,9 +89,11 @@ def fetch_missing_prices(self, area: str) -> None: now = datetime.now(UTC) end_of_tomorrow = (now + timedelta(days=2)).replace(hour=0, minute=0, second=0, microsecond=0) - result: VectorResult = self._mql_client.query( + query_result = self._mql_client.query( f'timestamp(openess_prices{{area="{area}", price="market"}}[8w])', time=end_of_tomorrow ) + assert isinstance(query_result, VectorResult) + result = query_result if len(result.series) == 0: fetch_start = now.replace(hour=0, minute=0, second=0, microsecond=0) fetch_start -= timedelta(weeks=8) @@ -108,8 +110,8 @@ def fetch_missing_prices(self, area: str) -> None: if prices: self._upsert_prices(area, prices) - def _upsert_prices(self, area: str, prices: list[tuple[datetime, datetime, float]]): - def make_sample(_ts: datetime, _price_type: str, _price: float): + def _upsert_prices(self, area: str, prices: list[tuple[datetime, datetime, float]]) -> None: + def make_sample(_ts: datetime, _price_type: str, _price: float) -> Sample: return Sample( metric="openess_prices", labels={ diff --git a/open_ess/timeseries/base.py b/open_ess/timeseries/base.py index 61aa636..ac8cb5f 100644 --- a/open_ess/timeseries/base.py +++ b/open_ess/timeseries/base.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Literal if TYPE_CHECKING: - from open_ess.battery_system import BatterySystem + pass @dataclass @@ -129,7 +129,7 @@ def get_prices( area: str, start: datetime, end: datetime, - hourly=False, + hourly: bool = False, price: Literal["market", "buy", "sell"] = "market", ) -> list[tuple[datetime, float]]: """Prices are returned in currency per Kwh (usually €/kWh).""" @@ -153,6 +153,3 @@ def get_prices( if not result.series: return [] return list(result.series[0].values) - - def get_schedule(self, battery_system: "BatterySystem"): - return [] diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py index 07cae70..88ab86d 100644 --- a/open_ess/timeseries/metricsqlite/backend.py +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -1,6 +1,7 @@ import logging from datetime import UTC, datetime +import fastapi from metricsqlite import MetricsQLiteClient from metricsqlite.engine import InstantVector, MatrixResult, RangeVectorResult, ScalarResult from metricsqlite.fastapi import create_router @@ -106,8 +107,9 @@ def _convert_range_result(self, result: MatrixResult) -> RangeQueryResult: series.append(RangeSeries(metric=labels, values=values)) return RangeQueryResult(series=series) - def create_fastapi_router(self): - return create_router(self._client) + def create_fastapi_router(self) -> fastapi.APIRouter: + router: fastapi.APIRouter = create_router(self._client) + return router def close(self) -> None: """Close the database connection.""" diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index 5eb31b6..7bd51dc 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -59,7 +59,8 @@ def initialize(self) -> bool: if not self._config.monitor_only: self.write(self.system_id, System.ESS_MODE, 3) - self._current_soc = self.read(self.system_id, System.BATTERY_SOC) + soc = self.read(self.system_id, System.BATTERY_SOC) + self._current_soc = soc if isinstance(soc, (int, float)) else None return True @@ -146,7 +147,8 @@ def scrape_metrics(self) -> None: samples: list[Sample] = [] def add(metric: str, value: float | None, labels: dict[str, str]) -> None: - labels["device"] = self.serial + if self.serial is not None: + labels["device"] = self.serial if value is not None: samples.append(Sample(metric, value, timestamp, labels)) From c13816b97fe83886cc08748eadf491c62492e6ae Mon Sep 17 00:00:00 2001 From: david Date: Sat, 9 May 2026 09:28:25 +0200 Subject: [PATCH 17/18] generate api.js --- open_ess/frontend/static/api.js | 197 +++++++++++++++++++++-------- open_ess/scripts/generate_types.py | 22 +++- 2 files changed, 162 insertions(+), 57 deletions(-) diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index 2e75a23..49942b1 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -5,12 +5,43 @@ // === Types === // ============ +/** @typedef {"ok" | "warning" | "error"} Status */ + +/** @typedef {} StrEnum */ + /** * @typedef {Object} TimeSeries * @property {string[]} [timestamps] * @property {number[]} [values] */ +/** + * @typedef {Object} BatteryCycle + * @property {string} [start_time] + * @property {string} [end_time] + * @property {number} [duration_hours] + * @property {number} [min_soc] + * @property {(number | null)} [ac_energy_in] + * @property {(number | null)} [ac_energy_out] + * @property {(number | null)} [dc_energy_in] + * @property {(number | null)} [dc_energy_out] + * @property {(number | null)} [system_efficiency] + * @property {(number | null)} [battery_efficiency] + * @property {(number | null)} [charger_efficiency] + * @property {(number | null)} [inverter_efficiency] + * @property {(number | null)} [profit] + * @property {(number | null)} [scheduled_profit] + */ + +/** + * @typedef {Object} BatteryPowerValues + * @property {(number | null)} [charger] + * @property {(number | null)} [inverter] + * @property {(number | null)} [battery] + * @property {(number | null)} [losses] + * @property {(number | null)} [soc] + */ + /** * @typedef {Object} BatteryQueriesResponse * @property {string} [soc_query] @@ -19,29 +50,39 @@ */ /** - * @typedef {Object} EnergyQueryDef - * @property {string} query - * @property {string} label - * @property {string} color - * @property {boolean} [negate] + * @typedef {Object} BatterySystemInfo + * @property {string} [id] + * @property {string} [name] */ /** - * @typedef {Object} EnergyViewConfig - * @property {string} id - * @property {string} name - * @property {EnergyQueryDef[]} queries + * @typedef {Object} ChartsPowerResponse + * @property {PowerQueryDef[]} [queries] + * @property {string[]} [phases] + */ + +/** + * @typedef {Object} EfficiencyScatterPoint + * @property {string} [time] + * @property {number} [battery_power] + * @property {number} [inverter_charger_power] + * @property {number} [losses] + * @property {(number | null)} [efficiency] + * @property {(number | null)} [soc] + * @property {string} [category] */ /** * @typedef {Object} EnergyQueriesResponse - * @property {EnergyViewConfig[]} views + * @property {EnergyViewConfig[]} [views] */ /** - * @typedef {Object} ChartsPowerResponse - * @property {PowerQueryDef[]} [queries] - * @property {string[]} [phases] + * @typedef {Object} EnergyQueryDef + * @property {string} [query] + * @property {string} [label] + * @property {string} [color] + * @property {boolean} [negate] */ /** @@ -49,6 +90,13 @@ * @property {Object.} [series] */ +/** + * @typedef {Object} EnergyViewConfig + * @property {string} [id] + * @property {string} [name] + * @property {EnergyQueryDef[]} [queries] + */ + /** * @typedef {Object} HealthResponse * @property {string} [status] @@ -56,6 +104,14 @@ * @property {string[]} [tables] */ +/** + * @typedef {Object} PowerFlowData + * @property {Object.} [grid] + * @property {(number | null)} [solar] + * @property {Object.} [consumption] + * @property {Object.} [batteries] + */ + /** * @typedef {Object} PowerQueryDef * @property {string} [label] @@ -77,6 +133,30 @@ * @property {string} [currency] */ +/** + * @typedef {Object} ServiceMessage + * @property {string} [message] + */ + +/** + * @typedef {Object} ServiceStatus + * @property {Status} [status] + * @property {ServiceMessage[]} [messages] + */ + +/** + * @typedef {Object} ServicesStatusResponse + * @property {(ServiceStatus | null)} [database] + * @property {(ServiceStatus | null)} [optimizer] + */ + +/** + * @typedef {Object} SystemLayoutData + * @property {number[]} [phases] + * @property {boolean} [has_solar] + * @property {BatterySystemInfo[]} [battery_systems] + */ + /** * @typedef {Object} TimeSeries * @property {string[]} [timestamps] @@ -103,10 +183,10 @@ }, /** - * @returns {Promise} + * @returns {Promise} */ - chartsEnergyQueries: async function() { - var response = await fetch('/api/charts/energy-queries'); + systemLayout: async function() { + var response = await fetch('/api/system-layout'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -114,14 +194,10 @@ }, /** - * @param {Annotated} params.mql_client - * @returns {Promise} + * @returns {Promise} */ - chartsPowerQueries: async function(params) { - var searchParams = new URLSearchParams(); - if (params.mql_client !== undefined) searchParams.set('mql_client', String(params.mql_client)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/charts/power-queries' + query); + powerFlow: async function() { + var response = await fetch('/api/power-flow'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -129,14 +205,10 @@ }, /** - * @param {(string | null)} [params.area] - * @returns {Promise} + * @returns {Promise} */ - chartsPriceQueries: async function(params) { - var searchParams = new URLSearchParams(); - if (params.area !== undefined) searchParams.set('area', String(params.area)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/charts/price-queries' + query); + servicesStatus: async function() { + var response = await fetch('/api/services-status'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -144,10 +216,10 @@ }, /** - * @returns {Promise>} + * @returns {Promise} */ - chartsBatteryQueries: async function() { - var response = await fetch('/api/charts/battery-queries'); + batteryIds: async function() { + var response = await fetch('/api/battery-ids'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -155,10 +227,10 @@ }, /** - * @returns {Promise} + * @returns {Promise} */ - systemLayout: async function() { - var response = await fetch('/api/system-layout'); + chartsEnergyQueries: async function() { + var response = await fetch('/api/charts/energy-queries'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -166,10 +238,10 @@ }, /** - * @returns {Promise} + * @returns {Promise} */ - powerFlow: async function() { - var response = await fetch('/api/power-flow'); + chartsPowerQueries: async function() { + var response = await fetch('/api/charts/power-queries'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -177,10 +249,25 @@ }, /** - * @returns {Promise} + * @param {(string | null)} [params.area] + * @returns {Promise} */ - servicesStatus: async function() { - var response = await fetch('/api/services-status'); + chartsPriceQueries: async function(params) { + var searchParams = new URLSearchParams(); + if (params.area !== undefined) searchParams.set('area', String(params.area)); + var query = searchParams.toString() ? '?' + searchParams.toString() : ''; + var response = await fetch('/api/charts/price-queries' + query); + if (!response.ok) { + throw new Error('HTTP ' + response.status); + } + return response.json(); + }, + + /** + * @returns {Promise>} + */ + chartsBatteryQueries: async function() { + var response = await fetch('/api/charts/battery-queries'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -188,15 +275,21 @@ }, /** - * @param {Object} params + * @param {(string | null)} [params.battery_id] + * @param {(string | null)} [params.start] + * @param {(string | null)} [params.end] * @param {number} [params.aggregate_minutes] + * @param {number} [params.idle_threshold] * @param {number} [params.limit] - * @returns {Promise} + * @returns {Promise} */ efficiencyScatter: async function(params) { - params = params || {}; var searchParams = new URLSearchParams(); + if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); + if (params.start !== undefined) searchParams.set('start', String(params.start)); + if (params.end !== undefined) searchParams.set('end', String(params.end)); if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); + if (params.idle_threshold !== undefined) searchParams.set('idle_threshold', String(params.idle_threshold)); if (params.limit !== undefined) searchParams.set('limit', String(params.limit)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; var response = await fetch('/api/efficiency-scatter' + query); @@ -207,17 +300,17 @@ }, /** - * @param {Object} params - * @param {string} [params.start] - * @param {string} [params.end] + * @param {(string | null)} [params.battery_id] + * @param {(string | null)} [params.start] + * @param {(string | null)} [params.end] * @param {number} [params.min_soc_swing] - * @returns {Promise} + * @returns {Promise} */ cycles: async function(params) { - params = params || {}; var searchParams = new URLSearchParams(); - if (params.start !== undefined) searchParams.set('start', params.start); - if (params.end !== undefined) searchParams.set('end', params.end); + if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); + if (params.start !== undefined) searchParams.set('start', String(params.start)); + if (params.end !== undefined) searchParams.set('end', String(params.end)); if (params.min_soc_swing !== undefined) searchParams.set('min_soc_swing', String(params.min_soc_swing)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; var response = await fetch('/api/cycles' + query); diff --git a/open_ess/scripts/generate_types.py b/open_ess/scripts/generate_types.py index 93c4833..98cdb4e 100644 --- a/open_ess/scripts/generate_types.py +++ b/open_ess/scripts/generate_types.py @@ -14,8 +14,9 @@ from enum import Enum from pathlib import Path from types import NoneType, UnionType -from typing import Any, TypedDict, get_args, get_origin +from typing import Annotated, Any, TypedDict, get_args, get_origin +from fastapi.params import Depends from fastapi.routing import APIRoute from pydantic import BaseModel @@ -29,6 +30,17 @@ class _ParamInfo(TypedDict): logger = logging.getLogger(__name__) +def is_dependency_injection(annotation: Any) -> bool: + """Check if an annotation is a FastAPI dependency (Annotated[X, Depends(...)]).""" + if get_origin(annotation) is Annotated: + args = get_args(annotation) + # args[0] is the base type, args[1:] are the metadata + for metadata in args[1:]: + if isinstance(metadata, Depends): + return True + return False + + def python_type_to_jsdoc(python_type: Any, models: dict[str, type]) -> str: """Convert a Python type annotation to JSDoc type.""" origin = get_origin(python_type) @@ -166,11 +178,11 @@ def generate_api_function(route: APIRoute, models_dict: dict[str, type]) -> tupl sig = inspect.signature(endpoint) for param_name, param in sig.parameters.items(): - # Skip dependency injection parameters - if param_name in ("db", "battery_configs", "price_config", "battery_systems"): - continue - annotation = param.annotation + + # Skip dependency injection parameters (Annotated[X, Depends(...)]) + if is_dependency_injection(annotation): + continue if annotation is inspect.Parameter.empty: continue From 7a3f6f7d150f09cb976b0b5fab6be792ca1c5787 Mon Sep 17 00:00:00 2001 From: david Date: Sat, 9 May 2026 16:16:43 +0200 Subject: [PATCH 18/18] api.py: refactor SystemLayoutResponse --- open_ess/frontend/app.py | 4 +- open_ess/frontend/routes/api.py | 57 ++++++++++++--------------- open_ess/frontend/static/api.js | 24 ++++------- open_ess/frontend/static/dashboard.js | 33 ++++++++-------- open_ess/victron_modbus/client.py | 6 +-- 5 files changed, 53 insertions(+), 71 deletions(-) diff --git a/open_ess/frontend/app.py b/open_ess/frontend/app.py index 07ba8ff..8cbc009 100644 --- a/open_ess/frontend/app.py +++ b/open_ess/frontend/app.py @@ -29,7 +29,7 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: _app.state.battery_systems = battery_systems _app.state.mql_client = mql_client yield - # _app.state.mql_client.close() + _app.state.mql_client.close() app = FastAPI( title="OpenESS", @@ -40,7 +40,7 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: app.include_router(pages_router) app.include_router(api_router, prefix="/api") - # Mount timeseries query endpoints + # Mount /query and /range_query if mql_client is not None: if isinstance(mql_client, MetricSQLiteBackend): app.include_router(mql_client.create_fastapi_router(), prefix="/api/v1") diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index 4bda32d..4ae87a2 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -7,9 +7,7 @@ from pydantic import BaseModel from open_ess.frontend.dependencies import BatterySystemsDep, MqlClientDep, PriceConfigDep -from open_ess.timeseries import TimeseriesBackend - -from .util import TimeSeries +from open_ess.timeseries import TimeseriesBackend, VectorResult if TYPE_CHECKING: pass @@ -18,12 +16,9 @@ router = APIRouter(tags=["api"]) -class PowerResponse(BaseModel): - series: dict[str, TimeSeries] - - -class EnergyResponse(BaseModel): - series: dict[str, TimeSeries] +# --------- # +# /health # +# --------- # class HealthResponse(BaseModel): @@ -54,38 +49,36 @@ class BatterySystemInfo(BaseModel): name: str -class SystemLayoutData(BaseModel): - phases: list[int] - has_solar: bool +class SolarInverterInfo(BaseModel): + id: str + name: str + + +class SystemLayoutResponse(BaseModel): + grid_phases: list[str] battery_systems: list[BatterySystemInfo] + solar_inverters: list[SolarInverterInfo] -@router.get("/system-layout", response_model=SystemLayoutData) +@router.get("/system-layout", response_model=SystemLayoutResponse) async def get_system_layout( mql_client: MqlClientDep, battery_systems: BatterySystemsDep, -) -> SystemLayoutData: - # Discover phases from grid power metrics - phases: list[int] = [] +) -> SystemLayoutResponse: + phases = ["L1", "L2", "L3"] if mql_client: - result = mql_client.query('openess_power_watts{from="grid"}') - phase_set: set[int] = set() - if hasattr(result, "series"): - for series in result.series: - phase_label = series.metric.get("phase", "") - if phase_label.startswith("L"): - phase_set.add(int(phase_label[1:])) - phases = sorted(phase_set) if phase_set else [1, 2, 3] - else: - phases = [1, 2, 3] - - # TODO: detect solar from metrics - has_solar = False + result: VectorResult = mql_client.query('openess_power_watts{from="grid"}') + phase_labels: set[str] = set() + for series in result.series: + phase_label = series.metric.get("phase", "") + phase_labels.add(phase_label) + if phase_labels: + phases = sorted(phase_labels) - return SystemLayoutData( - phases=phases, - has_solar=has_solar, + return SystemLayoutResponse( + grid_phases=phases, battery_systems=[BatterySystemInfo(id=b.id, name=b.name or b.id) for b in battery_systems], + solar_inverters=[], # TODO ) diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index 49942b1..363bf0b 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -85,11 +85,6 @@ * @property {boolean} [negate] */ -/** - * @typedef {Object} EnergyResponse - * @property {Object.} [series] - */ - /** * @typedef {Object} EnergyViewConfig * @property {string} [id] @@ -119,11 +114,6 @@ * @property {(boolean | null)} is_total */ -/** - * @typedef {Object} PowerResponse - * @property {Object.} [series] - */ - /** * @typedef {Object} PriceQueriesResponse * @property {string} [market_query] @@ -151,16 +141,16 @@ */ /** - * @typedef {Object} SystemLayoutData - * @property {number[]} [phases] - * @property {boolean} [has_solar] - * @property {BatterySystemInfo[]} [battery_systems] + * @typedef {Object} SolarInverterInfo + * @property {string} [id] + * @property {string} [name] */ /** - * @typedef {Object} TimeSeries - * @property {string[]} [timestamps] - * @property {number[]} [values] + * @typedef {Object} SystemLayoutData + * @property {string[]} [grid_phases] + * @property {BatterySystemInfo[]} [battery_systems] + * @property {SolarInverterInfo[]} [solar_inverters] */ // =================== diff --git a/open_ess/frontend/static/dashboard.js b/open_ess/frontend/static/dashboard.js index 3eeaab1..eaba6a8 100644 --- a/open_ess/frontend/static/dashboard.js +++ b/open_ess/frontend/static/dashboard.js @@ -15,6 +15,7 @@ function renderPowerFlowDiagram(container, layout) { var batteryCount = layout.battery_systems.length; + var solarCount = layout.solar_inverters.length; var html = '
' + '' + @@ -29,12 +30,12 @@ '
' + '
Grid
' + '
' + - layout.phases.map(function(p) { return '
L' + p + ': -- W
'; }).join('') + + layout.grid_phases.map(function(p) { return '
' + p + ': -- W
'; }).join('') + '
' + '
-- W
' + ''; - if (layout.has_solar) { + if (solarCount >= 1) { html += '
' + '
' + '' + @@ -54,7 +55,7 @@ '
' + '
Consumption
' + '
' + - layout.phases.map(function(p) { return '
L' + p + ': -- W
'; }).join('') + + layout.grid_phases.map(function(p) { return '
' + p + ': -- W
'; }).join('') + '
' + '
-- W
' + '
'; @@ -124,7 +125,7 @@ paths += ''; } - if (layout.has_solar) { + if (layout.solar_inverters.length >= 1) { var solarBlock = document.getElementById('block-solar'); if (solarBlock) { var solarRect = solarBlock.getBoundingClientRect(); @@ -150,12 +151,12 @@ function updatePowerFlowData(layout, data) { var gridTotal = 0; - for (var i = 0; i < layout.phases.length; i++) { - var phase = layout.phases[i]; - var value = data.grid['L' + phase] || 0; + for (var i = 0; i < layout.grid_phases.length; i++) { + var phase = layout.grid_phases[i]; + var value = data.grid[phase] || 0; gridTotal += value; - var el = document.getElementById('grid-L' + phase); - if (el) el.textContent = 'L' + phase + ': ' + formatPower(value); + var el = document.getElementById('grid-' + phase); + if (el) el.textContent = phase + ': ' + formatPower(value); } var gridTotalEl = document.getElementById('grid-total'); if (gridTotalEl) { @@ -164,17 +165,17 @@ } var consTotal = 0; - for (var j = 0; j < layout.phases.length; j++) { - var p = layout.phases[j]; - var v = data.consumption['L' + p] || 0; + for (var j = 0; j < layout.grid_phases.length; j++) { + var p = layout.grid_phases[j]; + var v = data.consumption[p] || 0; consTotal += v; - var cel = document.getElementById('consumption-L' + p); - if (cel) cel.textContent = 'L' + p + ': ' + formatPower(v); + var cel = document.getElementById('consumption-' + p); + if (cel) cel.textContent = p + ': ' + formatPower(v); } var consTotalEl = document.getElementById('consumption-total'); if (consTotalEl) consTotalEl.textContent = formatPower(consTotal); - if (layout.has_solar && data.solar !== null) { + if (layout.solar_inverters.length >= 1 && data.solar !== null) { var solarEl = document.getElementById('solar-total'); if (solarEl) solarEl.textContent = formatPower(data.solar); } @@ -227,7 +228,7 @@ consLine.classList.toggle('flow-active', Math.abs(consTotal) > 50); } - if (layout.has_solar && data.solar !== null) { + if (layout.solar_inverters.length >= 1 && data.solar !== null) { var solarLine = document.getElementById('line-solar'); if (solarLine) { solarLine.classList.toggle('flow-generating', data.solar > 50); diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index 7bd51dc..ea04360 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -221,13 +221,11 @@ def add(metric: str, value: float | None, labels: dict[str, str]) -> None: ) add(POWER_METRIC, vebus_battery_power, {"from": "system", "to": "battery", "unit": "vebus"}) add(VOLTAGE_METRIC, vebus_dc_voltage, {"node": "battery", "unit": "vebus"}) - - # VEBus SOC vebus_soc = _get_float(vebus_values, VEBus.SOC) if vebus_soc is not None: add(SOC_METRIC, vebus_soc / 100, {"node": "battery", "unit": "vebus"}) - # VEBus energy flows + # VEBus energy add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT), {"from": "ac_in", "to": "ac_out"}) add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY), {"from": "ac_in", "to": "system"}) add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1), {"from": "ac_out", "to": "ac_in"}) @@ -235,7 +233,7 @@ def add(metric: str, value: float | None, labels: dict[str, str]) -> None: add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT), {"from": "system", "to": "ac_out"}) add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY), {"from": "ac_out", "to": "system"}) - # BMS (direct battery measurements) + # BMS bms_soc = None if self.battery_id is not None: bms_values = self.read_many(