From f1a42275d469b8c56f3db547ce8ce9cd6a3adea0 Mon Sep 17 00:00:00 2001 From: Chang Gao Date: Mon, 14 Sep 2026 14:10:48 +0200 Subject: [PATCH] Release 2.2.6 with server telemetry and hardened shared workspaces --- .github/workflows/ci.yml | 8 + README.md | 6 +- deployment/web/GpuContainerfile | 13 +- deployment/web/opendpd-gpu.service | 2 + docs/architecture/api.md | 2 +- docs/architecture/public-studio.md | 13 +- docs/contracts/openapi.json | 295 ++++++++++++++++- docs/documentation.md | 4 + docs/guides/server-load.md | 26 ++ docs/install.md | 6 +- docs/performance/studio-2.2.6.md | 23 ++ docs/performance/studio-2.2.6/gpu.json | 27 ++ .../performance/studio-2.2.6/screenshots.json | 18 + .../studio-2.2.6/security-scan.json | 313 ++++++++++++++++++ .../performance/studio-2.2.6/status-http.json | 9 + docs/releases/release-notes-2.2.6.md | 25 ++ docs/releases/security-review-2.2.6.md | 35 ++ docs/releases/support-matrix.md | 2 +- docs/testing.md | 4 + docs/tutorials/gui-quickstart.md | 8 +- docs/whats-new.md | 9 + frontend/package-lock.json | 4 +- frontend/package.json | 2 +- frontend/src/App.tsx | 2 + frontend/src/api/schema.ts | 110 +++++- frontend/src/components/WorkspaceExpiry.tsx | 22 ++ frontend/src/i18n/de.json | 35 +- frontend/src/i18n/en.json | 35 +- frontend/src/i18n/es.json | 35 +- frontend/src/i18n/fr.json | 35 +- frontend/src/i18n/it.json | 35 +- frontend/src/i18n/ja.json | 35 +- frontend/src/i18n/ko.json | 35 +- frontend/src/i18n/nl.json | 35 +- frontend/src/i18n/zh.json | 35 +- frontend/src/layout/AppShell.tsx | 17 +- frontend/src/pages/ServerStatusPage.test.tsx | 34 ++ frontend/src/pages/ServerStatusPage.tsx | 96 ++++++ frontend/vite.config.ts | 2 +- mkdocs.yml | 2 + opendpd/__init__.py | 2 +- opendpd/schemas/system.py | 43 +++ opendpd/server/app.py | 8 +- opendpd/server/routes.py | 11 +- opendpd/server/security.py | 48 ++- opendpd/services/server_status.py | 100 ++++++ opendpd/services/workspace.py | 2 +- opendpd/web/app.py | 53 ++- opendpd/web/gpu_agent.py | 24 +- opendpd/web/gpu_broker.py | 20 ++ opendpd/web/policy.py | 10 +- opendpd/web/runtime.py | 87 ++++- pics/studio-home.png | Bin 349177 -> 351169 bytes pics/studio-server-load.png | Bin 0 -> 111472 bytes pyproject.toml | 14 +- scripts/audit_requirements.py | 18 + tests/integration/test_server_status.py | 174 ++++++++++ tests/integration/test_studio_api.py | 10 + tests/unit/test_resource_telemetry.py | 48 +++ tests/unit/test_security.py | 17 +- 60 files changed, 2065 insertions(+), 78 deletions(-) create mode 100644 docs/guides/server-load.md create mode 100644 docs/performance/studio-2.2.6.md create mode 100644 docs/performance/studio-2.2.6/gpu.json create mode 100644 docs/performance/studio-2.2.6/screenshots.json create mode 100644 docs/performance/studio-2.2.6/security-scan.json create mode 100644 docs/performance/studio-2.2.6/status-http.json create mode 100644 docs/releases/release-notes-2.2.6.md create mode 100644 docs/releases/security-review-2.2.6.md create mode 100644 frontend/src/components/WorkspaceExpiry.tsx create mode 100644 frontend/src/pages/ServerStatusPage.test.tsx create mode 100644 frontend/src/pages/ServerStatusPage.tsx create mode 100644 opendpd/schemas/system.py create mode 100644 opendpd/services/server_status.py create mode 100644 pics/studio-server-load.png create mode 100644 scripts/audit_requirements.py create mode 100644 tests/integration/test_server_status.py create mode 100644 tests/unit/test_resource_telemetry.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50d4c44..1742138 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,11 @@ jobs: uv pip install --python .venv -e ".[dev]" --torch-backend=auto - name: Check default dependencies and available compute run: uv run --no-project --python .venv python scripts/check_install.py + - name: Audit the resolved Python dependencies + if: matrix.os == 'ubuntu-latest' + run: | + uv run --no-project --python .venv python scripts/audit_requirements.py > /tmp/opendpd-audit.txt + uvx pip-audit --strict --no-deps --disable-pip -r /tmp/opendpd-audit.txt - name: Real browser fallback remains reachable run: uv run --no-project --python .venv python -m pytest tests/integration/test_launcher_browser_fallback.py -q @@ -143,6 +148,9 @@ jobs: - name: Install run: npm ci + - name: Audit frontend dependencies + run: npm audit --audit-level=moderate + - name: Generated API types match docs/contracts/openapi.json run: npm run types:check diff --git a/README.md b/README.md index 0345dd0..873aec9 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ OpenDPD is a PyTorch framework for power amplifier (PA) modeling and digital pre ## What's new -**OpenDPD 2.2.5** adds LaTeX equations with interactive PA parameter highlighting, shared or per-channel OFDMA settings, and **ILC + ILA DPD** with a separate waveform-specific Ideal benchmark. Next-step controls sit above settings, and metric calculation methods have clearer names. +**OpenDPD 2.2.6** adds a **Server load** tab with active sessions, queued/running jobs, CPU, memory and GPU utilization. The top bar shows one exact workspace cleanup timestamp. Stronger request, filesystem and resource limits protect shared compute. **Signal Generator → PA Library → PA training → DPD training/testing.** Generate a PA input waveform, simulate its output with one of nine Virtual PAs, or upload existing input/output CSV data. Results label synthetic, surrogate and measured evidence and show separate PSD charts at each signal-chain position. -[2.2.5 release notes](https://lab-emi.github.io/OpenDPD/releases/release-notes-2.2.5/) · [Signal Generator](https://lab-emi.github.io/OpenDPD/guides/signal-generator/) · [ILC guide](https://lab-emi.github.io/OpenDPD/guides/ilc-dpd/). Standard presets are engineering stimuli, not conformance waveforms; Wi-Fi 8 is experimental. +[2.2.6 release notes](https://lab-emi.github.io/OpenDPD/releases/release-notes-2.2.6/) · [Signal Generator](https://lab-emi.github.io/OpenDPD/guides/signal-generator/) · [ILC guide](https://lab-emi.github.io/OpenDPD/guides/ilc-dpd/). Standard presets are engineering stimuli, not conformance waveforms; Wi-Fi 8 is experimental. [Feature history](docs/whats-new.md) · [Verified platform status](docs/releases/support-matrix.md) @@ -70,7 +70,7 @@ powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | ie mkdir opendpd-lab cd opendpd-lab uv venv --python 3.12 -uv pip install --python .venv "opendpd==2.2.5" --torch-backend=auto +uv pip install --python .venv "opendpd==2.2.6" --torch-backend=auto uv run --no-project --python .venv opendpd gui ``` diff --git a/deployment/web/GpuContainerfile b/deployment/web/GpuContainerfile index 5496b88..12ed9d3 100644 --- a/deployment/web/GpuContainerfile +++ b/deployment/web/GpuContainerfile @@ -1,6 +1,12 @@ -FROM docker.io/pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime@sha256:417bd75df6365104c283ea4c1651fb3530d9eb5a4c2fafa51943cff2a94e6385 +FROM docker.io/pytorch/pytorch:2.14.0-cuda13.2-cudnn9-runtime@sha256:773f5f0122b8d6413ff1e47559d98a3a22896434822e2d7752ba8d5b85a363d7 WORKDIR /opt/opendpd -RUN pip install --no-cache-dir numpy scipy pandas matplotlib pillow tqdm rich 'pydantic>=2.5,<3' psutil +# Apply Ubuntu security updates and remove build headers from the runtime. +# Containers use the host kernel; linux-libc-dev is an unused header package. +RUN apt-get update && apt-get upgrade -y && apt-get purge -y linux-libc-dev python3-pip python3-wheel binutils-common libctf-nobfd0 libsframe1 libc-dev-bin && rm -rf /var/lib/apt/lists/* +RUN uv pip install --system --break-system-packages --no-cache --upgrade 'setuptools>=83' +RUN uv venv --system-site-packages --python /usr/bin/python /opt/opendpd-runtime +ENV PATH="/opt/opendpd-runtime/bin:${PATH}" +RUN uv pip install --python /opt/opendpd-runtime --no-cache --upgrade pip setuptools wheel numpy scipy pandas matplotlib pillow tqdm rich 'pydantic>=2.5,<3' psutil COPY pyproject.toml README.md arguments.py project.py models.py main.py ./ COPY opendpd ./opendpd COPY backbones ./backbones @@ -11,5 +17,8 @@ COPY quant ./quant COPY datasets ./datasets COPY dataset ./dataset RUN pip install --no-cache-dir --no-deps --no-build-isolation . && chmod -R a-w /opt/opendpd +# No package installation is needed in an offline inference/training worker. +# Remove installers and their bundled dependencies after building the app. +RUN uv pip uninstall --python /opt/opendpd-runtime pip && uv pip uninstall --system --break-system-packages pip uv USER 65532:65532 ENTRYPOINT ["python", "-m", "opendpd.web.gpu_container"] diff --git a/deployment/web/opendpd-gpu.service b/deployment/web/opendpd-gpu.service index b91a7fa..cc38e29 100644 --- a/deployment/web/opendpd-gpu.service +++ b/deployment/web/opendpd-gpu.service @@ -19,6 +19,8 @@ UMask=0077 LimitCORE=0 MemoryMax=10G MemorySwapMax=0 +CPUQuota=400% +TasksMax=512 # Worker copies exist only in this service's mount namespace. The container's # read-only image and log-driver=none keep user files out of OCI storage. TemporaryFileSystem=/run/opendpd-gpu:rw,size=2G,mode=0700,nodev,nosuid,noexec diff --git a/docs/architecture/api.md b/docs/architecture/api.md index ac005ae..b39b475 100644 --- a/docs/architecture/api.md +++ b/docs/architecture/api.md @@ -8,7 +8,7 @@ file (S05). ## Sessions 1. The launcher prints `http://127.0.0.1:/bootstrap?token=…`. Opening it - exchanges the one-time token for an `HttpOnly; SameSite=Strict` cookie and + exchanges the launcher secret for an `HttpOnly; SameSite=Strict` cookie and redirects to `/` so the token never stays in the address bar. 2. `GET /api/v1/session` returns `{authenticated, csrf_token}`. Every state-changing request (POST) must send `X-OpenDPD-CSRF: `. diff --git a/docs/architecture/public-studio.md b/docs/architecture/public-studio.md index 67ef3f4..2e7568a 100644 --- a/docs/architecture/public-studio.md +++ b/docs/architecture/public-studio.md @@ -50,7 +50,7 @@ it terminates remaining requests and workers, then discards its private tmpfs. The stop timeout is 20 seconds with SIGKILL as the final fallback. This leaves margin before any session data reaches 24 hours. Sessions do not get a fresh 24 hours when they are accessed or when new files are generated. Late visitors -therefore have a shorter session; the UI displays their exact expiry. +therefore have a shorter session; the persistent top bar displays the exact scheduled cleanup start in UTC, with local time in its tooltip. Active requests can delay individual deletions until the next sweep; the independent reset bounds that delay. Refreshing a browser or reconnecting within the same tab resumes the existing session, events and saved plots. Restarting the API or VM discards every session; @@ -118,10 +118,11 @@ Cloudflare response header for `/studio/` because a meta CSP cannot enforce it. | HTTP requests | 120/minute per session, 600/minute per IP; preflight has a separate 600/minute bucket; 8 concurrent globally, 3/session | | JSON request body | 64 KiB, 10-second receive deadline; no multipart | | Jobs | 8/session, 12/IP/day, 60/day globally; 2 pending/session | +| Heavy API work | 2 concurrent analyses/generations/exports globally; one workspace mutation at a time; cancellation is exempt | | Concurrent training/inference | 1 globally across every session, oldest queued experiment first | | Job runtime | 30 minutes, followed by cancellation and forced termination | | Public model/training parameters | bounded layers, widths, batches, frames, epochs and threads | -| Storage | 256 MiB/session checked every 15 seconds; **2 GiB hard limit globally** | +| Storage | 256 MiB/session checked every 15 seconds; **2 GiB hard limit globally**; heavy writers reserve capacity before receiving data, retaining 64 MiB free headroom | | Files/processes | 64 MiB/file, 256 tasks, 4 CPU equivalents, 6 GiB API cgroup RAM | IPv6 addresses share a /64 rate-limit bucket. IP quotas use an ephemeral HMAC @@ -359,3 +360,11 @@ termination and empty host temporary storage before publishing the frontend. Deploy the reviewed 2.2.4 source consistently to the isolated API, the private GPU agent and its pinned container image, and the `opendpd-site` Pages build. Drain queued/running experiments before replacing the API; retain prior source and image pins for rollback. Restarting the API expires existing temporary sessions. Verify reported versions before the public generation → Virtual PA → paired dataset → CUDA PA/DPD training/testing journey. Generator records contain input only. The `/pa-library/` routes perform bounded mathematical simulation inside the owning temporary workspace, preserve synthetic provenance and require explicit paired-dataset creation. New PSD `signal_node` fields are display metadata; tenant boundaries, quotas, expiry and numerical metric protocols remain unchanged. Publication-figure and optional GitHub dataset-contribution capabilities keep their existing operator policy. + +## 2.2.6 resource status and security update + +Authenticated `GET /api/v1/system/status` exposes aggregate session/job counts and cached resource samples. A single background sampler per API process reads CPU/RAM every five seconds; the private GPU agent samples host CPU/RAM and GPU 0 and sends validated telemetry to `POST /_gpu/resources`. That endpoint requires the existing private bridge credential and refuses browser origins. Status reads neither run GPU commands nor disclose process names, IPs, workspace identifiers or dataset details. Job counts are cached across viewers; collection errors produce unknown counts instead of zero. Samples older than 20 seconds are stale. + +Workspace POST/PUT operations serialize within a tenant; cancellation remains available. Heavy API work has a separate global admission limit, and temporary-space reservations prevent concurrent writers from spending the same free capacity. Cleanup scans run off the event loop and tolerate files atomically replaced by legitimate writers. Failed cleanup still disables new sessions. + +The runtime image pins PyTorch 2.14/CUDA 13.2 by digest, applies Ubuntu updates and removes unused build headers and package installers. Keep the existing network-free, non-root, read-only worker limits. Deploy reviewed source to the API, agent, image and Pages together, with no queued/running jobs at the swap and rollback copies retained. See [load semantics](../guides/server-load.md) and the [2.2.6 security review](../releases/security-review-2.2.6.md). diff --git a/docs/contracts/openapi.json b/docs/contracts/openapi.json index e303428..01ccea9 100644 --- a/docs/contracts/openapi.json +++ b/docs/contracts/openapi.json @@ -5511,6 +5511,52 @@ "title": "GoldenCase", "type": "object" }, + "GpuLoad": { + "additionalProperties": false, + "properties": { + "memory_total_bytes": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "maximum": 1.152921504606847e+18, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Memory Total Bytes" + }, + "memory_used_bytes": { + "anyOf": [ + { + "maximum": 1.152921504606847e+18, + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Memory Used Bytes" + }, + "utilization_percent": { + "anyOf": [ + { + "maximum": 100.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Utilization Percent" + } + }, + "title": "GpuLoad", + "type": "object" + }, "HTTPValidationError": { "properties": { "detail": { @@ -6897,6 +6943,87 @@ "title": "LogPage", "type": "object" }, + "MachineLoad": { + "additionalProperties": false, + "properties": { + "cpu_percent": { + "anyOf": [ + { + "maximum": 100.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Cpu Percent" + }, + "gpu": { + "anyOf": [ + { + "$ref": "#/components/schemas/GpuLoad" + }, + { + "type": "null" + } + ] + }, + "memory_percent": { + "anyOf": [ + { + "maximum": 100.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Memory Percent" + }, + "memory_total_bytes": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "maximum": 1.152921504606847e+18, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Memory Total Bytes" + }, + "memory_used_bytes": { + "anyOf": [ + { + "maximum": 1.152921504606847e+18, + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Memory Used Bytes" + }, + "sampled_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Sampled At" + } + }, + "title": "MachineLoad", + "type": "object" + }, "ManifestUpdate": { "properties": { "display_name": { @@ -9658,6 +9785,33 @@ "title": "ResourceEstimate", "type": "object" }, + "ResourceStatus": { + "additionalProperties": false, + "properties": { + "age_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Age Seconds" + }, + "load": { + "$ref": "#/components/schemas/MachineLoad" + }, + "stale": { + "default": true, + "title": "Stale", + "type": "boolean" + } + }, + "title": "ResourceStatus", + "type": "object" + }, "ReviewBand": { "additionalProperties": false, "properties": { @@ -10319,6 +10473,122 @@ "title": "ScalingInfo", "type": "object" }, + "ServerStatus": { + "additionalProperties": false, + "properties": { + "active_sessions": { + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Active Sessions" + }, + "active_window_seconds": { + "default": 300, + "title": "Active Window Seconds", + "type": "integer" + }, + "api": { + "$ref": "#/components/schemas/ResourceStatus" + }, + "compute": { + "anyOf": [ + { + "$ref": "#/components/schemas/ResourceStatus" + }, + { + "type": "null" + } + ] + }, + "mode": { + "enum": [ + "web", + "local" + ], + "title": "Mode", + "type": "string" + }, + "parallel_capacity": { + "anyOf": [ + { + "minimum": 1.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Parallel Capacity" + }, + "queued_jobs": { + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Queued Jobs" + }, + "refresh_seconds": { + "default": 5, + "title": "Refresh Seconds", + "type": "integer" + }, + "running_jobs": { + "anyOf": [ + { + "minimum": 0.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Running Jobs" + }, + "sampled_at": { + "format": "date-time", + "title": "Sampled At", + "type": "string" + }, + "workspace_capacity": { + "anyOf": [ + { + "minimum": 1.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Workspace Capacity" + }, + "workspaces": { + "minimum": 0.0, + "title": "Workspaces", + "type": "integer" + } + }, + "required": [ + "mode", + "sampled_at", + "workspaces", + "running_jobs", + "queued_jobs", + "api" + ], + "title": "ServerStatus", + "type": "object" + }, "SessionCapture": { "additionalProperties": false, "properties": { @@ -10416,7 +10686,7 @@ "title": "Csrf Token" }, "version": { - "default": "2.2.5", + "default": "2.2.6", "title": "Version", "type": "string" } @@ -12606,7 +12876,7 @@ }, "info": { "title": "OpenDPD Studio API", - "version": "2.2.5" + "version": "2.2.6" }, "openapi": "3.1.0", "paths": { @@ -17200,6 +17470,27 @@ "system" ] } + }, + "/api/v1/system/status": { + "get": { + "operationId": "server_status_api_v1_system_status_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServerStatus" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Server Status", + "tags": [ + "system" + ] + } } } } diff --git a/docs/documentation.md b/docs/documentation.md index 3542b27..5e5dcf0 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -63,3 +63,7 @@ The README's `brand` snippet uses GitHub-compatible `