From bdd66d64ddaf4f5a7e2f4c218e3c6ae8e3781746 Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Wed, 22 Jul 2026 17:16:11 +0800 Subject: [PATCH 1/6] fix(orion): return all tasks for a CL instead of Multiple tasks GET /v2/task/{cl} now returns Vec (empty when none), so repeated pushes no longer break Checks. Also document regenerating gitmono.json / generated.ts via moon/script/gen-client, and bump tokio-stream to 0.1.19. --- .github/workflows/mono-engine-deploy.yml | 2 +- .github/workflows/orion-server-deploy.yml | 2 +- .github/workflows/web-deploy.yml | 2 +- .github/workflows/web-sync-server-deploy.yml | 2 +- Cargo.lock | 4 +-- Cargo.toml | 2 +- docs/architecture.md | 2 +- docs/development.md | 4 +++ moon/api/README.md | 36 +++++++++++++++++++ moon/apps/web/hooks/SSE/useGetClTask.ts | 6 +--- moon/packages/types/generated.ts | 2 +- orion-server/Dockerfile | 1 + orion-server/src/api.rs | 6 ++-- .../src/repository/orion_tasks_repo.rs | 3 +- orion-server/src/service/api_v2_service.rs | 8 ++--- 15 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 moon/api/README.md diff --git a/.github/workflows/mono-engine-deploy.yml b/.github/workflows/mono-engine-deploy.yml index 6824d1d0a..923ab5c9a 100644 --- a/.github/workflows/mono-engine-deploy.yml +++ b/.github/workflows/mono-engine-deploy.yml @@ -128,7 +128,7 @@ jobs: deploy-aws: needs: manifest - if: ${{ github.repository == 'gitmono-dev/mega' }} + if: false # disabled runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/orion-server-deploy.yml b/.github/workflows/orion-server-deploy.yml index d1932920c..c9f2bbb2e 100644 --- a/.github/workflows/orion-server-deploy.yml +++ b/.github/workflows/orion-server-deploy.yml @@ -118,7 +118,7 @@ jobs: deploy-aws: needs: manifest - if: ${{ github.repository == 'gitmono-dev/mega' }} + if: false # disabled runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/web-deploy.yml b/.github/workflows/web-deploy.yml index b9f695d57..52c398864 100644 --- a/.github/workflows/web-deploy.yml +++ b/.github/workflows/web-deploy.yml @@ -128,7 +128,7 @@ jobs: deploy-aws: needs: manifest - if: ${{ github.repository == 'gitmono-dev/mega' }} + if: false # disabled runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/web-sync-server-deploy.yml b/.github/workflows/web-sync-server-deploy.yml index 73fcca023..c4c93dad9 100644 --- a/.github/workflows/web-sync-server-deploy.yml +++ b/.github/workflows/web-sync-server-deploy.yml @@ -68,7 +68,7 @@ jobs: deploy-aws: needs: build-and-push - if: ${{ github.repository == 'gitmono-dev/mega' }} + if: false # disabled runs-on: ubuntu-latest strategy: matrix: diff --git a/Cargo.lock b/Cargo.lock index 651b79d51..58007b01e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10027,9 +10027,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index c8b2eef95..87c8b8301 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ clap = "4.6.4" #==== tokio = "1.53.1" -tokio-stream = "0.1.18" +tokio-stream = "0.1.19" tokio-util = "0.7.19" async-trait = "0.1.89" async-stream = "0.3.6" diff --git a/docs/architecture.md b/docs/architecture.md index ede036d96..e63ad898d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ When the `mono` HTTP server is running (default port `8000`): | Swagger UI | `http://localhost:8000/swagger-ui` | | OpenAPI JSON | `http://localhost:8000/api/openapi.json` | -REST handlers are defined in `mono/src/api/` with `utoipa` annotations. Do not maintain a separate hand-written API catalog. +REST handlers are defined in `mono/src/api/` with `utoipa` annotations. Do not maintain a separate hand-written API catalog. Frontend typed clients are generated from those specs — do not hand-edit `moon/api/gen/gitmono.json` or `moon/packages/types/generated.ts`; see [moon/api/README.md](../moon/api/README.md). ### Git LFS diff --git a/docs/development.md b/docs/development.md index 0e0d27dfd..4614aca9f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,6 +38,10 @@ Database migrations apply automatically on first `Storage::new` when the `migrat Swagger UI: `http://localhost:8000/swagger-ui` (default HTTP port). +## Frontend OpenAPI client types + +Do **not** hand-edit `moon/api/gen/gitmono.json` or `moon/packages/types/generated.ts`. Export OpenAPI from the running services and regenerate with `moon/script/gen-client`. See [moon/api/README.md](../moon/api/README.md). + ## Post-start initialization (optional) To seed Buckal bundles and third-party imports via API: diff --git a/moon/api/README.md b/moon/api/README.md new file mode 100644 index 000000000..202229773 --- /dev/null +++ b/moon/api/README.md @@ -0,0 +1,36 @@ +# moon/api — OpenAPI specs and typed client + +Frontend API types live in `@gitmono/types` (`packages/types/generated.ts`). They are **generated** from the OpenAPI JSON files under `api/gen/`. Do not treat those outputs as hand-edited sources of truth. + +## Do not edit these by hand + +| File | Why | +|------|-----| +| [`gen/gitmono.json`](gen/gitmono.json) | Exported from the **mono** OpenAPI (`utoipa`). Refresh from a running mono server. | +| [`../packages/types/generated.ts`](../packages/types/generated.ts) | Produced by [`script/gen-client`](../script/gen-client) via `swagger-typescript-api`. | + +Also avoid hand-editing other checked-in OpenAPI dumps that feed the same pipeline (for example [`gen/orion.json`](gen/orion.json) from **orion-server**). Change the Rust `utoipa` annotations instead, then re-export and regenerate. + +## Regenerate workflow + +From the `moon/` directory: + +```bash +# 1) Refresh mono OpenAPI (mono HTTP must be running; default port 8000) +curl -sS http://localhost:8000/api/openapi.json -o api/gen/gitmono.json + +# Optional: refresh orion OpenAPI when orion-server routes/DTOs changed (default port 8004) +curl -sS http://localhost:8004/api-doc/openapi.json -o api/gen/orion.json + +# 2) Merge api/gen/*.json → api/gen/merged_swagger.json and write packages/types/generated.ts +./script/gen-client +``` + +`script/gen-client` runs [`merge-swagger.js`](merge-swagger.js) (merges every `api/gen/*.json` except `merged_swagger.json` / `openapi_schema.json`), then regenerates `packages/types/generated.ts`. + +## Related endpoints + +| Service | Swagger UI | OpenAPI JSON | +|---------|------------|--------------| +| mono | `http://localhost:8000/swagger-ui` | `http://localhost:8000/api/openapi.json` | +| orion-server | `http://localhost:8004/swagger-ui` | `http://localhost:8004/api-doc/openapi.json` | diff --git a/moon/apps/web/hooks/SSE/useGetClTask.ts b/moon/apps/web/hooks/SSE/useGetClTask.ts index 67e001760..99a2d0113 100644 --- a/moon/apps/web/hooks/SSE/useGetClTask.ts +++ b/moon/apps/web/hooks/SSE/useGetClTask.ts @@ -54,11 +54,7 @@ export function useGetClTask(cl: string, params?: RequestParams) { return hasActive ? 5000 : false }, queryFn: async () => { - const resp = await orionApiClient.task.getTaskByClV2().request(cl, params) - - if (!resp) return [] - - const tasks = Array.isArray(resp) ? resp : [resp] + const tasks = (await orionApiClient.task.getTaskByClV2().request(cl, params)) ?? [] const hydratedTasks = await Promise.all( tasks.map(async (task): Promise => { diff --git a/moon/packages/types/generated.ts b/moon/packages/types/generated.ts index ec620b396..6f74214e2 100644 --- a/moon/packages/types/generated.ts +++ b/moon/packages/types/generated.ts @@ -7927,7 +7927,7 @@ export type GetTargetsByTaskIdV2Error = MessageResponse export type PostTaskV2Data = any -export type GetTaskByClV2Data = OrionTaskDTO +export type GetTaskByClV2Data = OrionTaskDTO[] export type GetTaskByClV2Error = MessageResponse diff --git a/orion-server/Dockerfile b/orion-server/Dockerfile index eeb58544d..da6289dec 100644 --- a/orion-server/Dockerfile +++ b/orion-server/Dockerfile @@ -41,6 +41,7 @@ COPY orion/buck/Cargo.toml orion/buck/ COPY orion-scheduler/Cargo.toml orion-scheduler/ COPY orion-server/Cargo.toml orion-server/ COPY clients/orion-client/Cargo.toml clients/orion-client/ +COPY clients/orion-scheduler-client/Cargo.toml clients/orion-scheduler-client/ COPY saturn/Cargo.toml saturn/ COPY vault/Cargo.toml vault/ diff --git a/orion-server/src/api.rs b/orion-server/src/api.rs index 593fbc011..76909a8e1 100644 --- a/orion-server/src/api.rs +++ b/orion-server/src/api.rs @@ -292,16 +292,14 @@ pub async fn build_retry_handler( tag = "Task", params(("cl" = String, Path, description = "Change List")), responses( - (status = 200, description = "Get task successfully", body = OrionTaskDTO), - (status = 400, description = "Multiple tasks", body = MessageResponse), - (status = 404, description = "Not found task", body = MessageResponse), + (status = 200, description = "Get tasks by CL successfully", body = Vec), (status = 500, description = "Database error", body = MessageResponse), ) )] pub async fn task_get_handler( State(state): State, Path(cl): Path, -) -> Result, (StatusCode, Json)> { +) -> Result>, (StatusCode, Json)> { api_v2_service::task_get(&state, &cl).await } diff --git a/orion-server/src/repository/orion_tasks_repo.rs b/orion-server/src/repository/orion_tasks_repo.rs index fd17ac2f5..085e1b7c8 100644 --- a/orion-server/src/repository/orion_tasks_repo.rs +++ b/orion-server/src/repository/orion_tasks_repo.rs @@ -1,7 +1,7 @@ use api_model::buck2::{status::Status, types::ProjectRelativePath}; use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, - QueryFilter as _, QuerySelect as _, + QueryFilter as _, QueryOrder as _, QuerySelect as _, }; use serde_json::to_value; use uuid::Uuid; @@ -39,6 +39,7 @@ impl OrionTasksRepo { ) -> Result, DbErr> { callisto::orion_tasks::Entity::find() .filter(callisto::orion_tasks::Column::Cl.eq(cl)) + .order_by_asc(callisto::orion_tasks::Column::CreatedAt) .all(conn) .await } diff --git a/orion-server/src/service/api_v2_service.rs b/orion-server/src/service/api_v2_service.rs index fe5ad61dc..e84018629 100644 --- a/orion-server/src/service/api_v2_service.rs +++ b/orion-server/src/service/api_v2_service.rs @@ -215,7 +215,7 @@ async fn task_exists_by_id( pub async fn task_get( state: &AppState, cl: &str, -) -> Result, JsonValueErrorResponse> { +) -> Result>, JsonValueErrorResponse> { let tasks = OrionTasksRepo::find_by_cl(&state.conn, cl) .await .map_err(|e| { @@ -223,11 +223,7 @@ pub async fn task_get( value_error(StatusCode::INTERNAL_SERVER_ERROR, "Database error") })?; - match tasks.len() { - 0 => Err(value_error(StatusCode::NOT_FOUND, "Not found task")), - 1 => Ok(Json(OrionTaskDTO::from(&tasks[0]))), - _ => Err(value_error(StatusCode::BAD_REQUEST, "Multiple tasks")), - } + Ok(Json(tasks.iter().map(OrionTaskDTO::from).collect())) } pub async fn build_event_get( From fe80d5ff2eaa3a0e77105bda3858461743b4d15f Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Thu, 23 Jul 2026 02:09:07 +0000 Subject: [PATCH 2/6] fix(orion): reclaim Antares overlays and reject builds under disk pressure Prevent long-lived VMs from filling the guest root FS (which drops WebSocket heartbeats and marks workers Lost) by deleting upper/cl after umount, pruning orphans on runner start, and refusing TaskBuild when usage exceeds the reject threshold. Also raise default image_disk_gb to 50. --- orion-scheduler/DESIGN.md | 20 ++- orion-scheduler/README.md | 2 +- orion-scheduler/TESTING.md | 5 +- orion-scheduler/src/config.rs | 4 +- orion-scheduler/src/orion_deployer.rs | 21 ++++ orion-scheduler/target_config.json.template | 2 +- orion/runner-config/cleanup.sh | 98 ++++++++++++++- orion/src/antares.rs | 132 ++++++++++++++++++++ orion/src/buck_controller.rs | 22 +++- orion/src/disk.rs | 101 +++++++++++++++ orion/src/lib.rs | 1 + orion/src/main.rs | 1 + orion/src/ws.rs | 37 ++++++ 13 files changed, 433 insertions(+), 13 deletions(-) create mode 100644 orion/src/disk.rs diff --git a/orion-scheduler/DESIGN.md b/orion-scheduler/DESIGN.md index 9762ca4f4..4b0baa7e9 100644 --- a/orion-scheduler/DESIGN.md +++ b/orion-scheduler/DESIGN.md @@ -364,7 +364,7 @@ flowchart TD "default_image": { "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", - "image_disk_gb": 30, + "image_disk_gb": 50, "image_cpus": 8, "image_memory_mb": 16000 } @@ -383,7 +383,8 @@ flowchart TD | `default_image` | object | 见模板 | 默认 VM 镜像五参数；webhook 未传 `image_*` 时使用 | | `default_image.image_path` | string | — | 本地 qcow2 路径 | | `default_image.image_digest` | string | — | SHA256 校验和 | -| `default_image.image_disk_gb` | u32 | 30 | 磁盘 GB | +| `default_image.image_disk_gb` | u32 | 50 | 磁盘 GB（长生命周期 VM 建议 ≥50；已部署的 `/etc/.../target_config.json` 需人工改） | + | `default_image.image_cpus` | u32 | 8 | vCPU 数 | | `default_image.image_memory_mb` | u32 | 16000 | 内存 MB | @@ -531,10 +532,23 @@ tracing::error!("[orion-deploy] Orion start failed: {}", error); | `.env.prod` | `/home/orion/orion-runner/.env` | Orion 运行时的环境变量，包括 `SERVER_WS`（WebSocket 服务器地址）、`BUCK_PROJECT_ROOT`（Buck 项目路径）等 | | `run.sh` | `/home/orion/orion-runner/run.sh` | Orion 启动脚本，加载 `.env` 环境变量，执行 `preflight.sh` 前置检查，然后启动 orion 进程 | | `preflight.sh` | `/home/orion/orion-runner/preflight.sh` | 前置检查脚本，验证 FUSE 能力和设备访问权限，确保环境满足 Orion 运行要求 | -| `cleanup.sh` | `/home/orion/orion-runner/cleanup.sh` | 清理脚本，在 Orion 启动前杀死旧进程并卸载 FUSE 文件系统 | +| `cleanup.sh` | `/home/orion/orion-runner/cleanup.sh` | 启动前杀旧进程、卸 FUSE、清理孤儿 Antares upper/cl、轮转 orion.log、磁盘水位告警 | | `orion-runner.service` | `/etc/systemd/system/orion-runner.service` | systemd 服务单元定义，负责配置 Orion 服务的启动参数、运行环境、权限和能力、停止超时等 | | `orion` | `/home/orion/orion-runner/orion` | Orion 主程序二进制文件，Buck 构建任务的 WebSocket 客户端，接收并执行构建任务 | +### 6.1 Guest 磁盘水位（防打满） + +长生命周期 VM 上 Antares `upper/`/`cl/` 与 `orion.log` 会占满默认盘，导致 WS 心跳失败、worker 被标 Lost。 + +| 机制 | 行为 | +|------|------| +| 构建结束 umount | 默认删除对应 `upper_dir` / `cl_dir`（`ORION_RETAIN_ANTARES_MOUNTS=1` 时保留） | +| `cleanup.sh` | 无活跃挂载时 prune 孤儿 overlay；`orion.log` > 200MB 轮转 | +| Orion 接 `TaskBuild` | `df /` ≥ `ORION_DISK_REJECT_PCT`（默认 92）则拒绝任务并尝试 prune；心跳继续 | +| 默认盘 | `image_disk_gb` 默认 **50**（已有 `/etc/orion-scheduler/target_config.json` 需手动改） | + +排障：`GET /scorpio/status?domain=` 响应中的 `disk.df_root` / `disk.du`。 + ## 7. 部署与运行 ### 7.1 资源回收 diff --git a/orion-scheduler/README.md b/orion-scheduler/README.md index 1cb09b4b5..3c3758718 100644 --- a/orion-scheduler/README.md +++ b/orion-scheduler/README.md @@ -40,7 +40,7 @@ flowchart LR "default_image": { "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", - "image_disk_gb": 30, + "image_disk_gb": 50, "image_cpus": 8, "image_memory_mb": 16000 } diff --git a/orion-scheduler/TESTING.md b/orion-scheduler/TESTING.md index 9c61436b6..faeb5c681 100644 --- a/orion-scheduler/TESTING.md +++ b/orion-scheduler/TESTING.md @@ -33,7 +33,7 @@ curl -i -X POST http://localhost:8080/webhook \ "scorpio_lfs_url": "https://git.gitmega.com", "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", - "image_disk_gb": 30, + "image_disk_gb": 50, "image_cpus": 8, "image_memory_mb": 16000 }' @@ -248,6 +248,7 @@ aws s3 cp ~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2 \ | QEMU 桥接失败 | `/etc/qemu/bridge.conf` 是否 `allow qlbr0` | | VM 启动超时 | cloud-init、SSH 是否可达 | | Orion 启动失败 | `curl -N '.../logs/orion/stream?domain=...'` | -| Scorpio 挂载问题 | `curl '.../scorpio/status?domain=...'` | +| Scorpio 挂载问题 | `curl '.../scorpio/status?domain=...'`（看 `disk.df_root` / `disk.du`） | +| Guest 磁盘打满 / worker Lost | VM 内 `df -h /`；清 `/data/scorpio/antares/{upper,cl}` 或 `systemctl restart orion-runner`；新盘建议 `image_disk_gb: 50` | | 重启后状态丢了 | 内存 map；磁盘 qemu 靠启动 reap；重新 POST webhook | | 进 VM 调试 | [SSH 进入 VM](#ssh-进入-vm) | diff --git a/orion-scheduler/src/config.rs b/orion-scheduler/src/config.rs index 0768751e9..86996b614 100644 --- a/orion-scheduler/src/config.rs +++ b/orion-scheduler/src/config.rs @@ -32,7 +32,7 @@ impl Default for DefaultImageConfig { .to_string(), image_digest: "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5" .to_string(), - image_disk_gb: 30, + image_disk_gb: 50, image_cpus: 8, image_memory_mb: 16000, } @@ -232,7 +232,7 @@ mod tests { #[test] fn default_image_has_expected_defaults() { let d = DefaultImageConfig::default(); - assert_eq!(d.image_disk_gb, 30); + assert_eq!(d.image_disk_gb, 50); assert_eq!(d.image_cpus, 8); assert_eq!(d.image_memory_mb, 16000); assert!(d.image_digest.starts_with("sha256:")); diff --git a/orion-scheduler/src/orion_deployer.rs b/orion-scheduler/src/orion_deployer.rs index 4d947f462..c94ca4ede 100644 --- a/orion-scheduler/src/orion_deployer.rs +++ b/orion-scheduler/src/orion_deployer.rs @@ -437,9 +437,30 @@ pub async fn get_scorpio_status( let network_test = machine.exec("curl -sI --connect-timeout 5 https://git.gitmega.com 2>&1 | head -5 || echo 'Connection failed'").await?; let network_info = String::from_utf8_lossy(&network_test.stdout); + let df_output = machine + .exec("df -h / 2>/dev/null || echo 'df failed'") + .await?; + let df_info = String::from_utf8_lossy(&df_output.stdout) + .trim() + .to_string(); + + let du_output = machine + .exec( + "du -sh /data/scorpio/store /data/scorpio/antares/upper /data/scorpio/antares/cl \ + /data/scorpio/antares/mnt /home/orion/orion-runner/log 2>/dev/null || true", + ) + .await?; + let du_info = String::from_utf8_lossy(&du_output.stdout) + .trim() + .to_string(); + let status = serde_json::json!({ "status": "ok", "directories": results, + "disk": { + "df_root": df_info, + "du": du_info + }, "mounts": mount_info, "orion_process": orion_info, "scorpio_process": scorpio_info, diff --git a/orion-scheduler/target_config.json.template b/orion-scheduler/target_config.json.template index ae1b306d0..b17532d94 100644 --- a/orion-scheduler/target_config.json.template +++ b/orion-scheduler/target_config.json.template @@ -7,7 +7,7 @@ "default_image": { "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", - "image_disk_gb": 30, + "image_disk_gb": 50, "image_cpus": 8, "image_memory_mb": 16000 } diff --git a/orion/runner-config/cleanup.sh b/orion/runner-config/cleanup.sh index 86c762d26..3768cdf8e 100755 --- a/orion/runner-config/cleanup.sh +++ b/orion/runner-config/cleanup.sh @@ -5,12 +5,18 @@ # # 说明： # - 在服务启动前执行，清理旧进程和 FUSE 挂载 -# - 即使失败也不会阻止服务启动 +# - 清理 Antares 孤儿 upper/cl（无活跃挂载时） +# - 轮转过大的 orion.log +# - 即使失败也不会阻止服务启动（始终 exit 0） # ============================================================================== set +e # 允许命令失败，不中断脚本 SCORPIO_TOML="${SCORPIO_CONFIG:-/home/orion/orion-runner/scorpio.toml}" +ORION_LOG="${ORION_LOG_PATH:-/home/orion/orion-runner/log/orion.log}" +ORION_LOG_MAX_BYTES="${ORION_LOG_MAX_BYTES:-209715200}" # 200 MiB +DISK_WARN_PCT="${ORION_DISK_WARN_PCT:-90}" +DISK_CRIT_PCT="${ORION_DISK_CRIT_PCT:-95}" if [ ! -f "./config.toml" ]; then echo "==> [清理] 未发现 ./config.toml，正在创建..." @@ -26,6 +32,10 @@ read_scorpio_path() { fi } +disk_used_pct() { + df -P / 2>/dev/null | awk 'NR==2 { gsub(/%/,"",$5); print $5 }' +} + echo "==> [清理] 停止旧进程..." if command -v buck2 &>/dev/null; then echo " - 正在执行 'buck2 killall'..." @@ -44,6 +54,12 @@ fi echo "==> [清理] 卸载 FUSE 挂载点..." MOUNT_DIR="$(read_scorpio_path "workspace" "${SCORPIO_TOML}")" MOUNT_DIR="${MOUNT_DIR:-/workspace/mount}" +UPPER_ROOT="$(read_scorpio_path "antares_upper_root" "${SCORPIO_TOML}")" +UPPER_ROOT="${UPPER_ROOT:-/data/scorpio/antares/upper}" +CL_ROOT="$(read_scorpio_path "antares_cl_root" "${SCORPIO_TOML}")" +CL_ROOT="${CL_ROOT:-/data/scorpio/antares/cl}" +MNT_ROOT="$(read_scorpio_path "antares_mount_root" "${SCORPIO_TOML}")" +MNT_ROOT="${MNT_ROOT:-/data/scorpio/antares/mnt}" echo " - 正在卸载 ${MOUNT_DIR}..." fusermount -uz "${MOUNT_DIR}" 2>/dev/null || true @@ -58,5 +74,85 @@ else echo " - ${MOUNT_DIR} 仍是挂载点，跳过删除" fi +echo "==> [清理] Antares mount_root 残留..." +if [ -d "${MNT_ROOT}" ]; then + for mp in "${MNT_ROOT}"/*; do + [ -e "$mp" ] || continue + if mountpoint -q "$mp" 2>/dev/null; then + echo " - fusermount -uz $mp" + fusermount -uz "$mp" 2>/dev/null || true + umount -lf "$mp" 2>/dev/null || true + fi + if ! mountpoint -q "$mp" 2>/dev/null; then + rm -rf "$mp" 2>/dev/null || true + fi + done +fi + +prune_overlay_root() { + local root="$1" + local kind="$2" + local removed=0 + if [ ! -d "$root" ]; then + return 0 + fi + # Only prune when no Antares mounts remain active. + local active=0 + if [ -d "${MNT_ROOT}" ]; then + for mp in "${MNT_ROOT}"/*; do + [ -e "$mp" ] || continue + if mountpoint -q "$mp" 2>/dev/null; then + active=1 + break + fi + done + fi + if [ "$active" -eq 1 ]; then + echo " - 跳过 ${kind} 清理：仍有活跃 Antares 挂载" + return 0 + fi + for d in "${root}"/*; do + [ -d "$d" ] || continue + rm -rf "$d" 2>/dev/null && removed=$((removed + 1)) + done + echo " - 已删除 ${removed} 个孤儿 ${kind} 目录 (${root})" +} + +echo "==> [清理] Antares 孤儿 overlay..." +prune_overlay_root "${UPPER_ROOT}" "upper" +prune_overlay_root "${CL_ROOT}" "cl" + +echo "==> [清理] orion.log 轮转..." +if [ -f "${ORION_LOG}" ]; then + size=$(wc -c < "${ORION_LOG}" 2>/dev/null || echo 0) + if [ "${size}" -gt "${ORION_LOG_MAX_BYTES}" ]; then + echo " - ${ORION_LOG} 为 ${size} bytes (> ${ORION_LOG_MAX_BYTES}), rotating" + mv -f "${ORION_LOG}" "${ORION_LOG}.1" 2>/dev/null || true + : > "${ORION_LOG}" 2>/dev/null || true + # Keep ownership usable for the runner user when present. + chown orion:orion "${ORION_LOG}" 2>/dev/null || true + else + echo " - ${ORION_LOG} size=${size} bytes (ok)" + fi +else + echo " - 无 orion.log" +fi + +USED_PCT="$(disk_used_pct)" +USED_PCT="${USED_PCT:-0}" +echo "==> [清理] 磁盘用量: ${USED_PCT}%" + +if [ "${USED_PCT}" -ge "${DISK_WARN_PCT}" ] 2>/dev/null; then + echo " - 用量 ≥ ${DISK_WARN_PCT}%，再次 prune orphan overlays" + prune_overlay_root "${UPPER_ROOT}" "upper" + prune_overlay_root "${CL_ROOT}" "cl" +fi + +USED_PCT="$(disk_used_pct)" +USED_PCT="${USED_PCT:-0}" +if [ "${USED_PCT}" -ge "${DISK_CRIT_PCT}" ] 2>/dev/null; then + echo "ERROR: disk usage still ${USED_PCT}% (≥ ${DISK_CRIT_PCT}%). Orion will refuse new builds until space is freed." +fi + echo "==> [清理] 完成" exit 0 # 总是返回成功 diff --git a/orion/src/antares.rs b/orion/src/antares.rs index e6cddf107..cbcd12fc2 100644 --- a/orion/src/antares.rs +++ b/orion/src/antares.rs @@ -935,6 +935,128 @@ Hint: set SCORPIO_CONFIG=/path/to/scorpio.toml or create /etc/scorpio/scorpio.to .map_err(Into::into) } + /// Best-effort delete of Antares COW layers left after FUSE unmount. + /// + /// Scorpio's `umount_job` only tears down the mount; `upper/` and `cl/` dirs + /// otherwise accumulate across builds on long-lived VMs and fill the disk. + pub async fn reclaim_overlay_dirs(config: &AntaresConfig) { + best_effort_remove_dir("upper", &config.upper_dir).await; + if let Some(cl_dir) = config.cl_dir.as_ref() { + best_effort_remove_dir("cl", cl_dir).await; + } + } + + async fn best_effort_remove_dir(kind: &str, path: &Path) { + if !path.exists() { + return; + } + match fs::remove_dir_all(path).await { + Ok(()) => tracing::info!( + kind = kind, + path = %path.display(), + "Reclaimed Antares overlay directory" + ), + Err(e) => tracing::warn!( + kind = kind, + path = %path.display(), + error = %e, + "Failed to reclaim Antares overlay directory" + ), + } + } + + /// When no Antares FUSE mounts are active, delete all `upper/` and `cl/` + /// children (they are orphans after umount). Returns `(upper_removed, cl_removed)`. + pub async fn prune_orphan_overlay_dirs() -> Result<(usize, usize), DynError> { + // Ensure scorpio config / paths are available. + let _ = get_manager().await?; + let paths = AntaresPaths::from_global_config(); + + // Best-effort: tear down leftover mountpoints under mount_root. + if let Ok(mut entries) = fs::read_dir(&paths.mount_root).await { + while let Ok(Some(entry)) = entries.next_entry().await { + let mp = entry.path(); + if is_mountpoint(&mp).await { + let _ = Command::new("fusermount") + .args(["-uz", &mp.to_string_lossy()]) + .status() + .await; + let _ = Command::new("umount") + .args(["-lf", &mp.to_string_lossy()]) + .status() + .await; + } + let _ = fs::remove_dir_all(&mp).await; + } + } + + if has_any_mountpoint(&paths.mount_root).await { + tracing::warn!( + "Skipping Antares overlay prune: active mountpoints still present under {}", + paths.mount_root.display() + ); + return Ok((0, 0)); + } + + let upper_removed = prune_all_child_dirs(&paths.upper_root, "upper").await; + let cl_removed = prune_all_child_dirs(&paths.cl_root, "cl").await; + Ok((upper_removed, cl_removed)) + } + + async fn prune_all_child_dirs(root: &Path, kind: &str) -> usize { + let Ok(mut entries) = fs::read_dir(root).await else { + return 0; + }; + let mut removed = 0usize; + while let Ok(Some(entry)) = entries.next_entry().await { + let path = entry.path(); + let is_dir = entry.file_type().await.map(|t| t.is_dir()).unwrap_or(false); + if !is_dir { + continue; + } + match fs::remove_dir_all(&path).await { + Ok(()) => { + tracing::info!( + kind = kind, + path = %path.display(), + "Pruned orphan Antares overlay directory" + ); + removed += 1; + } + Err(e) => tracing::warn!( + kind = kind, + path = %path.display(), + error = %e, + "Failed to prune orphan Antares overlay directory" + ), + } + } + removed + } + + async fn has_any_mountpoint(root: &Path) -> bool { + let Ok(mut entries) = fs::read_dir(root).await else { + return false; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + if is_mountpoint(&entry.path()).await { + return true; + } + } + false + } + + async fn is_mountpoint(path: &Path) -> bool { + let Ok(output) = Command::new("mountpoint") + .args(["-q", &path.to_string_lossy()]) + .status() + .await + else { + return false; + }; + output.success() + } + /// Convert panics within scorpiofs calls into regular errors. async fn run_with_panic_guard(context: String, future: F) -> Result where @@ -1067,6 +1189,8 @@ mod imp { pub struct AntaresConfig { pub mountpoint: PathBuf, pub job_id: String, + pub upper_dir: PathBuf, + pub cl_dir: Option, } /// Mounting Antares requires `scorpiofs` (Linux-only in this repository). @@ -1088,6 +1212,14 @@ mod imp { ))) } + #[allow(dead_code)] + pub async fn reclaim_overlay_dirs(_config: &AntaresConfig) {} + + #[allow(dead_code)] + pub async fn prune_orphan_overlay_dirs() -> Result<(usize, usize), DynError> { + Ok((0, 0)) + } + #[allow(dead_code)] pub(crate) async fn warmup_dicfuse() -> Result<(), DynError> { Ok(()) diff --git a/orion/src/buck_controller.rs b/orion/src/buck_controller.rs index 5d310158f..556b4fe26 100644 --- a/orion/src/buck_controller.rs +++ b/orion/src/buck_controller.rs @@ -229,19 +229,35 @@ pub async fn mount_antares_fs( Ok((mountpoint, mount_id)) } -/// Unmount an Antares overlay filesystem. +/// Unmount an Antares overlay filesystem and reclaim COW dirs unless retain is set. /// /// # Arguments /// * `mount_id` - The job/mount ID to unmount /// /// # Returns -/// * `Ok(true)` - Unmount succeeded +/// * `Ok(true)` - Unmount succeeded (dirs reclaimed when not retaining) +/// * `Ok(false)` - No mount was found for `mount_id` /// * `Err` - Unmount failed #[allow(dead_code)] pub async fn unmount_antares_fs(mount_id: &str) -> Result> { tracing::info!("Starting unmount for mount_id: {}", mount_id); - crate::antares::unmount_job(mount_id).await?; + let config = crate::antares::unmount_job(mount_id).await?; + let Some(config) = config else { + tracing::warn!("Unmount reported no-op for mount_id: {}", mount_id); + return Ok(false); + }; + + if retain_antares_mounts() { + tracing::info!( + "ORION_RETAIN_ANTARES_MOUNTS set — keeping overlay dirs for mount_id={} upper={} cl={:?}", + mount_id, + config.upper_dir.display(), + config.cl_dir.as_ref().map(|p| p.display().to_string()) + ); + } else { + crate::antares::reclaim_overlay_dirs(&config).await; + } tracing::info!("Unmount succeeded for mount_id: {}", mount_id); Ok(true) diff --git a/orion/src/disk.rs b/orion/src/disk.rs new file mode 100644 index 000000000..006d4b2e2 --- /dev/null +++ b/orion/src/disk.rs @@ -0,0 +1,101 @@ +//! Guest filesystem pressure helpers for long-lived Orion VMs. + +use std::path::Path; + +/// Default warn threshold (percent used). Overridable via `ORION_DISK_WARN_PCT`. +const DEFAULT_WARN_PCT: u64 = 85; +/// Default reject-new-builds threshold. Overridable via `ORION_DISK_REJECT_PCT`. +const DEFAULT_REJECT_PCT: u64 = 92; +/// Default critical threshold (still reject; prefer only heartbeats). `ORION_DISK_CRIT_PCT`. +const DEFAULT_CRIT_PCT: u64 = 98; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiskPressure { + Ok { used_pct: u64 }, + Warn { used_pct: u64 }, + Reject { used_pct: u64 }, + Critical { used_pct: u64 }, +} + +impl DiskPressure { + pub fn used_pct(self) -> u64 { + match self { + Self::Ok { used_pct } + | Self::Warn { used_pct } + | Self::Reject { used_pct } + | Self::Critical { used_pct } => used_pct, + } + } + + pub fn should_reject_builds(self) -> bool { + matches!(self, Self::Reject { .. } | Self::Critical { .. }) + } +} + +fn env_pct(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .map(|v| v.min(100)) + .unwrap_or(default) +} + +/// Return used percentage of the filesystem containing `path` (0–100). +pub fn disk_used_pct(path: impl AsRef) -> Option { + let path = path.as_ref(); + let output = std::process::Command::new("df") + .args(["-P", &path.to_string_lossy()]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let stdout = String::from_utf8_lossy(&output.stdout); + // Filesystem 1024-blocks Used Available Capacity Mounted on + let line = stdout.lines().nth(1)?; + let capacity = line.split_whitespace().nth(4)?; + capacity.trim_end_matches('%').parse().ok() +} + +pub fn assess_root_disk() -> DiskPressure { + let used = disk_used_pct("/").unwrap_or(0); + let warn = env_pct("ORION_DISK_WARN_PCT", DEFAULT_WARN_PCT); + let reject = env_pct("ORION_DISK_REJECT_PCT", DEFAULT_REJECT_PCT); + let crit = env_pct("ORION_DISK_CRIT_PCT", DEFAULT_CRIT_PCT); + + if used >= crit { + DiskPressure::Critical { used_pct: used } + } else if used >= reject { + DiskPressure::Reject { used_pct: used } + } else if used >= warn { + DiskPressure::Warn { used_pct: used } + } else { + DiskPressure::Ok { used_pct: used } + } +} + +/// Best-effort orphan Antares overlay prune when under disk pressure. +pub async fn reclaim_under_pressure() { + match crate::antares::prune_orphan_overlay_dirs().await { + Ok((upper, cl)) => { + if upper + cl > 0 { + tracing::warn!( + upper_removed = upper, + cl_removed = cl, + "Pruned Antares overlay dirs due to disk pressure" + ); + } + } + Err(e) => tracing::warn!("Disk-pressure overlay prune failed: {e}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn assess_root_disk_returns_a_variant() { + let _ = assess_root_disk(); + } +} diff --git a/orion/src/lib.rs b/orion/src/lib.rs index cea91deb3..b1f212348 100644 --- a/orion/src/lib.rs +++ b/orion/src/lib.rs @@ -1,6 +1,7 @@ mod antares; mod api; mod buck_controller; +mod disk; pub mod repo; mod util; pub mod ws; diff --git a/orion/src/main.rs b/orion/src/main.rs index 1ea72e60e..2347f1d1d 100644 --- a/orion/src/main.rs +++ b/orion/src/main.rs @@ -2,6 +2,7 @@ mod antares; mod api; mod buck_controller; +mod disk; pub mod repo; mod util; mod ws; diff --git a/orion/src/ws.rs b/orion/src/ws.rs index df2904d2f..ebce6e57d 100644 --- a/orion/src/ws.rs +++ b/orion/src/ws.rs @@ -376,6 +376,43 @@ async fn process_server_message( changes, } => { tracing::info!("Received task: id={}", build_id); + + let pressure = crate::disk::assess_root_disk(); + if pressure.should_reject_builds() { + tracing::error!( + used_pct = pressure.used_pct(), + "Rejecting task {} due to disk pressure", + build_id + ); + crate::disk::reclaim_under_pressure().await; + // Re-check after prune — may have recovered. + let after = crate::disk::assess_root_disk(); + if after.should_reject_builds() { + if let Err(e) = sender.send(WSMessage::TaskAck { + build_id, + success: false, + message: format!( + "disk pressure: root filesystem {}% used (threshold {}%). Free space under /data/scorpio or restart orion-runner after cleanup.", + after.used_pct(), + std::env::var("ORION_DISK_REJECT_PCT").unwrap_or_else(|_| "92".into()) + ), + }) { + tracing::error!("Failed to send TaskAck: {}", e); + } + return ControlFlow::Continue(()); + } + tracing::warn!( + used_pct = after.used_pct(), + "Disk pressure cleared after prune; accepting task" + ); + } else if matches!(pressure, crate::disk::DiskPressure::Warn { .. }) { + tracing::warn!( + used_pct = pressure.used_pct(), + "Disk usage elevated before accepting task {}", + build_id + ); + } + tokio::spawn(async move { let task_id_uuid = match Uuid::parse_str(&build_id) { Ok(uuid) => uuid, From 12658cccbc2bf9ef8ea9e66c041e6d53d2afa06c Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Thu, 23 Jul 2026 10:11:38 +0800 Subject: [PATCH 3/6] feat(orion): stream runner startup logs to the /oc UI Proxy orion-scheduler's /logs/orion/stream through mono as admin-only SSE, and show live provision/startup output on the Orion Clients page after Start Runner. --- clients/orion-scheduler-client/Cargo.toml | 2 +- .../orion-scheduler-client/src/http_client.rs | 60 ++++++++++++++ clients/orion-scheduler-client/src/lib.rs | 11 ++- mono/src/api/router/orion_runner_router.rs | 57 +++++++++++++- .../web/hooks/OrionClient/useRunnerLogsSSE.ts | 78 +++++++++++++++++++ moon/apps/web/pages/[org]/oc/index.tsx | 35 +++++++++ 6 files changed, 240 insertions(+), 3 deletions(-) create mode 100644 moon/apps/web/hooks/OrionClient/useRunnerLogsSSE.ts diff --git a/clients/orion-scheduler-client/Cargo.toml b/clients/orion-scheduler-client/Cargo.toml index 5ddaf1eb9..7e22ce311 100644 --- a/clients/orion-scheduler-client/Cargo.toml +++ b/clients/orion-scheduler-client/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [dependencies] common = { workspace = true } -reqwest = { workspace = true, features = ["json"] } +reqwest = { workspace = true, features = ["json", "stream"] } anyhow = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/clients/orion-scheduler-client/src/http_client.rs b/clients/orion-scheduler-client/src/http_client.rs index 91a4bb988..32e59b7b7 100644 --- a/clients/orion-scheduler-client/src/http_client.rs +++ b/clients/orion-scheduler-client/src/http_client.rs @@ -128,6 +128,60 @@ impl OrionSchedulerHttpClient { )) } } + + /// Open an SSE stream of live Orion runner logs (`GET /logs/orion/stream`). + /// + /// Prefer `vm_id`; when absent, `domain` is used. At least one must be set. + /// The returned response body is a long-lived `text/event-stream` — do not + /// apply a short request timeout. + pub async fn stream_orion_logs( + &self, + vm_id: Option<&str>, + domain: Option<&str>, + ) -> anyhow::Result { + if vm_id.is_none() && domain.is_none() { + return Err(anyhow::anyhow!( + "stream_orion_logs requires vm_id or domain" + )); + } + + let mut url = format!("{}/logs/orion/stream?", self.base_url); + let mut params: Vec = Vec::new(); + if let Some(id) = vm_id { + params.push(format!("vm_id={}", urlencoding_query(id))); + } + if let Some(d) = domain { + params.push(format!("domain={}", urlencoding_query(d))); + } + url.push_str(¶ms.join("&")); + + let req = self.client.get(&url); + let res = self.auth_headers(req).send().await?; + let status = res.status(); + if status.is_success() { + Ok(res) + } else { + let body = res.text().await.unwrap_or_default(); + Err(anyhow::anyhow!( + "Scheduler stream_orion_logs failed ({}): {}", + status, + body + )) + } + } +} + +fn urlencoding_query(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for b in value.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out } #[cfg(test)] @@ -139,4 +193,10 @@ mod tests { let client = OrionSchedulerHttpClient::new("http://127.0.0.1:8080/", ""); assert_eq!(client.base_url, "http://127.0.0.1:8080"); } + + #[test] + fn urlencoding_query_encodes_reserved_bytes() { + assert_eq!(urlencoding_query("a b"), "a%20b"); + assert_eq!(urlencoding_query("orion.gitmega.com"), "orion.gitmega.com"); + } } diff --git a/clients/orion-scheduler-client/src/lib.rs b/clients/orion-scheduler-client/src/lib.rs index fc5bcf323..5e0580f53 100644 --- a/clients/orion-scheduler-client/src/lib.rs +++ b/clients/orion-scheduler-client/src/lib.rs @@ -1,4 +1,4 @@ -//! HTTP client for orion-scheduler VM provisioning (`/webhook`, `/status`, `/vms/{id}`). +//! HTTP client for orion-scheduler VM provisioning (`/webhook`, `/status`, `/vms/{id}`, logs SSE). mod http_client; @@ -95,4 +95,13 @@ impl OrionSchedulerClient { pub async fn get_status(&self) -> anyhow::Result { self.http.get_status().await } + + /// Proxy-friendly SSE stream of runner / orion-client startup logs. + pub async fn stream_orion_logs( + &self, + vm_id: Option<&str>, + domain: Option<&str>, + ) -> anyhow::Result { + self.http.stream_orion_logs(vm_id, domain).await + } } diff --git a/mono/src/api/router/orion_runner_router.rs b/mono/src/api/router/orion_runner_router.rs index 15db2850a..a267abca5 100644 --- a/mono/src/api/router/orion_runner_router.rs +++ b/mono/src/api/router/orion_runner_router.rs @@ -2,11 +2,14 @@ use anyhow::anyhow; use api_model::common::CommonResult; use axum::{ Json, + body::Body, extract::{Path, State}, - http::StatusCode, + http::{HeaderValue, StatusCode, header}, + response::Response, }; use ceres::model::orion_runner::{RunnerStatusResponse, StartRunnerRequest, StartRunnerResponse}; use common::config::BuildConfig; +use futures::TryStreamExt; use orion_scheduler_client::{OrionSchedulerClient, StartRunnerPayload}; use utoipa_axum::{router::OpenApiRouter, routes}; @@ -20,6 +23,7 @@ pub fn routers() -> OpenApiRouter { "/orion/runners", OpenApiRouter::new() .routes(routes!(start_runner)) + .routes(routes!(stream_runner_logs)) .routes(routes!(get_runner_status)), ) } @@ -252,6 +256,57 @@ async fn get_runner_status( })))) } +/// Proxy live Orion runner / client startup logs from orion-scheduler as SSE. +#[utoipa::path( + get, + path = "/{id}/logs/stream", + params( + ("id" = String, Path, description = "VM ID returned by start_runner") + ), + responses( + (status = 200, description = "SSE log stream (text/event-stream)"), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Forbidden - admin only"), + (status = 503, description = "Scheduler not configured"), + (status = 502, description = "Scheduler unreachable"), + ), + tag = ORION_RUNNER_TAG +)] +async fn stream_runner_logs( + user: LoginUser, + State(state): State, + Path(id): Path, +) -> Result { + ensure_admin(&state, &user).await?; + let client = scheduler_client(&state)?; + + let upstream = client + .stream_orion_logs(Some(id.as_str()), None) + .await + .map_err(|e| { + ApiError::with_status( + StatusCode::BAD_GATEWAY, + anyhow!("Scheduler log stream failed: {}", e), + ) + })?; + + let byte_stream = upstream + .bytes_stream() + .map_err(|e| std::io::Error::other(e)); + + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/event-stream") + .header(header::CACHE_CONTROL, "no-cache") + .header(header::CONNECTION, "keep-alive") + .header( + header::HeaderName::from_static("x-accel-buffering"), + HeaderValue::from_static("no"), + ) + .body(Body::from_stream(byte_stream)) + .map_err(|e| ApiError::with_status(StatusCode::INTERNAL_SERVER_ERROR, anyhow!(e))) +} + #[cfg(test)] mod tests { use super::*; diff --git a/moon/apps/web/hooks/OrionClient/useRunnerLogsSSE.ts b/moon/apps/web/hooks/OrionClient/useRunnerLogsSSE.ts new file mode 100644 index 000000000..183839337 --- /dev/null +++ b/moon/apps/web/hooks/OrionClient/useRunnerLogsSSE.ts @@ -0,0 +1,78 @@ +import { useEffect, useRef, useState } from 'react' + +import { MONO_API_URL } from '@gitmono/config' + +export type RunnerLogsStatus = 'idle' | 'connecting' | 'streaming' | 'error' + +const MAX_LOG_CHARS = 400_000 + +function runnerLogsStreamUrl(vmId: string): string { + const base = MONO_API_URL.replace(/\/$/, '') + + return `${base}/api/v1/orion/runners/${encodeURIComponent(vmId)}/logs/stream` +} + +/** + * Subscribe to mono-proxied Orion runner startup logs (SSE). + * Enabled while `vmId` is set; closes the EventSource when cleared or unmounted. + */ +export function useRunnerLogsSSE(vmId: string | null) { + const [logs, setLogs] = useState('') + const [status, setStatus] = useState('idle') + const [error, setError] = useState(null) + const esRef = useRef(null) + + useEffect(() => { + if (!vmId) { + esRef.current?.close() + esRef.current = null + setLogs('') + setStatus('idle') + setError(null) + return + } + + setLogs('') + setError(null) + setStatus('connecting') + + const es = new EventSource(runnerLogsStreamUrl(vmId), { withCredentials: true }) + + esRef.current = es + + es.onopen = () => { + setStatus('streaming') + } + + es.onmessage = (event) => { + const chunk = event.data + + if (!chunk) return + + setLogs((prev) => { + const next = prev ? `${prev}${chunk}` : chunk + + if (next.length <= MAX_LOG_CHARS) return next + return next.slice(next.length - MAX_LOG_CHARS) + }) + } + + es.onerror = () => { + // EventSource reconnects automatically on transient errors; only surface a + // sticky error once the connection is permanently closed. + if (es.readyState === EventSource.CLOSED) { + setStatus('error') + setError('Log stream disconnected') + } + } + + return () => { + es.close() + if (esRef.current === es) { + esRef.current = null + } + } + }, [vmId]) + + return { logs, status, error } +} diff --git a/moon/apps/web/pages/[org]/oc/index.tsx b/moon/apps/web/pages/[org]/oc/index.tsx index ded0c3747..8db8cc6f0 100644 --- a/moon/apps/web/pages/[org]/oc/index.tsx +++ b/moon/apps/web/pages/[org]/oc/index.tsx @@ -20,6 +20,7 @@ import { useAdminCheck } from '@/hooks/admin/useAdminCheck' import { usePostOrionClientsInfo } from '@/hooks/OrionClient/OrionClientsInfo' import { useGetRunnerStatus } from '@/hooks/OrionClient/useGetRunnerStatus' import { usePostStartRunner } from '@/hooks/OrionClient/usePostStartRunner' +import { useRunnerLogsSSE } from '@/hooks/OrionClient/useRunnerLogsSSE' import { PageWithLayout } from '@/utils/types' const OrionClientPage: PageWithLayout = () => { @@ -38,6 +39,7 @@ const OrionClientPage: PageWithLayout = () => { const { mutate: startRunner, isPending: isStartingRunner } = usePostStartRunner() const { data: runnerStatus } = useGetRunnerStatus(activeVmId, activePhase) + const { logs: runnerLogs, status: runnerLogsStatus, error: runnerLogsError } = useRunnerLogsSSE(activeVmId) const { mutate, isPending, error } = usePostOrionClientsInfo() const [clientsPage, setClientsPage] = React.useState(null) @@ -207,6 +209,11 @@ const OrionClientPage: PageWithLayout = () => { VM IP: {runnerStatus.vm_ip} ) : null} + {runnerStatus?.log_file ? ( + + Log file: {runnerStatus.log_file} + + ) : null} {runnerStatus?.error ? ( {runnerStatus.error} @@ -218,6 +225,34 @@ const OrionClientPage: PageWithLayout = () => { ) : null} + +
+
+ + Startup logs + + + {runnerLogsStatus === 'connecting' + ? 'Connecting…' + : runnerLogsStatus === 'streaming' + ? 'Live' + : runnerLogsStatus === 'error' + ? 'Disconnected' + : 'Idle'} + +
+ {runnerLogsError ? ( + + {runnerLogsError} + + ) : null} + 
+                  {runnerLogs ||
+                    (runnerLogsStatus === 'connecting'
+                      ? 'Waiting for log stream…'
+                      : 'No log lines yet. Logs appear while the runner provisions.')}
+
+
) : null} From d5fef1ce429ad9ca193dd8709afa13df9ac59a9c Mon Sep 17 00:00:00 2001 From: "benjamin.747" Date: Thu, 23 Jul 2026 11:06:53 +0800 Subject: [PATCH 4/6] fix(orion): drop DashMap guard before await on build dispatch Holding workers.get_mut across DB awaits on retry could stall the tokio runtime under heartbeat/health-check contention. Extract claim_worker_for_build and cover with a concurrency regression test. --- ceres/src/model/admin.rs | 3 +- mono/src/api/api_common/group_permission.rs | 14 +- mono/src/api/guard/cedar_guard.rs | 14 +- mono/src/api/guard/guarded_endpoints.json | 28 ++-- mono/src/api/oauth/model.rs | 20 +++ mono/src/api/router/admin_router.rs | 11 +- moon/api/README.md | 17 +- .../BlobView/MegaCedarAdminPicker.tsx | 13 +- moon/apps/web/hooks/useGetLatestRelease.ts | 3 +- moon/packages/types/generated.ts | 33 +++- orion-server/src/scheduler.rs | 153 +++++++++++++++--- orion-server/src/service/api_v2_service.rs | 51 ++---- saturn/mega.cedarschema | 6 +- saturn/mega_policies.cedar | 12 +- saturn/src/entitystore.rs | 15 +- saturn/src/lib.rs | 28 ++-- saturn/src/objects.rs | 6 +- saturn/test/project/.mega.json | 2 +- saturn/test/project/private/.mega.json | 2 +- 19 files changed, 288 insertions(+), 143 deletions(-) diff --git a/ceres/src/model/admin.rs b/ceres/src/model/admin.rs index 848a7d16a..7a9a1aada 100644 --- a/ceres/src/model/admin.rs +++ b/ceres/src/model/admin.rs @@ -11,9 +11,10 @@ pub struct AdminListResponse { pub admins: Vec, } -/// Request body for generating `.mega_cedar.json` content from admin usernames. +/// Request body for generating `.mega_cedar.json` content from admin GitHub logins. #[derive(Debug, Deserialize, ToSchema)] pub struct GenerateCedarRequest { + /// GitHub login names used as Cedar `User` euids (e.g. `octocat`). pub admins: Vec, } diff --git a/mono/src/api/api_common/group_permission.rs b/mono/src/api/api_common/group_permission.rs index f0ad8d0f8..a5287336b 100644 --- a/mono/src/api/api_common/group_permission.rs +++ b/mono/src/api/api_common/group_permission.rs @@ -8,17 +8,21 @@ use http::StatusCode; use crate::api::{MonoApiServiceState, error::ApiError, oauth::model::LoginUser}; pub async fn ensure_admin(state: &MonoApiServiceState, user: &LoginUser) -> Result<(), ApiError> { - if state - .services() - .admin() - .check_is_admin(&user.username) - .await? + let admin = state.services().admin(); + let cedar_id = user.cedar_user_id(); + + // Prefer GitHub login (cedar_user_id). Also accept Campsite username while + // `.mega_cedar.json` is still migrating from username → github_login. + if admin.check_is_admin(cedar_id).await? + || (cedar_id != user.username.as_str() && admin.check_is_admin(&user.username).await?) { return Ok(()); } tracing::warn!( actor = %user.username, + cedar_user_id = %cedar_id, + github_login = ?user.github_login, "admin check failed: access forbidden" ); diff --git a/mono/src/api/guard/cedar_guard.rs b/mono/src/api/guard/cedar_guard.rs index efd1a5570..001b29d32 100644 --- a/mono/src/api/guard/cedar_guard.rs +++ b/mono/src/api/guard/cedar_guard.rs @@ -140,7 +140,7 @@ pub async fn cedar_guard( if let Ok(bot) = BotAuth::from_request_parts(&mut parts, &state).await { ("Bot".to_string(), bot.bot_id.to_string()) } else if let Ok(user) = LoginUser::from_request_parts(&mut parts, &state).await { - ("User".to_string(), user.username.clone()) + ("User".to_string(), user.cedar_user_id().to_string()) } else { ("User".to_string(), "reader".to_string()) }; @@ -219,12 +219,12 @@ mod tests { let patterns: HashMap = HashMap::from([ ( "/{link}/approve".to_string(), - "approveMergeRequest".to_string(), + "approveChangeList".to_string(), ), - ("/{link}/close".to_string(), "editMergeRequest".to_string()), + ("/{link}/close".to_string(), "editChangeList".to_string()), ( "/{link}/review/delete".to_string(), - "editMergeRequest".to_string(), + "editChangeList".to_string(), ), ]); @@ -232,14 +232,14 @@ mod tests { let result = match_operation(suffix, &patterns); assert_eq!( result, - Some((ActionEnum::ApproveMergeRequest, "my-cl-link".to_string())) + Some((ActionEnum::ApproveChangeList, "my-cl-link".to_string())) ); let suffix = "/another-cl-link/close"; let result = match_operation(suffix, &patterns); assert_eq!( result, - Some((ActionEnum::EditMergeRequest, "another-cl-link".to_string())) + Some((ActionEnum::EditChangeList, "another-cl-link".to_string())) ); let suffix = "/no-match-link/delete"; @@ -250,7 +250,7 @@ mod tests { let result = match_operation(suffix, &patterns); assert_eq!( result, - Some((ActionEnum::EditMergeRequest, "path/subpath".to_string())) + Some((ActionEnum::EditChangeList, "path/subpath".to_string())) ); } } diff --git a/mono/src/api/guard/guarded_endpoints.json b/mono/src/api/guard/guarded_endpoints.json index 69eb42fbf..2ab2dcf6e 100644 --- a/mono/src/api/guard/guarded_endpoints.json +++ b/mono/src/api/guard/guarded_endpoints.json @@ -1,25 +1,25 @@ { "/cl": { "/list": "unprotectedRequest", - "/{link}/reopen": "editMergeRequest", - "/{link}/close": "editMergeRequest", - "/{link}/merge": "approveMergeRequest", - "/{link}/comment": "editMergeRequest", - "/{link}/title": "editMergeRequest", - "/{link}/labels": "editMergeRequest", - "/{link}/assignees": "editMergeRequest", - "/{link}/reviewers": "editMergeRequest", - "/{link}/approve": "approveMergeRequest", - "/{link}/resolve": "editMergeRequest", + "/{link}/reopen": "editChangeList", + "/{link}/close": "editChangeList", + "/{link}/merge": "approveChangeList", + "/{link}/comment": "editChangeList", + "/{link}/title": "editChangeList", + "/{link}/labels": "editChangeList", + "/{link}/assignees": "editChangeList", + "/{link}/reviewers": "editChangeList", + "/{link}/approve": "approveChangeList", + "/{link}/resolve": "editChangeList", "/{link}/detail": "viewRepo", "/{link}/mui-tree": "viewRepo", "/{link}/files-changed": "viewRepo", "/{link}/files-list": "viewRepo", "/{link}/merge-box": "viewRepo", - "/{link}/status": "editMergeRequest", + "/{link}/status": "editChangeList", "/{link}/update-status": "viewRepo", - "/{link}/update-branch": "editMergeRequest", + "/{link}/update-branch": "editChangeList", "/reviewer/{link}": "viewRepo", - "/reviewer": "editMergeRequest" + "/reviewer": "editChangeList" } -} \ No newline at end of file +} diff --git a/mono/src/api/oauth/model.rs b/mono/src/api/oauth/model.rs index b829da3f7..c1e1f22a1 100644 --- a/mono/src/api/oauth/model.rs +++ b/mono/src/api/oauth/model.rs @@ -6,6 +6,8 @@ pub struct CampsiteUserJson { pub id: String, pub avatar_url: String, pub email: Option, + #[serde(default)] + pub github_login: Option, } impl From for LoginUser { @@ -15,6 +17,9 @@ impl From for LoginUser { email: value.email.unwrap_or_default(), avatar_url: value.avatar_url, campsite_user_id: value.id, + github_login: value + .github_login + .filter(|s| !s.trim().is_empty()), } } } @@ -49,6 +54,7 @@ impl From for LoginUser { username: value.name, email: value.email.unwrap_or_default(), avatar_url: value.image.unwrap_or_default(), + github_login: None, } } } @@ -57,6 +63,20 @@ impl From for LoginUser { pub struct LoginUser { pub campsite_user_id: String, pub username: String, + /// GitHub login when Campsite authenticated via GitHub; used as Cedar User euid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub github_login: Option, pub avatar_url: String, pub email: String, } + +impl LoginUser { + /// Identity for Cedar / `.mega_cedar.json` admin checks: GitHub login when present. + pub fn cedar_user_id(&self) -> &str { + self.github_login + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .unwrap_or(self.username.as_str()) + } +} diff --git a/mono/src/api/router/admin_router.rs b/mono/src/api/router/admin_router.rs index 80d9903b0..e6fe3e56b 100644 --- a/mono/src/api/router/admin_router.rs +++ b/mono/src/api/router/admin_router.rs @@ -60,11 +60,10 @@ async fn is_admin_me( user: LoginUser, State(state): State, ) -> Result>, ApiError> { - let is_admin = state - .services() - .admin() - .check_is_admin(&user.username) - .await?; + let admin = state.services().admin(); + let cedar_id = user.cedar_user_id(); + let is_admin = admin.check_is_admin(cedar_id).await? + || (cedar_id != user.username.as_str() && admin.check_is_admin(&user.username).await?); Ok(Json(CommonResult::success(Some(IsAdminResponse { is_admin, @@ -100,7 +99,7 @@ async fn admin_list( /// POST /api/v1/admin/cedar/generate /// -/// Generate `.mega_cedar.json` content from a list of admin usernames. +/// Generate `.mega_cedar.json` content from a list of admin GitHub logins. /// Only admins can access this endpoint. Does not write to the repository. #[utoipa::path( post, diff --git a/moon/api/README.md b/moon/api/README.md index 202229773..e4c547ef9 100644 --- a/moon/api/README.md +++ b/moon/api/README.md @@ -4,12 +4,15 @@ Frontend API types live in `@gitmono/types` (`packages/types/generated.ts`). The ## Do not edit these by hand -| File | Why | -|------|-----| -| [`gen/gitmono.json`](gen/gitmono.json) | Exported from the **mono** OpenAPI (`utoipa`). Refresh from a running mono server. | +| File | Why | +| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------- | +| [`gen/gitmono.json`](gen/gitmono.json) | Exported from the **mono** OpenAPI (`utoipa`). Refresh from a running mono server. | +| [`gen/1schema_swagger.json`](gen/1schema_swagger.json) | Exported from **Campsite** Apigen (`rake apigen:…`). Owns types like `SyncUser` / `CurrentUser`. | | [`../packages/types/generated.ts`](../packages/types/generated.ts) | Produced by [`script/gen-client`](../script/gen-client) via `swagger-typescript-api`. | -Also avoid hand-editing other checked-in OpenAPI dumps that feed the same pipeline (for example [`gen/orion.json`](gen/orion.json) from **orion-server**). Change the Rust `utoipa` annotations instead, then re-export and regenerate. +Also avoid hand-editing other checked-in OpenAPI dumps that feed the same pipeline (for example [`gen/orion.json`](gen/orion.json) from **orion-server**). Change the Rust `utoipa` annotations (mono/orion) or Campsite serializers + Apigen export, then re-export and regenerate. + +**Note:** Refreshing only `gitmono.json` from mono will **not** pick up Campsite fields such as `SyncUser.github_login`. Those live in `1schema_swagger.json`. ## Regenerate workflow @@ -30,7 +33,7 @@ curl -sS http://localhost:8004/api-doc/openapi.json -o api/gen/orion.json ## Related endpoints -| Service | Swagger UI | OpenAPI JSON | -|---------|------------|--------------| -| mono | `http://localhost:8000/swagger-ui` | `http://localhost:8000/api/openapi.json` | +| Service | Swagger UI | OpenAPI JSON | +| ------------ | ---------------------------------- | -------------------------------------------- | +| mono | `http://localhost:8000/swagger-ui` | `http://localhost:8000/api/openapi.json` | | orion-server | `http://localhost:8004/swagger-ui` | `http://localhost:8004/api-doc/openapi.json` | diff --git a/moon/apps/web/components/CodeView/BlobView/MegaCedarAdminPicker.tsx b/moon/apps/web/components/CodeView/BlobView/MegaCedarAdminPicker.tsx index ec9e920a1..b53c7a739 100644 --- a/moon/apps/web/components/CodeView/BlobView/MegaCedarAdminPicker.tsx +++ b/moon/apps/web/components/CodeView/BlobView/MegaCedarAdminPicker.tsx @@ -127,7 +127,7 @@ export function MegaCedarAdminPicker({ fileContent, onContentGenerated, disabled
System admins
- Select users to regenerate .mega_cedar.json + Select GitHub logins to regenerate .mega_cedar.json
{generateCedar.isPending && ( @@ -171,8 +171,8 @@ export function MegaCedarAdminPicker({ fileContent, onContentGenerated, disabled ) : (
{members.map((member) => { - const username = member.user.username - const isSelected = selectedAdmins.includes(username) + const cedarId = member.user.github_login || member.user.username + const isSelected = selectedAdmins.includes(cedarId) return (