From fc18b2fe6d60c2e9b7dce796075e89b28928dea1 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 9 Jun 2026 11:59:16 +0100 Subject: [PATCH 01/43] chore(scripts): update grafana datasource downloads --- docker-compose.yml | 2 ++ scripts/install.sh | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 076a110..5330439 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: - ./workspace:/app/workspace - ./infra/caddy/routes:/caddy/routes - /var/run/docker.sock:/var/run/docker.sock + - railpack-cache:/tmp/railpack depends_on: buildkit: condition: service_started @@ -247,3 +248,4 @@ volumes: prometheus-data: loki-data: grafana-data: + railpack-cache: diff --git a/scripts/install.sh b/scripts/install.sh index 15e5fae..488505b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -85,7 +85,9 @@ download_configs() { download_if_missing "$BASE_URL/infra/monitoring/$f" "$INSTALL_DIR/infra/monitoring/$f" done - download_if_missing "$BASE_URL/infra/monitoring/grafana/datasources/datasources.yml" "$INSTALL_DIR/infra/monitoring/grafana/datasources/datasources.yml" + for f in loki.yml prometheus.yml; do + download_if_missing "$BASE_URL/infra/monitoring/grafana/datasources/$f" "$INSTALL_DIR/infra/monitoring/grafana/datasources/$f" + done } prompt_config() { From ebf0409ab86de1b019749ee73f3919210f929062 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 16 Jun 2026 12:09:28 +0100 Subject: [PATCH 02/43] refactor(infra): improve monitoring and configuration management - Update docker-compose and install scripts to handle monitoring configs and Grafana dashboards more effectively - Strengthen pipeline workspace cleanup to prevent blocking deployments - Centralize and update system configurations in the API - Add default admin email and network filtering to monitoring stack --- .github/workflows/release.yml | 6 +- apps/api/src/orchestrator/pipeline.ts | 24 ++- apps/api/src/utils/config.ts | 137 +++++++++++++----- docker-compose.yml | 27 ++-- infra/caddy/Caddyfile | 2 +- infra/monitoring/grafana/datasources/loki.yml | 1 + .../grafana/datasources/prometheus.yml | 1 + infra/monitoring/promtail-config.yml | 3 + scripts/install.sh | 50 +++++-- 9 files changed, 179 insertions(+), 72 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19c64a9..581fc02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,11 +71,7 @@ jobs: cp docker-compose.yml "$TAR_DIR/" cp scripts/dequel "$TAR_DIR/dequel" cp infra/caddy/Caddyfile "$TAR_DIR/infra/caddy/" - cp infra/monitoring/prometheus.yml "$TAR_DIR/infra/monitoring/" - cp infra/monitoring/loki-config.yml "$TAR_DIR/infra/monitoring/" - cp infra/monitoring/promtail-config.yml "$TAR_DIR/infra/monitoring/" - cp infra/monitoring/grafana/datasources/loki.yml "$TAR_DIR/infra/monitoring/grafana/datasources/" - cp infra/monitoring/grafana/datasources/prometheus.yml "$TAR_DIR/infra/monitoring/grafana/datasources/" + cp -r infra/monitoring "$TAR_DIR/infra/" cd "$TAR_DIR" && tar -czf "../dequel-config-${VERSION}.tar.gz" . - name: Create GitHub Release diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 34367ad..916b100 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -626,14 +626,22 @@ export class PipelineOrchestrator { return false; } finally { this.abortControllers.delete(deploymentId); - if (workspacePath) - await cleanupWorkspace( - workspacePath, - ); - if (uploadedArchivePath) - await rm(uploadedArchivePath, { - force: true, - }); + try { + if (workspacePath) + await cleanupWorkspace( + workspacePath, + ); + } catch { + // cleanup failures must never mask the deployment result + } + try { + if (uploadedArchivePath) + await rm(uploadedArchivePath, { + force: true, + }); + } catch { + // cleanup failures must never mask the deployment result + } } } diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index 8c6772e..ad4cdb9 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -1,43 +1,112 @@ -import { loadFileConfig, type FileConfig } from "./config-loader"; +import { loadFileConfig } from "./config-loader"; const fileConfig = loadFileConfig(); const withFile = ( - key: string, - envDefault: string, - transform?: (v: string) => T, + key: string, + envDefault: string, + transform?: (v: string) => T, ): T => { - const envVal = process.env[key]; - if (envVal !== undefined) { - return transform ? transform(envVal) : (envVal as unknown as T); - } - const fileVal = (fileConfig as Record)[key]; - if (fileVal !== undefined) return fileVal as T; - return transform ? transform(envDefault) : (envDefault as unknown as T); + const envVal = process.env[key]; + if (envVal !== undefined) { + return transform + ? transform(envVal) + : (envVal as unknown as T); + } + const fileVal = ( + fileConfig as Record + )[key]; + if (fileVal !== undefined) + return fileVal as T; + return transform + ? transform(envDefault) + : (envDefault as unknown as T); }; +const SYSTEM = { + databasePath: "/app/data/dequel.db", + workspaceRoot: "/app/workspace", + caddyRoutesDir: "/caddy/routes", + dockerNetwork: "dequel_net", + buildkitHost: "tcp://buildkit:1234", + redisUrl: "redis://redis:6379", +} as const; + export const config = { - port: withFile("PORT", "3001", Number), - databasePath: withFile("DATABASE_PATH", "/app/data/dequel.db"), - workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), - caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), - caddyBaseDomain: withFile("CADDY_BASE_DOMAIN", "localhost"), - dockerNetwork: withFile("DOCKER_NETWORK", "dequel_net"), - appInternalPort: withFile("APP_INTERNAL_PORT", "3000", Number), - buildkitHost: withFile("BUILDKIT_HOST", "tcp://buildkit:1234"), - envEncryptionKey: withFile("ENV_ENCRYPTION_KEY", "dev-env-key-change-me"), - redisUrl: withFile("REDIS_URL", "redis://redis:6379"), - queueConcurrency: withFile("QUEUE_CONCURRENCY", "1", Number), - queueRetryMax: withFile("QUEUE_RETRY_MAX", "5", Number), - queueRetryBaseMs: withFile("QUEUE_RETRY_BASE_MS", "5000", Number), - smtpHost: withFile("SMTP_HOST", ""), - smtpPort: withFile("SMTP_PORT", "587", Number), - smtpUser: withFile("SMTP_USER", ""), - smtpPass: withFile("SMTP_PASS", ""), - smtpFrom: withFile("SMTP_FROM", "dequel@localhost"), - alertEvalIntervalMs: withFile("ALERT_EVAL_INTERVAL_MS", "60000", Number), - githubClientId: withFile("GITHUB_CLIENT_ID", ""), - githubClientSecret: withFile("GITHUB_CLIENT_SECRET", ""), - githubAppName: withFile("GITHUB_APP_NAME", "Dequel"), - githubWebhookSecret: withFile("GITHUB_WEBHOOK_SECRET", ""), + ...SYSTEM, + port: withFile( + "PORT", + "17474", + Number, + ), + caddyBaseDomain: withFile( + "CADDY_BASE_DOMAIN", + "localhost", + ), + appInternalPort: withFile( + "APP_INTERNAL_PORT", + "17476", + Number, + ), + envEncryptionKey: withFile( + "ENV_ENCRYPTION_KEY", + "dev-env-key-change-me", + ), + queueConcurrency: withFile( + "QUEUE_CONCURRENCY", + "3", + Number, + ), + queueRetryMax: withFile( + "QUEUE_RETRY_MAX", + "5", + Number, + ), + queueRetryBaseMs: withFile( + "QUEUE_RETRY_BASE_MS", + "5000", + Number, + ), + smtpHost: withFile( + "SMTP_HOST", + "", + ), + smtpPort: withFile( + "SMTP_PORT", + "587", + Number, + ), + smtpUser: withFile( + "SMTP_USER", + "", + ), + smtpPass: withFile( + "SMTP_PASS", + "", + ), + smtpFrom: withFile( + "SMTP_FROM", + "dequel@localhost", + ), + alertEvalIntervalMs: withFile( + "ALERT_EVAL_INTERVAL_MS", + "60000", + Number, + ), + githubClientId: withFile( + "GITHUB_CLIENT_ID", + "", + ), + githubClientSecret: withFile( + "GITHUB_CLIENT_SECRET", + "", + ), + githubAppName: withFile( + "GITHUB_APP_NAME", + "Dequel", + ), + githubWebhookSecret: withFile( + "GITHUB_WEBHOOK_SECRET", + "", + ), }; diff --git a/docker-compose.yml b/docker-compose.yml index df049a8..18a28c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,11 +18,8 @@ services: healthcheck: test: [ - "CMD", - "wget", - "--spider", - "-q", - "tcp://localhost:1234", + "CMD-SHELL", + "wget -q -T 2 -O /dev/null http://localhost:1234 >/dev/null 2>&1; [ $$? -ne 4 ]", ] interval: 5s timeout: 3s @@ -104,7 +101,7 @@ services: "--watch", ] environment: - CADDY_EMAIL: ${CADDY_EMAIL:-} + CADDY_EMAIL: ${CADDY_EMAIL:-admin@dequel.local} CADDY_BASE_DOMAIN: ${CADDY_BASE_DOMAIN:-localhost} ports: - "80:80" @@ -166,12 +163,19 @@ services: prometheus: image: prom/prometheus:latest + stop_grace_period: 60s command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.path=/prometheus - - --web.console.libraries=/usr/share/prometheus/console_libraries - - --web.console.templates=/usr/share/prometheus/consoles - - --storage.tsdb.retention.time=30d + - sh + - -c + - | + promtool tsdb rebuild-index /prometheus 2>/dev/null || true + exec prometheus \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --web.console.templates=/usr/share/prometheus/consoles \ + --storage.tsdb.retention.time=30d \ + --storage.tsdb.wal-compression volumes: - ./infra/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus @@ -245,6 +249,7 @@ services: GF_INSTALL_PLUGINS: "" volumes: - ./infra/monitoring/grafana/datasources:/etc/grafana/provisioning/datasources:ro + - ./infra/monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro - grafana-data:/var/lib/grafana depends_on: prometheus: diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index 6b34c41..0dcac11 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -1,5 +1,5 @@ { - email {$CADDY_EMAIL} + email {$CADDY_EMAIL:admin@dequel.local} } import /etc/caddy/routes/*.caddy diff --git a/infra/monitoring/grafana/datasources/loki.yml b/infra/monitoring/grafana/datasources/loki.yml index 67ce8b7..9d8ab1e 100644 --- a/infra/monitoring/grafana/datasources/loki.yml +++ b/infra/monitoring/grafana/datasources/loki.yml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Loki + uid: loki type: loki access: proxy url: http://loki:3100 diff --git a/infra/monitoring/grafana/datasources/prometheus.yml b/infra/monitoring/grafana/datasources/prometheus.yml index bb009bb..00f9915 100644 --- a/infra/monitoring/grafana/datasources/prometheus.yml +++ b/infra/monitoring/grafana/datasources/prometheus.yml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Prometheus + uid: prometheus type: prometheus access: proxy url: http://prometheus:9090 diff --git a/infra/monitoring/promtail-config.yml b/infra/monitoring/promtail-config.yml index 0bf72ff..0adcfd9 100644 --- a/infra/monitoring/promtail-config.yml +++ b/infra/monitoring/promtail-config.yml @@ -14,6 +14,9 @@ scrape_configs: - host: unix:///var/run/docker.sock refresh_interval: 15s relabel_configs: + - source_labels: ['__meta_docker_container_network_dequel_net'] + regex: 'true' + action: keep - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' target_label: 'container' diff --git a/scripts/install.sh b/scripts/install.sh index 14dde29..f43e82b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -54,7 +54,7 @@ check_prerequisites() { setup_directories() { header "Setting up installation directory" - mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/workspace" "$INSTALL_DIR/infra/caddy/routes" "$INSTALL_DIR/infra/monitoring/grafana/datasources" + mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/workspace" "$INSTALL_DIR/infra/caddy/routes" "$INSTALL_DIR/infra/monitoring/grafana/datasources" "$INSTALL_DIR/infra/monitoring/grafana/dashboards" info "Installing to: $INSTALL_DIR" } @@ -113,28 +113,52 @@ download_configs() { for f in loki.yml prometheus.yml; do download_if_missing "$BASE_URL/infra/monitoring/grafana/datasources/$f" "$INSTALL_DIR/infra/monitoring/grafana/datasources/$f" done + + for f in dashboards.yml deployed-apps.json; do + download_if_missing "$BASE_URL/infra/monitoring/grafana/dashboards/$f" "$INSTALL_DIR/infra/monitoring/grafana/dashboards/$f" + done } prompt_config() { header "Configuration" - if [ ! -t 0 ]; then - warn "Non-interactive mode: skipping configuration prompt" - warn "Set CADDY_EMAIL and CADDY_BASE_DOMAIN manually in $INSTALL_DIR/.env" + local ADMIN_EMAIL="" + local HOSTNAME="" + + if [ -t 0 ]; then + read -r -p " Admin email (for SSL notifications, optional): " ADMIN_EMAIL + read -r -p " Base domain (e.g. dequel.example.com, optional): " HOSTNAME + elif (: /dev/null; then + read -r -p " Admin email (for SSL notifications, optional): " ADMIN_EMAIL < /dev/tty + read -r -p " Base domain (e.g. dequel.example.com, optional): " HOSTNAME < /dev/tty + else + warn "No terminal — skipping configuration prompt" + warn "Set CADDY_EMAIL and CADDY_BASE_DOMAIN in $INSTALL_DIR/.env after install" return fi - read -r -p " Admin email (for SSL notifications, optional): " ADMIN_EMAIL - read -r -p " Hostname (e.g. dequel.example.com, optional): " HOSTNAME + mkdir -p "$INSTALL_DIR/data" - if [ -n "$ADMIN_EMAIL" ] || [ -n "$HOSTNAME" ]; then - cat > "$INSTALL_DIR/.env" </dev/null || dd if=/dev/urandom bs=32 count=1 status=none 2>/dev/null | od -A n -t x1 | tr -d ' \n' || echo "dev-env-key-change-me") + + cat > "$INSTALL_DIR/data/dequel.json" < "$INSTALL_DIR/.env" + chmod 600 "$INSTALL_DIR/.env" + success "Created $INSTALL_DIR/.env" } pull_images() { From ffdceb55071291ada138d97f627dab3eaf76f6d4 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 16 Jun 2026 12:09:57 +0100 Subject: [PATCH 03/43] chore: add issue templates, PR template, and security CI - Add GitHub issue templates for bug reports and feature requests - Add a pull request template for standardized contributions - Add a security workflow for scanning for forbidden patterns and vulnerabilities - Add a version bump utility script - Add Grafana dashboard provisioning for monitoring --- .github/ISSUE_TEMPLATE/bug_report.yml | 111 +++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 + .github/ISSUE_TEMPLATE/feature_request.yml | 88 ++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 43 +++++ .github/workflows/vuln-scan.yml | 42 +++++ CONTRIBUTING.md | 110 ++++++++++++ LICENSE | 21 +++ bump.sh | 1 + .../grafana/dashboards/dashboards.yml | 11 ++ .../grafana/dashboards/deployed-apps.json | 157 ++++++++++++++++++ scripts/workflow/bump.sh | 124 ++++++++++++++ scripts/workflow/forbidden-pattern-scan.sh | 20 +++ 12 files changed, 736 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/vuln-scan.yml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 120000 bump.sh create mode 100644 infra/monitoring/grafana/dashboards/dashboards.yml create mode 100644 infra/monitoring/grafana/dashboards/deployed-apps.json create mode 100755 scripts/workflow/bump.sh create mode 100644 scripts/workflow/forbidden-pattern-scan.sh diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..7b48faf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,111 @@ +name: Bug Report +description: File a bug report to help us improve Dequel +title: "[Bug]: " +labels: ["bug", "triage"] +projects: [] +assignees: [] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report. Please provide as much detail as possible. + + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please search to see if an issue already exists for the bug you encountered. + options: + - label: I have searched the existing issues and found nothing matching + required: true + + - type: textarea + attributes: + label: Describe the Bug + description: A clear and concise description of what the bug is. + placeholder: When I do X, Y happens instead of Z... + validations: + required: true + + - type: textarea + attributes: + label: Steps to Reproduce + description: Steps to reproduce the behavior. + placeholder: | + 1. Go to '...' + 2. Click on '...' + 3. Scroll down to '...' + 4. See error + validations: + required: true + + - type: textarea + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + placeholder: I expected X to happen... + validations: + required: true + + - type: textarea + attributes: + label: Screenshots / Logs + description: | + If applicable, add screenshots or relevant logs to help explain your problem. + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + placeholder: Drag and drop screenshots or paste log output here... + + - type: dropdown + attributes: + label: Affected Component + description: Which component of Dequel is affected? + multiple: true + options: + - API (Backend) + - Web Dashboard (Frontend) + - Docs + - CLI / Install Script + - Docker / Deployment + - Build System (Railpack / BuildKit) + - Caddy / Ingress + - Monitoring (Prometheus / Grafana / Loki) + - Other + validations: + required: true + + - type: input + attributes: + label: Dequel Version + description: What version of Dequel are you running? (Check `VERSION` or `scripts/dequel status`) + placeholder: e.g. v0.1.0 + validations: + required: true + + - type: textarea + attributes: + label: Environment + description: | + Examples: + - **OS**: Ubuntu 22.04 + - **Docker**: 24.0.7 + - **Bun**: 1.1.0 + - **Browser**: Chrome 125 + - **Deployment**: Docker Compose / Bare metal + value: | + - OS: + - Docker version: + - Bun version (if relevant): + - Browser (if relevant): + validations: + required: false + + - type: textarea + attributes: + label: Additional Context + description: Add any other context about the problem here (e.g. related issues, workarounds, research). + placeholder: Any additional context... + + - type: markdown + attributes: + value: | + --- + Before submitting, please ensure you've filled out all required fields marked with *. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..9d933a6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Dequel Documentation + url: https://dequel.intrep.xyz + about: Check the documentation before opening an issue. + - name: Discussions + url: https://github.com/Lftobs/dequel/discussions + about: Please ask questions and start discussions here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..12b1830 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,88 @@ +name: Feature Request +description: Suggest an idea for Dequel +title: "[Feature]: " +labels: ["enhancement"] +projects: [] +assignees: [] +body: + - type: markdown + attributes: + value: | + Thanks for suggesting a feature for Dequel. Please describe your idea clearly so we can understand the use case. + + - type: checkboxes + attributes: + label: Is there an existing feature request for this? + description: Please search to see if a similar feature request already exists. + options: + - label: I have searched the existing issues and found nothing matching + required: true + + - type: textarea + attributes: + label: Problem Statement + description: Is your feature request related to a problem? Please describe what you're trying to solve. + placeholder: I'm frustrated when [...] because [...] / I wish I could [...] + validations: + required: true + + - type: textarea + attributes: + label: Proposed Solution + description: A clear and concise description of what you want to happen. Be as specific as possible. + placeholder: The system should [...]. This could work by [...] + validations: + required: true + + - type: textarea + attributes: + label: Alternative Solutions + description: A clear and concise description of any alternative solutions or features you've considered. + placeholder: I also considered [...], but it doesn't work because [...] + validations: + required: false + + - type: dropdown + attributes: + label: Affected Component + description: Which component of Dequel would this affect? + multiple: true + options: + - API (Backend) + - Web Dashboard (Frontend) + - Docs + - CLI / Install Script + - Build System (Railpack / BuildKit) + - Caddy / Ingress + - Monitoring (Prometheus / Grafana / Loki) + - Other + validations: + required: true + + - type: textarea + attributes: + label: Mockups / Examples + description: | + If applicable, add mockups, diagrams, or examples from other projects that demonstrate the feature. + Tip: You can attach images by clicking this area and dragging files in. + placeholder: Drag and drop images or paste links to references... + + - type: textarea + attributes: + label: Additional Context + description: Add any other context or screenshots about the feature request here. + placeholder: Any additional information... + + - type: checkboxes + attributes: + label: Would you be willing to contribute this feature? + description: Let us know if you'd like to help implement this yourself. + options: + - label: Yes, I'd be happy to submit a PR for this feature + required: false + + - type: markdown + attributes: + value: | + --- + Before submitting, please ensure you've filled out all required fields marked with *. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..c5bbfa3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,43 @@ +## Description + +Please provide a summary of the changes and the motivation behind them. What problem does this PR solve? + +Fixes #(issue) + +## Type of Change + +- [ ] Bug fix (non-breaking change that fixes an issue) +- [ ] New feature (non-breaking change that adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Refactor (no functional changes) +- [ ] CI / Build / Tooling +- [ ] Other (please describe): + +## How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. + +- [ ] Existing tests pass (`bun test` in `apps/api/`) +- [ ] New tests added (if applicable) +- [ ] Manual testing performed (describe steps) + +## Checklist + +- [ ] My code follows the project's code style (no comments, named exports, functional components, etc.) +- [ ] I have read the [contributing guidelines](../CONTRIBUTING.md) +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have updated the documentation (if applicable) +- [ ] My changes generate no new warnings or lint errors +- [ ] I have run `bun test` in `apps/api/` and all tests pass +- [ ] I have synced the VERSION file if needed (`bun run sync-versions`) + +## Screenshots (if applicable) + +| Before | After | +|--------|-------| +| (insert here) | (insert here) | + +## Additional Context + +Add any other context about the PR here (e.g., migration notes, deployment considerations, rollback strategy). diff --git a/.github/workflows/vuln-scan.yml b/.github/workflows/vuln-scan.yml new file mode 100644 index 0000000..1de96cb --- /dev/null +++ b/.github/workflows/vuln-scan.yml @@ -0,0 +1,42 @@ +name: Security Scans + +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: security-scans-${{ github.ref }} + cancel-in-progress: true + +jobs: + forbidden-pattern-scan: + name: Forbidden Pattern Scan + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run forbidden pattern scan + run: bash scripts/workflow/forbidden-pattern-scan.sh "${{ github.workspace }}" + + lazarus-scanner: + name: Lazarus Scanner + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Download lazarus_scanner.py + run: | + curl -fsSL -o lazarus_scanner.py \ + https://raw.githubusercontent.com/hngprojects/lazarus-scanner/main/lazarus_scanner.py + [ -s lazarus_scanner.py ] || { echo "FAIL: lazarus_scanner.py is empty or missing"; exit 1; } + + - name: Run Lazarus scanner + run: python3 lazarus_scanner.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7ccfe82 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,110 @@ +# Contributing to Dequel + +Thank you for your interest in contributing! Here's how to get started. + +## Getting Started + +1. Fork and clone the repo +2. Install dependencies: `bun install` +3. Read [`AGENTS.md`](./AGENTS.md) for the full architecture and conventions + +## Reporting Bugs + +Open a [Bug Report](https://github.com/Lftobs/dequel/issues/new?template=bug_report.yml). Include: + +- Steps to reproduce +- Expected vs actual behavior +- Dequel version (`VERSION` file or `scripts/dequel status`) +- Environment details (OS, Docker version, browser if relevant) + +## Suggesting Features + +Open a [Feature Request](https://github.com/Lftobs/dequel/issues/new?template=feature_request.yml). Describe: + +- The problem you're solving +- Your proposed solution +- Any alternatives considered +- Whether you'd like to implement it yourself + +## Development + +### Running Locally + +```bash +# API (port 3001) +bun apps/api/src/index.ts + +# Web dashboard (port 3000) +bun apps/web/src/main.tsx + +# Docs (port 4321) +bun apps/docs/src/main.tsx +``` + +### Code Conventions + +- **No comments** in source code unless absolutely necessary +- **Named exports** over default exports +- **Functional components + hooks** in React +- **Tailwind CSS** for styling (web and docs) +- Max ~500 lines per file — split into feature-grouped directories +- `set -euo pipefail` in all bash scripts + +### Database Migrations + +```bash +# Generate migration from schema changes +bunx drizzle-kit generate + +# Push schema directly (dev only) +bunx drizzle-kit push +``` + +### Testing + +```bash +# API tests +bun test +``` + +Always run `bun test` in `apps/api/` before committing API changes. + +### Versioning + +```bash +# Bump version across the codebase +./bump.sh v0.2.0 +``` + +This updates `VERSION`, all `package.json` files, and optionally adds a changelog entry. + +## Pull Requests + +1. Create a PR from your fork using the [PR template](./.github/PULL_REQUEST_TEMPLATE.md) +2. Ensure all tests pass (`bun test`) +3. Keep changes focused — one feature/fix per PR +4. Update documentation if your change affects user-facing behavior +5. If changing API behavior, update the docs site content + +### PR Checklist + +- [ ] Tests pass (`bun test` in `apps/api/`) +- [ ] No new warnings or lint errors +- [ ] Documentation updated (if applicable) +- [ ] Version synced (`bun run sync-versions`) if `VERSION` changed +- [ ] Follows code conventions (no comments, named exports, etc.) + +## Release Process + +Maintainers cut releases by tagging: + +```bash +git tag vX.Y.Z +git push origin vX.Y.Z +``` + +CI builds Docker images to `ghcr.io/lftobs/dequel/{api,web}:X.Y.Z` and creates a GitHub Release. + +## Questions? + +Open a [Discussion](https://github.com/Lftobs/dequel/discussions) for questions and community support. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..925f81e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Lftobs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bump.sh b/bump.sh new file mode 120000 index 0000000..526dabc --- /dev/null +++ b/bump.sh @@ -0,0 +1 @@ +scripts/workflow/bump.sh \ No newline at end of file diff --git a/infra/monitoring/grafana/dashboards/dashboards.yml b/infra/monitoring/grafana/dashboards/dashboards.yml new file mode 100644 index 0000000..dd16c25 --- /dev/null +++ b/infra/monitoring/grafana/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "Dequel" + orgId: 1 + folder: "" + type: file + disableDeletion: true + editable: false + options: + path: /etc/grafana/provisioning/dashboards diff --git a/infra/monitoring/grafana/dashboards/deployed-apps.json b/infra/monitoring/grafana/dashboards/deployed-apps.json new file mode 100644 index 0000000..dc9fc72 --- /dev/null +++ b/infra/monitoring/grafana/dashboards/deployed-apps.json @@ -0,0 +1,157 @@ +{ + "title": "Dequel — Deployed Apps", + "uid": "dequel-deployed-apps", + "tags": ["dequel"], + "schemaVersion": 39, + "version": 1, + "timezone": "browser", + "refresh": "30s", + "templating": { + "list": [ + { + "name": "container", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(container_cpu_usage_seconds_total{job=\"cadvisor\"}, name)", + "refId": "container-var" + }, + "regex": "/([^/]+$)", + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "refresh": 1, + "hide": 0, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Resource Usage", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "id": 1, + "type": "timeseries", + "title": "CPU Usage", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { "mode": "normal" }, + "fillOpacity": 30, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(container_cpu_usage_seconds_total{name=~\".*${container:regex}$\"}[$__rate_interval])", + "legendFormat": "{{name}}", + "refId": "A" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Memory Usage", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "stacking": { "mode": "normal" }, + "fillOpacity": 30, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_working_set_bytes{name=~\".*${container:regex}$\"}", + "legendFormat": "{{name}}", + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 } + }, + { + "id": 3, + "type": "logs", + "title": "Container Logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 11 }, + "options": { + "showLabels": true, + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{container=~\"${container:regex}\"}", + "refId": "A" + } + ] + } + ] +} diff --git a/scripts/workflow/bump.sh b/scripts/workflow/bump.sh new file mode 100755 index 0000000..fdb7335 --- /dev/null +++ b/scripts/workflow/bump.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" +VERSION_FILE="$ROOT_DIR/VERSION" + +validate_version() { + local version="$1" + if ! echo "$version" | grep -qP '^\d+\.\d+\.\d+$'; then + echo "Error: Version must be in format MAJOR.MINOR.PATCH (e.g. 0.1.1). Got: $version" + exit 1 + fi +} + +print_header() { + local current="$1" + local new="$2" + echo -e "${CYAN}Dequel Version Bump${NC}" + echo -e "${YELLOW}Current version:${NC} $current" + echo -e "${YELLOW}New version:${NC} $new" + echo "" +} + +confirm_bump() { + read -p "Proceed with bump? (y/N) " -n 1 -r + echo + [[ $REPLY =~ ^[Yy]$ ]] +} + +update_version_file() { + local version="$1" + echo "$version" > "$VERSION_FILE" + echo -e "${GREEN}✓${NC} Updated VERSION → $version" +} + +sync_package_jsons() { + (cd "$ROOT_DIR" && bun run sync-versions) + echo -e "${GREEN}✓${NC} Synced package.json files" +} + +add_changelog_entry() { + local version="$1" + echo "" + echo -e "${CYAN}Changelog entry${NC}" + echo -e "Format: ${YELLOW}# Changelog entry title (empty to skip)${NC}" + echo "Example: '### Added' or '### Fixed'" + read -p "Section header: " -r header + if [ -z "$header" ]; then + return 1 + fi + echo "Enter bullet points (one per line). Press Ctrl+D when done:" + bullets=$(cat) + if [ -z "$bullets" ]; then + return 1 + fi + local date + date=$(date +%Y-%m-%d) + local entry="\n## [$version] - $date\n\n$header\n" + while IFS= read -r line; do + if [ -n "$line" ]; then + entry="$entry\n- $line" + fi + done <<< "$bullets" + sed -i "2a\\$entry" "$ROOT_DIR/CHANGELOG.md" + echo -e "${GREEN}✓${NC} Added changelog entry" +} + +print_summary() { + local version="$1" + local changelog_updated="$2" + echo "" + echo -e "${CYAN}=== Summary ===${NC}" + echo -e " VERSION: ${GREEN}$version${NC}" + echo -e " package.json: ${GREEN}synced${NC}" + if [ "$changelog_updated" = true ]; then + echo -e " CHANGELOG: ${GREEN}updated${NC}" + else + echo -e " CHANGELOG: ${YELLOW}skipped${NC}" + fi + echo "" + echo -e "Next steps:" + echo -e " ${YELLOW}1. Review CHANGELOG.md${NC}" + echo -e " ${YELLOW}2. Commit: git add -A && git commit -m \"chore: bump to v$version\"${NC}" + echo -e " ${YELLOW}3. Tag: git tag v$version${NC}" + echo -e " ${YELLOW}4. Push: git push origin main --tags${NC}" +} + +main() { + if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo "Example: $0 v0.1.1" + exit 1 + fi + + local version="${1#v}" + validate_version "$version" + + local current + current=$(cat "$VERSION_FILE") + + print_header "$current" "$version" + + if ! confirm_bump; then + echo "Aborted." + exit 0 + fi + + update_version_file "$version" + sync_package_jsons + + local changelog_updated=false + if add_changelog_entry "$version"; then + changelog_updated=true + fi + + print_summary "$version" "$changelog_updated" +} + +main "$@" diff --git a/scripts/workflow/forbidden-pattern-scan.sh b/scripts/workflow/forbidden-pattern-scan.sh new file mode 100644 index 0000000..9e623fd --- /dev/null +++ b/scripts/workflow/forbidden-pattern-scan.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${1:-.}" +cd "$ROOT" + +FORBIDDEN=$'global[\'!\']' +EXCLUDE=${2:-":(exclude).github/workflows/forbidden-pattern-scan.yml"} + +matches=$(git grep -lF "$FORBIDDEN" -- . "$EXCLUDE" || true) +if [[ -n "$matches" ]]; then + echo "::error::Blocked literal pattern detected in repository files." >&2 + echo "Affected file(s):" >&2 + printf '%s\n' "$matches" >&2 + echo "" >&2 + git grep -nF "$FORBIDDEN" -- . "$EXCLUDE" >&2 || true + exit 1 +fi + +echo "OK: no files contain the forbidden pattern." From fb2e810dcc4a0c7cefb995910e4e601eb37da1b1 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 16 Jun 2026 12:17:18 +0100 Subject: [PATCH 04/43] chore: simplify issue templates Replace YAML issue forms with standard Markdown templates to reduce overhead. --- .github/ISSUE_TEMPLATE/bug_report.md | 53 ++++++++++ .github/ISSUE_TEMPLATE/bug_report.yml | 111 --------------------- .github/ISSUE_TEMPLATE/config.yml | 8 -- .github/ISSUE_TEMPLATE/feature_request.md | 41 ++++++++ .github/ISSUE_TEMPLATE/feature_request.yml | 88 ---------------- 5 files changed, 94 insertions(+), 207 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml delete mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..cacf5e1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,53 @@ +--- +name: Bug Report +about: Report a bug to help us improve Dequel +title: "[Bug]: " +labels: bug, triage +assignees: "" +--- + +## Description + +A clear and concise description of what the bug is. + +## Steps to Reproduce + +1. Go to '...' +2. Click on '...' +3. Scroll down to '...' +4. See error + +## Expected Behavior + +What did you expect to happen? + +## Actual Behavior + +What actually happened? + +## Screenshots / Logs + +If applicable, add screenshots or relevant logs. + +## Environment + +- **Dequel Version**: (check `VERSION` or `scripts/dequel status`) +- **OS**: +- **Docker version**: +- **Bun version** (if relevant): +- **Browser** (if relevant): + +## Affected Component + +- [ ] API (Backend) +- [ ] Web Dashboard (Frontend) +- [ ] Docs +- [ ] CLI / Install Script +- [ ] Docker / Deployment +- [ ] Build System (Railpack / BuildKit) +- [ ] Caddy / Ingress +- [ ] Monitoring (Prometheus / Grafana / Loki) + +## Additional Context + +Add any other context, workarounds, or related issues. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml deleted file mode 100644 index 7b48faf..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Bug Report -description: File a bug report to help us improve Dequel -title: "[Bug]: " -labels: ["bug", "triage"] -projects: [] -assignees: [] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report. Please provide as much detail as possible. - - - type: checkboxes - attributes: - label: Is there an existing issue for this? - description: Please search to see if an issue already exists for the bug you encountered. - options: - - label: I have searched the existing issues and found nothing matching - required: true - - - type: textarea - attributes: - label: Describe the Bug - description: A clear and concise description of what the bug is. - placeholder: When I do X, Y happens instead of Z... - validations: - required: true - - - type: textarea - attributes: - label: Steps to Reproduce - description: Steps to reproduce the behavior. - placeholder: | - 1. Go to '...' - 2. Click on '...' - 3. Scroll down to '...' - 4. See error - validations: - required: true - - - type: textarea - attributes: - label: Expected Behavior - description: A clear and concise description of what you expected to happen. - placeholder: I expected X to happen... - validations: - required: true - - - type: textarea - attributes: - label: Screenshots / Logs - description: | - If applicable, add screenshots or relevant logs to help explain your problem. - Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. - placeholder: Drag and drop screenshots or paste log output here... - - - type: dropdown - attributes: - label: Affected Component - description: Which component of Dequel is affected? - multiple: true - options: - - API (Backend) - - Web Dashboard (Frontend) - - Docs - - CLI / Install Script - - Docker / Deployment - - Build System (Railpack / BuildKit) - - Caddy / Ingress - - Monitoring (Prometheus / Grafana / Loki) - - Other - validations: - required: true - - - type: input - attributes: - label: Dequel Version - description: What version of Dequel are you running? (Check `VERSION` or `scripts/dequel status`) - placeholder: e.g. v0.1.0 - validations: - required: true - - - type: textarea - attributes: - label: Environment - description: | - Examples: - - **OS**: Ubuntu 22.04 - - **Docker**: 24.0.7 - - **Bun**: 1.1.0 - - **Browser**: Chrome 125 - - **Deployment**: Docker Compose / Bare metal - value: | - - OS: - - Docker version: - - Bun version (if relevant): - - Browser (if relevant): - validations: - required: false - - - type: textarea - attributes: - label: Additional Context - description: Add any other context about the problem here (e.g. related issues, workarounds, research). - placeholder: Any additional context... - - - type: markdown - attributes: - value: | - --- - Before submitting, please ensure you've filled out all required fields marked with *. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 9d933a6..0000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Dequel Documentation - url: https://dequel.intrep.xyz - about: Check the documentation before opening an issue. - - name: Discussions - url: https://github.com/Lftobs/dequel/discussions - about: Please ask questions and start discussions here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..cd40805 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,41 @@ +--- +name: Feature Request +about: Suggest an idea for Dequel +title: "[Feature]: " +labels: enhancement +assignees: "" +--- + +## Problem Statement + +Is your feature request related to a problem? Please describe what you're trying to solve. + +## Proposed Solution + +A clear and concise description of what you want to happen. + +## Alternative Solutions + +Any alternative solutions or features you've considered. + +## Affected Component + +- [ ] API (Backend) +- [ ] Web Dashboard (Frontend) +- [ ] Docs +- [ ] CLI / Install Script +- [ ] Build System (Railpack / BuildKit) +- [ ] Caddy / Ingress +- [ ] Monitoring (Prometheus / Grafana / Loki) + +## Mockups / Examples + +If applicable, add mockups, diagrams, or examples from other projects. + +## Additional Context + +Add any other context or screenshots. + +## Would you like to implement this? + +- [ ] Yes, I'd be happy to submit a PR diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml deleted file mode 100644 index 12b1830..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Feature Request -description: Suggest an idea for Dequel -title: "[Feature]: " -labels: ["enhancement"] -projects: [] -assignees: [] -body: - - type: markdown - attributes: - value: | - Thanks for suggesting a feature for Dequel. Please describe your idea clearly so we can understand the use case. - - - type: checkboxes - attributes: - label: Is there an existing feature request for this? - description: Please search to see if a similar feature request already exists. - options: - - label: I have searched the existing issues and found nothing matching - required: true - - - type: textarea - attributes: - label: Problem Statement - description: Is your feature request related to a problem? Please describe what you're trying to solve. - placeholder: I'm frustrated when [...] because [...] / I wish I could [...] - validations: - required: true - - - type: textarea - attributes: - label: Proposed Solution - description: A clear and concise description of what you want to happen. Be as specific as possible. - placeholder: The system should [...]. This could work by [...] - validations: - required: true - - - type: textarea - attributes: - label: Alternative Solutions - description: A clear and concise description of any alternative solutions or features you've considered. - placeholder: I also considered [...], but it doesn't work because [...] - validations: - required: false - - - type: dropdown - attributes: - label: Affected Component - description: Which component of Dequel would this affect? - multiple: true - options: - - API (Backend) - - Web Dashboard (Frontend) - - Docs - - CLI / Install Script - - Build System (Railpack / BuildKit) - - Caddy / Ingress - - Monitoring (Prometheus / Grafana / Loki) - - Other - validations: - required: true - - - type: textarea - attributes: - label: Mockups / Examples - description: | - If applicable, add mockups, diagrams, or examples from other projects that demonstrate the feature. - Tip: You can attach images by clicking this area and dragging files in. - placeholder: Drag and drop images or paste links to references... - - - type: textarea - attributes: - label: Additional Context - description: Add any other context or screenshots about the feature request here. - placeholder: Any additional information... - - - type: checkboxes - attributes: - label: Would you be willing to contribute this feature? - description: Let us know if you'd like to help implement this yourself. - options: - - label: Yes, I'd be happy to submit a PR for this feature - required: false - - - type: markdown - attributes: - value: | - --- - Before submitting, please ensure you've filled out all required fields marked with *. From b4dda9effa385b6e4878810c2188090216bd45e0 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 17 Jun 2026 14:41:27 +0100 Subject: [PATCH 05/43] feat(monitoring): implement automatic project dashboard generation Add logic to create and update Grafana dashboards for projects, including CPU/memory usage and logs via Prometheus and Loki. Introduce a supporting Python script for orchestrating dashboard synchronization. --- .github/workflows/vuln-scan.yml | 16 +- apps/api/src/api/projects/index.ts | 1 + apps/api/src/orchestrator/pipeline.ts | 19 ++ apps/api/src/utils/config.ts | 12 ++ apps/api/src/utils/grafana.ts | 239 ++++++++++++++++++++++++ infra/monitoring/promtail-config.yml | 8 +- scripts/ensure-dashboards.py | 259 ++++++++++++++++++++++++++ scripts/ensure-dashboards.sh | 12 ++ scripts/install.sh | 4 +- scripts/workflow/bump.sh | 10 +- 10 files changed, 566 insertions(+), 14 deletions(-) create mode 100644 apps/api/src/utils/grafana.ts create mode 100755 scripts/ensure-dashboards.py create mode 100755 scripts/ensure-dashboards.sh diff --git a/.github/workflows/vuln-scan.yml b/.github/workflows/vuln-scan.yml index 1de96cb..c83645d 100644 --- a/.github/workflows/vuln-scan.yml +++ b/.github/workflows/vuln-scan.yml @@ -9,13 +9,19 @@ concurrency: group: security-scans-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + pull-requests: read + jobs: forbidden-pattern-scan: name: Forbidden Pattern Scan runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false - name: Run forbidden pattern scan run: bash scripts/workflow/forbidden-pattern-scan.sh "${{ github.workspace }}" @@ -25,7 +31,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false - name: Setup Python uses: actions/setup-python@v5 @@ -35,8 +43,8 @@ jobs: - name: Download lazarus_scanner.py run: | curl -fsSL -o lazarus_scanner.py \ - https://raw.githubusercontent.com/hngprojects/lazarus-scanner/main/lazarus_scanner.py - [ -s lazarus_scanner.py ] || { echo "FAIL: lazarus_scanner.py is empty or missing"; exit 1; } + https://raw.githubusercontent.com/hngprojects/lazarus-scanner/b1367e3a7a1463fbef90919867555a73f6acdf66/lazarus_scanner.py + echo "ddbb2181479f9a07d5859ceeac9160fd45084d28d95311ce49fc63fc1959e831 lazarus_scanner.py" | sha256sum --check || { echo "FAIL: SHA256 mismatch"; exit 1; } - name: Run Lazarus scanner run: python3 lazarus_scanner.py diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index 11473a7..0c5045d 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -7,6 +7,7 @@ import { getProjectById, updateProject, deleteProject, + listDomains, } from "../../db/repo"; import { tryRun, reloadCaddy } from "../../orchestrator/runtime"; import { removeFromCaddyRoute } from "../../utils/domain-verifier"; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 916b100..6f82a91 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -26,6 +26,7 @@ import { reloadCaddy, tryRun, } from "./runtime"; +import { ensureProjectDashboard } from "../utils/grafana"; const now = () => new Date() @@ -542,6 +543,24 @@ export class PipelineOrchestrator { }, ); + if (projectName) { + const dashSlug = projectName + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63); + const containerRegex = `${dashSlug}-.*`; + ensureProjectDashboard( + projectName, + containerRegex, + ).catch((e) => + console.warn( + "[Pipeline] Grafana dashboard creation failed:", + e, + ), + ); + } + if ( oldContainerName && deployment.projectId diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index ad4cdb9..f6c8dd4 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -109,4 +109,16 @@ export const config = { "GITHUB_WEBHOOK_SECRET", "", ), + grafanaUrl: withFile( + "GRAFANA_URL", + "http://grafana:3000", + ), + grafanaUser: withFile( + "GRAFANA_USER", + "admin", + ), + grafanaPass: withFile( + "GRAFANA_PASS", + "admin", + ), }; diff --git a/apps/api/src/utils/grafana.ts b/apps/api/src/utils/grafana.ts new file mode 100644 index 0000000..08304bd --- /dev/null +++ b/apps/api/src/utils/grafana.ts @@ -0,0 +1,239 @@ +import { config } from "./config"; + +interface GrafanaDashboard { + dashboard: { + title: string; + uid: string; + tags: string[]; + schemaVersion: number; + version: number; + timezone: string; + refresh: string; + panels: unknown[]; + }; + overwrite: boolean; +} + +let sessionCookie: string | null = null; +let sessionExpires = 0; + +async function grafanaLogin(): Promise { + if (sessionCookie && Date.now() < sessionExpires) { + return sessionCookie; + } + try { + const res = await fetch(`${config.grafanaUrl}/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + user: config.grafanaUser, + password: config.grafanaPass, + }), + }); + const setCookie = res.headers.get("set-cookie"); + if (!setCookie) return null; + sessionCookie = setCookie.split(";")[0]; + sessionExpires = Date.now() + 60 * 60 * 1000; + return sessionCookie; + } catch { + return null; + } +} + +async function grafanaPost( + path: string, + body: unknown, +): Promise { + const cookie = await grafanaLogin(); + if (!cookie) return null; + try { + const res = await fetch(`${config.grafanaUrl}/api${path}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Cookie: cookie, + }, + body: JSON.stringify(body), + }); + return await res.json(); + } catch { + return null; + } +} + +export async function ensureProjectDashboard( + projectName: string, + containerRegex: string, +): Promise { + const slug = projectName + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63); + + const dashboard: GrafanaDashboard = { + dashboard: { + title: `Dequel \u2014 ${projectName}`, + uid: `dequel-project-${slug}`, + tags: ["dequel", "project", slug], + schemaVersion: 39, + version: 1, + timezone: "browser", + refresh: "30s", + panels: [ + { + type: "row", + title: "Resource Usage", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 0 }, + }, + { + id: 1, + type: "timeseries", + title: "CPU Usage", + datasource: { type: "prometheus", uid: "prometheus" }, + gridPos: { h: 9, w: 12, x: 0, y: 1 }, + fieldConfig: { + defaults: { + unit: "short", + custom: { + stacking: { mode: "normal" }, + fillOpacity: 30, + lineWidth: 1, + }, + }, + overrides: [], + }, + options: { + legend: { + displayMode: "table", + placement: "right", + showLegend: true, + }, + tooltip: { mode: "multi" }, + }, + targets: [ + { + datasource: { type: "prometheus", uid: "prometheus" }, + expr: `rate(container_cpu_usage_seconds_total{name=~"${containerRegex}"}[$__rate_interval])`, + legendFormat: "{{name}}", + refId: "A", + }, + ], + }, + { + id: 2, + type: "timeseries", + title: "Memory Usage", + datasource: { type: "prometheus", uid: "prometheus" }, + gridPos: { h: 9, w: 12, x: 12, y: 1 }, + fieldConfig: { + defaults: { + unit: "bytes", + custom: { + stacking: { mode: "normal" }, + fillOpacity: 30, + lineWidth: 1, + }, + }, + overrides: [], + }, + options: { + legend: { + displayMode: "table", + placement: "right", + showLegend: true, + }, + tooltip: { mode: "multi" }, + }, + targets: [ + { + datasource: { type: "prometheus", uid: "prometheus" }, + expr: `container_memory_working_set_bytes{name=~"${containerRegex}"}`, + legendFormat: "{{name}}", + refId: "A", + }, + ], + }, + { + type: "row", + title: "Request Metrics", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 10 }, + }, + { + id: 4, + type: "timeseries", + title: "Request Rate", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 9, w: 24, x: 0, y: 11 }, + fieldConfig: { + defaults: { + unit: "reqps", + custom: { + fillOpacity: 30, + lineWidth: 1, + }, + }, + overrides: [], + }, + options: { + legend: { + displayMode: "table", + placement: "right", + showLegend: true, + }, + tooltip: { mode: "multi" }, + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum by(host) (count_over_time({container=~"${containerRegex}"} | json [5m]))`, + legendFormat: "{{host}}", + refId: "A", + }, + ], + }, + { + type: "row", + title: "Logs", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 20 }, + }, + { + id: 3, + type: "logs", + title: "Container Logs", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 12, w: 24, x: 0, y: 21 }, + options: { + showLabels: true, + showTime: true, + wrapLogMessage: true, + enableLogDetails: true, + dedupStrategy: "none", + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `{container=~"${containerRegex}"}`, + refId: "A", + }, + ], + }, + ], + }, + overwrite: true, + }; + + const result = await grafanaPost("/dashboards/db", dashboard); + if (result) { + console.log( + `[Grafana] Dashboard created/updated for ${projectName}`, + ); + } else { + console.warn( + `[Grafana] Failed to create dashboard for ${projectName}`, + ); + } +} diff --git a/infra/monitoring/promtail-config.yml b/infra/monitoring/promtail-config.yml index 0adcfd9..e4f1be2 100644 --- a/infra/monitoring/promtail-config.yml +++ b/infra/monitoring/promtail-config.yml @@ -3,7 +3,7 @@ server: grpc_listen_port: 0 positions: - filename: /tmp/positions.yaml + filename: /data/positions.yaml clients: - url: http://loki:3100/loki/api/v1/push @@ -13,10 +13,10 @@ scrape_configs: docker_sd_configs: - host: unix:///var/run/docker.sock refresh_interval: 15s + filters: + - name: network + values: ["dequel_net"] relabel_configs: - - source_labels: ['__meta_docker_container_network_dequel_net'] - regex: 'true' - action: keep - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' target_label: 'container' diff --git a/scripts/ensure-dashboards.py b/scripts/ensure-dashboards.py new file mode 100755 index 0000000..4c83a2d --- /dev/null +++ b/scripts/ensure-dashboards.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +import json +import os +import re +import subprocess +import sys + +GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost/grafana") +GRAFANA_USER = os.environ.get("GRAFANA_USER", "admin") +GRAFANA_PASS = os.environ.get("GRAFANA_PASS", "admin") +API_URL = os.environ.get("API_URL", "http://localhost/api") + +login = subprocess.run( + [ + "curl", + "-sk", + "-X", + "POST", + f"{GRAFANA_URL}/login", + "-H", + "Content-Type: application/json", + "-d", + json.dumps({"user": GRAFANA_USER, "password": GRAFANA_PASS}), + "-c", + "/tmp/grafana_cookies", + ], + capture_output=True, + text=True, +) +resp = json.loads(login.stdout) if login.stdout else {} +if "message" not in resp or resp.get("message") != "Logged in": + print(f"Grafana login failed: {login.stdout[:200]}") + sys.exit(1) +print("Logged into Grafana") + +result = subprocess.run( + ["curl", "-sk", f"{API_URL}/projects"], capture_output=True, text=True +) +if not result.stdout: + print("API returned empty response") + sys.exit(1) +projects = json.loads(result.stdout) +print(f"Found {len(projects)} projects") + +# Get running containers +running = set() +result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], capture_output=True, text=True +) +for c in result.stdout.strip().split("\n"): + if c: + running.add(c) + + +def grafana_post(path, data): + return subprocess.run( + [ + "curl", + "-sk", + "-X", + "POST", + "-b", + "/tmp/grafana_cookies", + "-H", + "Content-Type: application/json", + "-d", + json.dumps(data), + f"{GRAFANA_URL}/api{path}", + ], + capture_output=True, + text=True, + ) + + +created = 0 +for p in projects: + slug = p["name"].lower().replace(" ", "-").replace("_", "-") + slug = re.sub(r"[^a-z0-9-]", "", slug).strip("-")[:63] + + matching = sorted(c for c in running if c.startswith(slug + "-")) + if not matching: + print(f" Skip {p['name']}: no running containers") + continue + + container_regex = slug + "-.*" + print(f" {p['name']} (containers: {', '.join(matching)})") + + dashboard = { + "dashboard": { + "title": f"Dequel \u2014 {p['name']}", + "uid": f"dequel-project-{slug}", + "tags": ["dequel", "project", slug], + "schemaVersion": 39, + "version": 1, + "timezone": "browser", + "refresh": "30s", + "panels": [ + { + "type": "row", + "title": "Resource Usage", + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + }, + { + "id": 1, + "type": "timeseries", + "title": "CPU Usage", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 1}, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": {"mode": "normal"}, + "fillOpacity": 30, + "lineWidth": 1, + }, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": True, + }, + "tooltip": {"mode": "multi"}, + }, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": f'rate(container_cpu_usage_seconds_total{{name=~"{container_regex}"}}[$__rate_interval])', + "legendFormat": "{{name}}", + "refId": "A", + } + ], + }, + { + "id": 2, + "type": "timeseries", + "title": "Memory Usage", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 1}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "stacking": {"mode": "normal"}, + "fillOpacity": 30, + "lineWidth": 1, + }, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": True, + }, + "tooltip": {"mode": "multi"}, + }, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": f'container_memory_working_set_bytes{{name=~"{container_regex}"}}', + "legendFormat": "{{name}}", + "refId": "A", + } + ], + }, + { + "type": "row", + "title": "Request Metrics", + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 10}, + }, + { + "id": 4, + "type": "timeseries", + "title": "Request Rate", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 9, "w": 24, "x": 0, "y": 11}, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": {"fillOpacity": 30, "lineWidth": 1}, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": True, + }, + "tooltip": {"mode": "multi"}, + }, + "targets": [ + { + "datasource": {"type": "loki", "uid": "loki"}, + "expr": f'sum by(host) (count_over_time({{container=~"{container_regex}"}} | json [5m]))', + "legendFormat": "{{host}}", + "refId": "A", + } + ], + }, + { + "type": "row", + "title": "Logs", + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 20}, + }, + { + "id": 3, + "type": "logs", + "title": f"Container Logs", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 21}, + "options": { + "showLabels": True, + "showTime": True, + "wrapLogMessage": True, + "enableLogDetails": True, + "dedupStrategy": "none", + }, + "targets": [ + { + "datasource": {"type": "loki", "uid": "loki"}, + "expr": f'{{container=~"{container_regex}"}}', + "refId": "A", + } + ], + }, + ], + }, + "overwrite": True, + } + + r = grafana_post("/dashboards/db", dashboard) + try: + resp = json.loads(r.stdout) + if resp.get("status") == "success": + url = resp.get("url", "") + full_url = ( + f"https://localhost{url}" + if url.startswith("/grafana/") + else f"{GRAFANA_URL}{url}" + ) + print(f" OK: {full_url}") + created += 1 + else: + print(f" Error: {resp}") + except json.JSONDecodeError: + print(f" Failed: {r.stdout[:300]}") + +print(f"\nDone: {created} dashboard(s) created/updated") + +subprocess.run(["rm", "-f", "/tmp/grafana_cookies"], capture_output=True) diff --git a/scripts/ensure-dashboards.sh b/scripts/ensure-dashboards.sh new file mode 100755 index 0000000..a651554 --- /dev/null +++ b/scripts/ensure-dashboards.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +GRAFANA_URL="${1:-https://localhost/grafana}" +GRAFANA_USER="${2:-admin}" +GRAFANA_PASS="${3:-admin}" +API_URL="${4:-https://localhost/api}" + +export GRAFANA_URL GRAFANA_USER GRAFANA_PASS API_URL + +exec python3 "$SCRIPT_DIR/ensure-dashboards.py" \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh index f43e82b..d62acc1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -137,10 +137,8 @@ prompt_config() { return fi - mkdir -p "$INSTALL_DIR/data" - local ENC_KEY - ENC_KEY=$(openssl rand -hex 32 2>/dev/null || dd if=/dev/urandom bs=32 count=1 status=none 2>/dev/null | od -A n -t x1 | tr -d ' \n' || echo "dev-env-key-change-me") + ENC_KEY=$(openssl rand -hex 32 2>/dev/null || dd if=/dev/urandom bs=32 count=1 status=none 2>/dev/null | od -A n -t x1 | tr -d ' \n' || fail "Cannot generate encryption key — openssl and dd both failed") cat > "$INSTALL_DIR/data/dequel.json" </dev/null || echo "main") + + print_summary "$version" "$changelog_updated" "$current_branch" } main "$@" From 5674aeeead599115f78054b04770aab7d9671123 Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 19 Jun 2026 11:32:07 +0100 Subject: [PATCH 06/43] fix(orchestrator): improve container network reconciliation - Add forced network disconnection before starting containers to prevent network attachment conflicts. - Update monitoring configuration to handle Prometheus block corruption and improve Promtail persistence. - update contributor documentation to reflect current deployment and testing workflows. --- CONTRIBUTING.md | 39 +++++++++++------------ apps/api/src/orchestrator/runtime.ts | 6 +++- docker-compose.yml | 15 +++------ infra/monitoring/prometheus-entrypoint.sh | 24 ++++++++++++++ 4 files changed, 52 insertions(+), 32 deletions(-) create mode 100755 infra/monitoring/prometheus-entrypoint.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ccfe82..b4d8d89 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,12 +5,12 @@ Thank you for your interest in contributing! Here's how to get started. ## Getting Started 1. Fork and clone the repo -2. Install dependencies: `bun install` +2. Install dependencies: `bun install` (from repo root — needed for tests, version syncing, and local tooling) 3. Read [`AGENTS.md`](./AGENTS.md) for the full architecture and conventions ## Reporting Bugs -Open a [Bug Report](https://github.com/Lftobs/dequel/issues/new?template=bug_report.yml). Include: +Open a [Bug Report](https://github.com/Lftobs/dequel/issues/new?template=bug_report.md). Include: - Steps to reproduce - Expected vs actual behavior @@ -19,7 +19,7 @@ Open a [Bug Report](https://github.com/Lftobs/dequel/issues/new?template=bug_rep ## Suggesting Features -Open a [Feature Request](https://github.com/Lftobs/dequel/issues/new?template=feature_request.yml). Describe: +Open a [Feature Request](https://github.com/Lftobs/dequel/issues/new?template=feature_request.md). Describe: - The problem you're solving - Your proposed solution @@ -30,17 +30,14 @@ Open a [Feature Request](https://github.com/Lftobs/dequel/issues/new?template=fe ### Running Locally -```bash -# API (port 3001) -bun apps/api/src/index.ts - -# Web dashboard (port 3000) -bun apps/web/src/main.tsx +Run the full stack with Docker Compose: -# Docs (port 4321) -bun apps/docs/src/main.tsx +```bash +docker compose up -d --build ``` +This starts all services (API, Web, Caddy, BuildKit, Redis, Prometheus, Loki, Grafana). The dashboard is at `https://localhost`, API at `https://localhost/api`, Grafana at `https://localhost/grafana` (admin/admin). + ### Code Conventions - **No comments** in source code unless absolutely necessary @@ -52,27 +49,27 @@ bun apps/docs/src/main.tsx ### Database Migrations +Run from `apps/api/`: + ```bash -# Generate migration from schema changes +cd apps/api bunx drizzle-kit generate - -# Push schema directly (dev only) bunx drizzle-kit push ``` ### Testing +Run from `apps/api/`: + ```bash -# API tests -bun test +cd apps/api && bun test ``` -Always run `bun test` in `apps/api/` before committing API changes. +Always run tests before committing API changes. ### Versioning ```bash -# Bump version across the codebase ./bump.sh v0.2.0 ``` @@ -81,14 +78,14 @@ This updates `VERSION`, all `package.json` files, and optionally adds a changelo ## Pull Requests 1. Create a PR from your fork using the [PR template](./.github/PULL_REQUEST_TEMPLATE.md) -2. Ensure all tests pass (`bun test`) +2. Ensure all tests pass (`cd apps/api && bun test`) 3. Keep changes focused — one feature/fix per PR 4. Update documentation if your change affects user-facing behavior 5. If changing API behavior, update the docs site content ### PR Checklist -- [ ] Tests pass (`bun test` in `apps/api/`) +- [ ] Tests pass (`cd apps/api && bun test`) - [ ] No new warnings or lint errors - [ ] Documentation updated (if applicable) - [ ] Version synced (`bun run sync-versions`) if `VERSION` changed @@ -103,7 +100,7 @@ git tag vX.Y.Z git push origin vX.Y.Z ``` -CI builds Docker images to `ghcr.io/lftobs/dequel/{api,web}:X.Y.Z` and creates a GitHub Release. +CI builds Docker images to `ghcr.io/lftobs/dequel/{api,web}:X.Y.Z`, deploys docs to Vercel, and creates a GitHub Release. ## Questions? diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 22e7422..30e7f4b 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -75,6 +75,7 @@ const waitForRunningContainer = async ( await new Promise(r => setTimeout(r, 500)); } await onLog(`Container ${containerName} did not reach running state — attempting docker start`); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); await tryRun(dockerBin, ['start', containerName]); await tryRun(dockerBin, ['network', 'connect', config.dockerNetwork, containerName]); }; @@ -82,7 +83,10 @@ const waitForRunningContainer = async ( export const ensureContainerRunning = async (containerName: string) => { try { const status = (await run(dockerBin, ['inspect', '-f', '{{.State.Status}}', containerName])).trim(); - if (status !== 'running') await run(dockerBin, ['start', containerName]); + if (status !== 'running') { + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await run(dockerBin, ['start', containerName]); + } await tryRun(dockerBin, ['network', 'connect', config.dockerNetwork, containerName]); } catch (error) { console.error(`Failed to reconcile container ${containerName}:`, error); diff --git a/docker-compose.yml b/docker-compose.yml index 18a28c9..dbb51b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -164,19 +164,12 @@ services: prometheus: image: prom/prometheus:latest stop_grace_period: 60s + entrypoint: [] command: - sh - - -c - - | - promtool tsdb rebuild-index /prometheus 2>/dev/null || true - exec prometheus \ - --config.file=/etc/prometheus/prometheus.yml \ - --storage.tsdb.path=/prometheus \ - --web.console.libraries=/usr/share/prometheus/console_libraries \ - --web.console.templates=/usr/share/prometheus/consoles \ - --storage.tsdb.retention.time=30d \ - --storage.tsdb.wal-compression + - /entrypoint.sh volumes: + - ./infra/monitoring/prometheus-entrypoint.sh:/entrypoint.sh:ro - ./infra/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus depends_on: @@ -233,6 +226,7 @@ services: - ./infra/monitoring/promtail-config.yml:/etc/promtail/promtail-config.yml:ro - /var/run/docker.sock:/var/run/docker.sock - /var/lib/docker/containers:/var/lib/docker/containers:ro + - promtail-data:/data depends_on: loki: condition: service_started @@ -284,3 +278,4 @@ volumes: prometheus-data: loki-data: grafana-data: + promtail-data: diff --git a/infra/monitoring/prometheus-entrypoint.sh b/infra/monitoring/prometheus-entrypoint.sh new file mode 100755 index 0000000..f87ac6e --- /dev/null +++ b/infra/monitoring/prometheus-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +for block in /prometheus/01*/; do + if [ -d "$block" ]; then + missing="" + [ ! -f "$block/index" ] && missing="$missing index" + [ ! -f "$block/meta.json" ] && missing="$missing meta.json" + [ ! -d "$block/chunks" ] && missing="$missing chunks" + if [ -n "$missing" ]; then + echo "warning: corrupted block $(basename "$block") (missing:$missing), moving to quarantine" + mkdir -p /prometheus/quarantine + mv "$block" /prometheus/quarantine/ + fi + fi +done + +exec prometheus \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --web.console.templates=/usr/share/prometheus/consoles \ + --storage.tsdb.retention.time=30d \ + --storage.tsdb.wal-compression From 76970847089dc3bc4eb9940bc1c396bb77880493 Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 19 Jun 2026 12:24:41 +0100 Subject: [PATCH 07/43] chore: bump version to 0.1.1 --- .gitignore | 2 + CHANGELOG.md | 25 ++++++++ VERSION | 2 +- apps/api/package.json | 2 +- apps/docs/package.json | 2 +- apps/web/package.json | 2 +- bump.sh | 1 - scripts/workflow/bump.sh | 128 --------------------------------------- 8 files changed, 31 insertions(+), 133 deletions(-) delete mode 120000 bump.sh delete mode 100755 scripts/workflow/bump.sh diff --git a/.gitignore b/.gitignore index 3b0d81f..4578e81 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ infra/caddy/routes/ bugs apps/api/index docker-compose.yml +bump.sh +scripts/workflow/bump.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f781212..c2c89aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.1] - 2026-06-19 + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup; Promtail scoped to `dequel_net` network; Grafana datasources use stable UIDs for reliable dashboard provisioning +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers, preventing Docker network ID changes from breaking deployments + +### Documentation + +- Installation guide, quickstart, and system configuration docs updated for `CADDY_BASE_DOMAIN` + ## [0.1.0] - 2026-06-08 ### Added diff --git a/VERSION b/VERSION index 6e8bf73..17e51c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0 +0.1.1 diff --git a/apps/api/package.json b/apps/api/package.json index 210da08..66d9c11 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -1,6 +1,6 @@ { "name": "dequel-api", - "version": "0.1.0", + "version": "0.1.1", "private": true, "type": "module", "scripts": { diff --git a/apps/docs/package.json b/apps/docs/package.json index cc2b340..136545a 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -1,7 +1,7 @@ { "name": "dequel-docs", "type": "module", - "version": "0.1.0", + "version": "0.1.1", "scripts": { "dev": "astro dev", "start": "astro dev", diff --git a/apps/web/package.json b/apps/web/package.json index bc71984..cea64d6 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "dequel-web", - "version": "0.1.0", + "version": "0.1.1", "private": true, "type": "module", "scripts": { diff --git a/bump.sh b/bump.sh deleted file mode 120000 index 526dabc..0000000 --- a/bump.sh +++ /dev/null @@ -1 +0,0 @@ -scripts/workflow/bump.sh \ No newline at end of file diff --git a/scripts/workflow/bump.sh b/scripts/workflow/bump.sh deleted file mode 100755 index e75bbd1..0000000 --- a/scripts/workflow/bump.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -CYAN='\033[0;36m' -NC='\033[0m' - -ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" -VERSION_FILE="$ROOT_DIR/VERSION" - -validate_version() { - local version="$1" - if ! echo "$version" | grep -qP '^\d+\.\d+\.\d+$'; then - echo "Error: Version must be in format MAJOR.MINOR.PATCH (e.g. 0.1.1). Got: $version" - exit 1 - fi -} - -print_header() { - local current="$1" - local new="$2" - echo -e "${CYAN}Dequel Version Bump${NC}" - echo -e "${YELLOW}Current version:${NC} $current" - echo -e "${YELLOW}New version:${NC} $new" - echo "" -} - -confirm_bump() { - read -p "Proceed with bump? (y/N) " -n 1 -r - echo - [[ $REPLY =~ ^[Yy]$ ]] -} - -update_version_file() { - local version="$1" - echo "$version" > "$VERSION_FILE" - echo -e "${GREEN}✓${NC} Updated VERSION → $version" -} - -sync_package_jsons() { - (cd "$ROOT_DIR" && bun run sync-versions) - echo -e "${GREEN}✓${NC} Synced package.json files" -} - -add_changelog_entry() { - local version="$1" - echo "" - echo -e "${CYAN}Changelog entry${NC}" - echo -e "Format: ${YELLOW}# Changelog entry title (empty to skip)${NC}" - echo "Example: '### Added' or '### Fixed'" - read -p "Section header: " -r header - if [ -z "$header" ]; then - return 1 - fi - echo "Enter bullet points (one per line). Press Ctrl+D when done:" - bullets=$(cat) - if [ -z "$bullets" ]; then - return 1 - fi - local date - date=$(date +%Y-%m-%d) - local entry="\n## [$version] - $date\n\n$header\n" - while IFS= read -r line; do - if [ -n "$line" ]; then - entry="$entry\n- $line" - fi - done <<< "$bullets" - sed -i'' "2a\\$entry" "$ROOT_DIR/CHANGELOG.md" - echo -e "${GREEN}✓${NC} Added changelog entry" -} - -print_summary() { - local version="$1" - local changelog_updated="$2" - local branch="${3:-main}" - echo "" - echo -e "${CYAN}=== Summary ===${NC}" - echo -e " VERSION: ${GREEN}$version${NC}" - echo -e " package.json: ${GREEN}synced${NC}" - if [ "$changelog_updated" = true ]; then - echo -e " CHANGELOG: ${GREEN}updated${NC}" - else - echo -e " CHANGELOG: ${YELLOW}skipped${NC}" - fi - echo "" - echo -e "Next steps:" - echo -e " ${YELLOW}1. Review CHANGELOG.md${NC}" - echo -e " ${YELLOW}2. Commit: git add -A && git commit -m \"chore: bump to v$version\"${NC}" - echo -e " ${YELLOW}3. Tag: git tag v$version${NC}" - echo -e " ${YELLOW}4. Push: git push origin $branch --tags${NC}" -} - -main() { - if [ $# -ne 1 ]; then - echo "Usage: $0 " - echo "Example: $0 v0.1.1" - exit 1 - fi - - local version="${1#v}" - validate_version "$version" - - local current - current=$(cat "$VERSION_FILE") - - print_header "$current" "$version" - - if ! confirm_bump; then - echo "Aborted." - exit 0 - fi - - update_version_file "$version" - sync_package_jsons - - local changelog_updated=false - if add_changelog_entry "$version"; then - changelog_updated=true - fi - - local current_branch - current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main") - - print_summary "$version" "$changelog_updated" "$current_branch" -} - -main "$@" From 3a6372ef0c6c02f915736d79e31073abe9d28ddc Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 19 Jun 2026 12:56:09 +0100 Subject: [PATCH 08/43] ci(release): extract version-specific notes for releases Update the release workflow to parse the changelog for the current version's notes instead of attaching the entire file. Also, ignore __pycache__ directories in the project. --- .github/workflows/release.yml | 10 +++++++++- .gitignore | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 581fc02..47be652 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,11 +74,19 @@ jobs: cp -r infra/monitoring "$TAR_DIR/infra/" cd "$TAR_DIR" && tar -czf "../dequel-config-${VERSION}.tar.gz" . + - name: Extract changelog entry for this version + run: | + VERSION="${{ steps.version.outputs.VERSION }}" + awk '/^## \['"$VERSION"'\] -/{flag=1; next} /^## \[/{if(flag) exit} flag' CHANGELOG.md > release-notes.md + if [ ! -s release-notes.md ]; then + echo "No changelog entry for v$VERSION, will use auto-generated notes" + fi + - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: name: v${{ steps.version.outputs.VERSION }} - body_path: CHANGELOG.md + body_path: release-notes.md generate_release_notes: true files: | scripts/install.sh diff --git a/.gitignore b/.gitignore index 4578e81..ce8189e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ apps/api/index docker-compose.yml bump.sh scripts/workflow/bump.sh +__pycache__ From a0aa7c0d9e339fa88be81ba12db354d488f44478 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 20 Jun 2026 02:06:49 +0100 Subject: [PATCH 09/43] chore(scripts): improve shell compatibility in installer --- scripts/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index d62acc1..dd9508a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -34,16 +34,16 @@ download_if_missing() { check_prerequisites() { header "Checking prerequisites" - if command -v docker &>/dev/null; then + if command -v docker >/dev/null 2>&1; then success "Docker found: $(docker --version)" else fail "Docker is not installed. See https://docs.docker.com/engine/install/" fi - if docker compose version &>/dev/null; then + if docker compose version >/dev/null 2>&1; then success "Docker Compose found: $(docker compose version)" COMPOSE_CMD="docker compose" - elif docker-compose --version &>/dev/null; then + elif docker-compose --version >/dev/null 2>&1; then success "docker-compose found: $(docker-compose --version)" warn "Consider upgrading to 'docker compose' (Docker Compose v2)" COMPOSE_CMD="docker-compose" From a6c69bc48349e2fed4a78c6f3760eaaa968e47d5 Mon Sep 17 00:00:00 2001 From: lftobs Date: Mon, 22 Jun 2026 05:09:33 +0100 Subject: [PATCH 10/43] feat(api): add PAM auth and token utilities - Add PAM-based login flow using a Python PAM verifier and a new pam-verify.py script - Implement token-based auth: sign/verify access tokens, rotate and store refresh tokens in SQLite - Add refresh_tokens table, Drizzle migration, and related utilities to manage tokens - Auto-generate and load a JWT secret on first API startup; initialize auth at boot - Expose /auth/login, /auth/logout, /auth/refresh, and /auth/me endpoints - Update Dockerfile to install Python3 for PAM script execution - Add frontend login page and route; integrate with auth client and layout - Add docs pages for authentication and installation; include new auth.md - Add unit tests for auth utilities and DB integrations - Update server-info endpoint to check base domain status --- apps/api/Dockerfile | 6 +- apps/api/src/api/auth/index.ts | 93 +++++++ apps/api/src/api/index.ts | 42 ++- apps/api/src/api/server-info/index.ts | 4 +- apps/api/src/db/__tests__/auth.test.ts | 95 +++++++ .../src/db/migrations/0001_refresh_tokens.sql | 8 + apps/api/src/db/migrations/meta/_journal.json | 11 +- apps/api/src/db/schema.ts | 9 + apps/api/src/index.ts | 6 + apps/api/src/utils/__tests__/auth.test.ts | 91 +++++++ apps/api/src/utils/auth.ts | 116 ++++++++ apps/api/src/utils/config.ts | 6 +- apps/api/src/utils/dns.ts | 30 +++ apps/api/src/utils/secrets.ts | 16 ++ apps/docs/src/content/docs/auth.md | 53 ++++ apps/docs/src/content/docs/installation.md | 12 + apps/web/src/api/client.ts | 18 +- apps/web/src/components/ConfigWarnings.tsx | 13 +- apps/web/src/components/Layout.tsx | 26 ++ apps/web/src/routes/Login.tsx | 247 ++++++++++++++++++ apps/web/src/routes/index.tsx | 9 +- docker-compose.yml | 3 + scripts/pam-verify.py | 127 +++++++++ 23 files changed, 1016 insertions(+), 25 deletions(-) create mode 100644 apps/api/src/api/auth/index.ts create mode 100644 apps/api/src/db/__tests__/auth.test.ts create mode 100644 apps/api/src/db/migrations/0001_refresh_tokens.sql create mode 100644 apps/api/src/utils/__tests__/auth.test.ts create mode 100644 apps/api/src/utils/auth.ts create mode 100644 apps/api/src/utils/secrets.ts create mode 100644 apps/docs/src/content/docs/auth.md create mode 100644 apps/web/src/routes/Login.tsx create mode 100644 scripts/pam-verify.py diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 4392ef5..867b354 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unzip \ tar \ gnupg \ + python3 \ && rm -rf /var/lib/apt/lists/* # Install Docker CLI + buildx plugin from Docker's official repo @@ -31,10 +32,11 @@ RUN curl -sSL https://mise.jdx.dev/install.sh | sh \ WORKDIR /app -COPY package.json ./ +COPY apps/api/package.json ./ RUN bun install -COPY src ./src +COPY apps/api/src ./src +COPY scripts ./scripts RUN mkdir -p /app/data /app/workspace /caddy/routes diff --git a/apps/api/src/api/auth/index.ts b/apps/api/src/api/auth/index.ts new file mode 100644 index 0000000..cab9e18 --- /dev/null +++ b/apps/api/src/api/auth/index.ts @@ -0,0 +1,93 @@ +import { Elysia } from "elysia"; +import { spawn } from "node:child_process"; +import { signAccessToken, verifyAccessToken, generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } from "../../utils/auth"; +import { config } from "../../utils/config"; +import { join } from "node:path"; + +const PAM_SCRIPT = "/app/scripts/pam-verify.py"; + +const callPam = (username: string, password: string): Promise<{ ok: boolean; username?: string; error?: string }> => + new Promise((resolve) => { + const proc = spawn("python3", [PAM_SCRIPT], { stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + proc.stdout.on("data", (chunk) => { stdout += String(chunk); }); + proc.stderr.on("data", (chunk) => { stderr += String(chunk); }); + proc.on("close", (code) => { + if (code !== 0) { + try { resolve(JSON.parse(stdout)); } + catch { resolve({ ok: false, error: stderr.trim() || "Authentication failed" }); } + return; + } + try { resolve(JSON.parse(stdout)); } + catch { resolve({ ok: false, error: "Invalid response from auth helper" }); } + }); + proc.stdin!.end(JSON.stringify({ username, password })); + }); + +const COOKIE_OPTS = { + path: "/", + httpOnly: true, + sameSite: "strict" as const, + secure: config.caddyBaseDomain !== "localhost", + maxAge: 7 * 24 * 60 * 60, +}; + +export const authRoutes = new Elysia() + .post("/auth/login", async ({ body, cookie: { dequel_session, dequel_refresh }, set }) => { + const { username, password } = body as { username?: string; password?: string }; + if (!username || !password) { + set.status = 400; + return { error: "Username and password required" }; + } + const result = await callPam(username, password); + if (!result.ok) { + set.status = 401; + return { error: result.error || "Authentication failed" }; + } + const accessToken = await signAccessToken(username); + const refreshToken = generateRefreshToken(); + await storeRefreshToken(username, refreshToken); + dequel_session.value = accessToken; + dequel_session.set(COOKIE_OPTS); + dequel_refresh.value = refreshToken; + dequel_refresh.set(COOKIE_OPTS); + return { ok: true, username }; + }) + .post("/auth/logout", async ({ cookie: { dequel_session, dequel_refresh } }) => { + const rt = dequel_refresh.value; + if (rt) { + try { await blacklistRefreshToken(rt); } catch {} + } + dequel_session.remove(); + dequel_refresh.remove(); + return { ok: true }; + }) + .post("/auth/refresh", async ({ cookie: { dequel_session, dequel_refresh }, set }) => { + const rt = dequel_refresh.value; + if (!rt) { + set.status = 401; + return { error: "No refresh token" }; + } + const username = await validateRefreshToken(rt); + if (!username) { + set.status = 401; + return { error: "Invalid or expired refresh token" }; + } + try { await blacklistRefreshToken(rt); } catch {} + const accessToken = await signAccessToken(username); + const newRefreshToken = generateRefreshToken(); + await storeRefreshToken(username, newRefreshToken); + dequel_session.value = accessToken; + dequel_session.set(COOKIE_OPTS); + dequel_refresh.value = newRefreshToken; + dequel_refresh.set(COOKIE_OPTS); + return { ok: true, username }; + }) + .get("/auth/me", async ({ cookie: { dequel_session } }) => { + const token = dequel_session.value; + if (!token) return { authenticated: false }; + const payload = await verifyAccessToken(token); + if (!payload) return { authenticated: false }; + return { authenticated: true, username: payload.sub }; + }); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index a47c08a..8d03eab 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,6 +1,7 @@ import { Elysia } from "elysia"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; +import { authRoutes } from "./auth"; import { databasesRoutes } from "./databases"; import { deploymentsRoutes } from "./deployments"; import { domainsRoutes } from "./domains"; @@ -15,27 +16,42 @@ import { serversRoutes } from "./servers"; import { volumesRoutes } from "./volumes"; import { settingsRoutes } from "./settings"; +const BYPASS_PATHS = new Set(["/api/auth/login", "/api/auth/logout", "/api/auth/refresh", "/api/auth/me", "/api/health"]); + const authMiddleware = (app: Elysia) => - app.onBeforeHandle(async ({ request, set }) => { - const authHeader = request.headers.get( - "authorization", - ); - if (!authHeader?.startsWith("Bearer ")) - return; - const token = authHeader.slice(7); - if (!token) return; - const { validateApiKey } = - await import("../db/repo"); - const key = await validateApiKey(token); - if (!key) { + app.onBeforeHandle(async ({ request, set, path }) => { + if (BYPASS_PATHS.has(path)) return; + + const cookie = request.headers.get("cookie") || ""; + const match = cookie.match(/(?:^|;\s*)dequel_session=([^;]+)/); + if (match) { + const { verifyAccessToken } = await import("../utils/auth"); + const payload = await verifyAccessToken(match[1]); + if (payload) return; set.status = 401; - return { error: "Invalid API key" }; + return { error: "Invalid session" }; } + + const authHeader = request.headers.get("authorization"); + if (authHeader?.startsWith("Bearer ")) { + const token = authHeader.slice(7); + if (token) { + const { validateApiKey } = await import("../db/repo"); + const key = await validateApiKey(token); + if (key) return; + set.status = 401; + return { error: "Invalid API key" }; + } + } + + set.status = 401; + return { error: "Authentication required" }; }); export const apiRoutes = new Elysia({ prefix: "/api", }) + .use(authRoutes) .use(authMiddleware) .use(healthRoutes) .use(projectsRoutes) diff --git a/apps/api/src/api/server-info/index.ts b/apps/api/src/api/server-info/index.ts index 9a163a4..c27604c 100644 --- a/apps/api/src/api/server-info/index.ts +++ b/apps/api/src/api/server-info/index.ts @@ -3,7 +3,7 @@ import { Elysia } from "elysia"; export const serverInfoRoutes = new Elysia().get( "/server/ip", async () => { - const { resolveServerIp } = await import("../../utils/dns"); - return { ip: await resolveServerIp() }; + const { checkBaseDomainStatus } = await import("../../utils/dns"); + return await checkBaseDomainStatus(); }, ); diff --git a/apps/api/src/db/__tests__/auth.test.ts b/apps/api/src/db/__tests__/auth.test.ts new file mode 100644 index 0000000..5b1c675 --- /dev/null +++ b/apps/api/src/db/__tests__/auth.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, mock, beforeAll, beforeEach, afterAll } from 'bun:test'; +import { Database } from 'bun:sqlite'; + +const TEST_SECRET = 'test-jwt-secret-for-testing-purposes-only'; + +let db: Database; + +const fileUrl = (path: string) => new URL(path, import.meta.url).toString(); +mock.module(fileUrl('../client.ts'), () => ({ + getDb: () => db, +})); + +beforeAll(async () => { + db = new Database(':memory:'); + db.run(` + CREATE TABLE refresh_tokens ( + id text PRIMARY KEY NOT NULL, + username text NOT NULL, + token_hash text NOT NULL UNIQUE, + expires_at text NOT NULL, + created_at text NOT NULL, + blacklisted_at text + ) + `); + const { initAuth } = await import('../../utils/auth'); + initAuth(TEST_SECRET); +}); + +beforeEach(() => { + db.run('DELETE FROM refresh_tokens'); +}); + +afterAll(() => { + db.close(); +}); + +describe('storeRefreshToken / validateRefreshToken', () => { + it('stores and validates a refresh token', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + const username = await validateRefreshToken(token); + expect(username).toBe('testuser'); + }); + + it('returns null for unknown token', async () => { + const { validateRefreshToken } = await import('../../utils/auth'); + const result = await validateRefreshToken('dqr_nonexistent'); + expect(result).toBeNull(); + }); +}); + +describe('blacklistRefreshToken', () => { + it('blacklists a refresh token', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + expect(await validateRefreshToken(token)).toBe('testuser'); + await blacklistRefreshToken(token); + expect(await validateRefreshToken(token)).toBeNull(); + }); + + it('does not affect other tokens when blacklisting one', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } = await import('../../utils/auth'); + const tokenA = generateRefreshToken(); + const tokenB = generateRefreshToken(); + await storeRefreshToken('user1', tokenA); + await storeRefreshToken('user2', tokenB); + await blacklistRefreshToken(tokenA); + expect(await validateRefreshToken(tokenA)).toBeNull(); + expect(await validateRefreshToken(tokenB)).toBe('user2'); + }); +}); + +describe('cleanupExpiredTokens', () => { + it('removes expired tokens', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken, cleanupExpiredTokens, hashToken } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + const tokenHash = hashToken(token); + db.run(`UPDATE refresh_tokens SET expires_at = '2000-01-01T00:00:00.000Z' WHERE token_hash = ?`, [tokenHash]); + await cleanupExpiredTokens(); + const remaining = db.query('SELECT COUNT(*) as c FROM refresh_tokens').get() as { c: number }; + expect(remaining.c).toBe(0); + }); + + it('keeps non-expired tokens', async () => { + const { generateRefreshToken, storeRefreshToken, cleanupExpiredTokens } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + await cleanupExpiredTokens(); + const remaining = db.query('SELECT COUNT(*) as c FROM refresh_tokens').get() as { c: number }; + expect(remaining.c).toBe(1); + }); +}); diff --git a/apps/api/src/db/migrations/0001_refresh_tokens.sql b/apps/api/src/db/migrations/0001_refresh_tokens.sql new file mode 100644 index 0000000..0c284c3 --- /dev/null +++ b/apps/api/src/db/migrations/0001_refresh_tokens.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS refresh_tokens ( + id text PRIMARY KEY NOT NULL, + username text NOT NULL, + token_hash text NOT NULL UNIQUE, + expires_at text NOT NULL, + created_at text NOT NULL, + blacklisted_at text +); diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index df2ce5b..8d4d821 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -2,12 +2,19 @@ "version": "6", "dialect": "sqlite", "entries": [ -{ - "idx": 0, + { + "idx": 0, "version": "6", "when": 1718000000000, "tag": "0000_baseline", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1782100000000, + "tag": "0001_refresh_tokens", + "breakpoints": true } ] } diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 1891d65..049e48a 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -152,6 +152,15 @@ export const servers = sqliteTable("servers", { updatedAt: text("updated_at").notNull(), }); +export const refreshTokens = sqliteTable("refresh_tokens", { + id: text().primaryKey(), + username: text().notNull(), + tokenHash: text("token_hash").notNull().unique(), + expiresAt: text("expires_at").notNull(), + createdAt: text("created_at").notNull(), + blacklistedAt: text("blacklisted_at"), +}); + export const apiKeys = sqliteTable("api_keys", { id: text().primaryKey(), name: text().notNull(), diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 3906e06..5fad630 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -12,11 +12,16 @@ import { serverManager } from './servers/manager'; import { startGitWatcher } from './git/watcher'; import { startDomainPolling } from './utils/domain-verifier'; import { alertEvaluator } from './monitoring/evaluator'; +import { loadOrCreateJwtSecret } from './utils/secrets'; +import { initAuth, cleanupExpiredTokens } from './utils/auth'; const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); await mkdir(config.caddyRoutesDir, { recursive: true }); + const jwtSecret = await loadOrCreateJwtSecret(dirname(config.databasePath)); + initAuth(jwtSecret); + await migrate(); await orchestrator.reconcileState(); orchestrator.startWorker(); @@ -25,6 +30,7 @@ const bootstrap = async () => { startGitWatcher(); startDomainPolling(); alertEvaluator.start(); + setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { requestsTotal: 0, diff --git a/apps/api/src/utils/__tests__/auth.test.ts b/apps/api/src/utils/__tests__/auth.test.ts new file mode 100644 index 0000000..af96521 --- /dev/null +++ b/apps/api/src/utils/__tests__/auth.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect, mock, beforeAll, beforeEach, afterAll } from 'bun:test'; +import { Database } from 'bun:sqlite'; + +const TEST_SECRET = 'test-jwt-secret-for-testing-purposes-only'; + +describe('JWT operations', () => { + beforeAll(async () => { + const { initAuth } = await import('../auth'); + initAuth(TEST_SECRET); + }); + + it('signs and verifies an access token', async () => { + const { signAccessToken, verifyAccessToken } = await import('../auth'); + const token = await signAccessToken('testuser'); + expect(typeof token).toBe('string'); + expect(token.split('.').length).toBe(3); + const payload = await verifyAccessToken(token); + expect(payload).not.toBeNull(); + expect(payload!.sub).toBe('testuser'); + expect(payload!.iat).toBeGreaterThan(0); + expect(payload!.exp).toBe(payload!.iat + 900); + }); + + it('rejects a tampered token', async () => { + const { signAccessToken, verifyAccessToken } = await import('../auth'); + const token = await signAccessToken('testuser'); + const parts = token.split('.'); + const tampered = ['bad', parts[1], parts[2]].join('.'); + expect(await verifyAccessToken(tampered)).toBeNull(); + }); + + it('rejects token with wrong secret', async () => { + const { signAccessToken, verifyAccessToken, initAuth } = await import('../auth'); + const token = await signAccessToken('testuser'); + initAuth('different-secret'); + const result = await verifyAccessToken(token); + expect(result).toBeNull(); + initAuth(TEST_SECRET); + }); + + it('rejects malformed token', async () => { + const { verifyAccessToken } = await import('../auth'); + expect(await verifyAccessToken('not-a-jwt')).toBeNull(); + expect(await verifyAccessToken('only.two.parts.here')).toBeNull(); + expect(await verifyAccessToken('')).toBeNull(); + }); + + it('rejects expired token', async () => { + const { verifyAccessToken } = await import('../auth'); + const parts = ['header', btoa(JSON.stringify({ + sub: 'testuser', + iat: 1000, + exp: 1, + })).replace(/=/g, ''), 'fakesig']; + const result = await verifyAccessToken(parts.join('.')); + expect(result).toBeNull(); + }); +}); + +describe('hashToken', () => { + it('produces consistent hashes', async () => { + const { hashToken } = await import('../auth'); + const h1 = hashToken('test-token'); + const h2 = hashToken('test-token'); + expect(h1).toBe(h2); + expect(h1.length).toBe(64); + }); + + it('produces different hashes for different tokens', async () => { + const { hashToken } = await import('../auth'); + const h1 = hashToken('token-a'); + const h2 = hashToken('token-b'); + expect(h1).not.toBe(h2); + }); +}); + +describe('generateRefreshToken', () => { + it('generates token with correct prefix', async () => { + const { generateRefreshToken } = await import('../auth'); + const token = generateRefreshToken(); + expect(token.startsWith('dqr_')).toBe(true); + expect(token.length).toBe(4 + 64); + }); + + it('generates unique tokens', async () => { + const { generateRefreshToken } = await import('../auth'); + const t1 = generateRefreshToken(); + const t2 = generateRefreshToken(); + expect(t1).not.toBe(t2); + }); +}); diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.ts new file mode 100644 index 0000000..fadbba0 --- /dev/null +++ b/apps/api/src/utils/auth.ts @@ -0,0 +1,116 @@ +import { randomBytes } from 'node:crypto'; +import { getDrizzle } from '../db/drizzle'; +import { eq, and, sql } from 'drizzle-orm'; +import { refreshTokens } from '../db/schema'; + +let jwtSecret = ''; + +export const initAuth = (secret: string) => { + jwtSecret = secret; +}; + +const b64url = (buf: ArrayBuffer): string => + btoa(String.fromCharCode(...new Uint8Array(buf))) + .replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_'); + +const b64urlDecode = (str: string): ArrayBuffer => { + str = str.replace(/-/g, '+').replace(/_/g, '/'); + while (str.length % 4) str += '='; + return Uint8Array.from(atob(str), c => c.charCodeAt(0)).buffer; +}; + +const encoder = new TextEncoder(); + +const hmacSign = async (data: string): Promise => { + const key = await crypto.subtle.importKey( + 'raw', encoder.encode(jwtSecret), + { name: 'HMAC', hash: 'SHA-256' }, + false, ['sign'], + ); + const sig = await crypto.subtle.sign('HMAC', key, encoder.encode(data)); + return b64url(sig); +}; + +export interface JwtPayload { + sub: string; + iat: number; + exp: number; +} + +export const signAccessToken = async (username: string): Promise => { + const header = b64url(encoder.encode(JSON.stringify({ alg: 'HS256', typ: 'JWT' }))); + const now = Math.floor(Date.now() / 1000); + const payload = b64url(encoder.encode(JSON.stringify({ + sub: username, + iat: now, + exp: now + 900, + }))); + const sig = await hmacSign(`${header}.${payload}`); + return `${header}.${payload}.${sig}`; +}; + +export const verifyAccessToken = async (token: string): Promise => { + const parts = token.split('.'); + if (parts.length !== 3) return null; + const [header, payload, sig] = parts; + const expected = await hmacSign(`${header}.${payload}`); + if (sig !== expected) return null; + try { + const data = JSON.parse(new TextDecoder().decode(b64urlDecode(payload))); + if (data.exp && data.exp < Math.floor(Date.now() / 1000)) return null; + return data; + } catch { + return null; + } +}; + +export const hashToken = (token: string): string => { + const hash = Bun.CryptoHasher.hash('sha256', token); + return Buffer.from(hash).toString('hex'); +}; + +export const generateRefreshToken = (): string => + `dqr_${randomBytes(32).toString('hex')}`; + +export const storeRefreshToken = async (username: string, token: string): Promise => { + const drizzle = await getDrizzle(); + const tokenHash = hashToken(token); + const now = new Date().toISOString(); + const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString(); + const id = randomBytes(16).toString('hex'); + drizzle.insert(refreshTokens).values({ + id, + username, + tokenHash, + expiresAt, + createdAt: now, + }).run(); +}; + +export const validateRefreshToken = async (token: string): Promise => { + const drizzle = await getDrizzle(); + const tokenHash = hashToken(token); + const row = drizzle.select().from(refreshTokens) + .where(and( + eq(refreshTokens.tokenHash, tokenHash), + sql`${refreshTokens.blacklistedAt} IS NULL`, + sql`${refreshTokens.expiresAt} > datetime('now')`, + )) + .get(); + return row?.username ?? null; +}; + +export const blacklistRefreshToken = async (token: string): Promise => { + const drizzle = await getDrizzle(); + const tokenHash = hashToken(token); + const now = new Date().toISOString(); + drizzle.update(refreshTokens) + .set({ blacklistedAt: now }) + .where(eq(refreshTokens.tokenHash, tokenHash)) + .run(); +}; + +export const cleanupExpiredTokens = async (): Promise => { + const drizzle = await getDrizzle(); + drizzle.run(sql`DELETE FROM refresh_tokens WHERE expires_at < datetime('now')`); +}; diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index f6c8dd4..d311de9 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -24,9 +24,6 @@ const withFile = ( }; const SYSTEM = { - databasePath: "/app/data/dequel.db", - workspaceRoot: "/app/workspace", - caddyRoutesDir: "/caddy/routes", dockerNetwork: "dequel_net", buildkitHost: "tcp://buildkit:1234", redisUrl: "redis://redis:6379", @@ -34,6 +31,9 @@ const SYSTEM = { export const config = { ...SYSTEM, + databasePath: withFile("DATABASE_PATH", "/app/data/dequel.db"), + workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), + caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), port: withFile( "PORT", "17474", diff --git a/apps/api/src/utils/dns.ts b/apps/api/src/utils/dns.ts index 831d10d..0097c91 100644 --- a/apps/api/src/utils/dns.ts +++ b/apps/api/src/utils/dns.ts @@ -1,4 +1,5 @@ import { promises as dns } from 'node:dns'; +import { config } from './config'; export const resolveServerIp = async (): Promise => { try { @@ -9,6 +10,35 @@ export const resolveServerIp = async (): Promise => { } }; +export type BaseDomainStatus = { + ip: string; + baseDomain: string; + resolves: boolean; + url: string; +}; + +export const checkBaseDomainStatus = async (): Promise => { + const ip = await resolveServerIp(); + const baseDomain = config.caddyBaseDomain; + const isLocalhost = baseDomain === 'localhost'; + + let resolves = false; + if (!isLocalhost) { + try { + const addresses = await dns.resolve4(baseDomain); + resolves = addresses.includes(ip); + } catch {} + } else { + resolves = true; + } + + const url = isLocalhost || resolves + ? `${isLocalhost ? 'http' : 'https'}://${baseDomain}${isLocalhost ? ':80' : ''}` + : `http://${ip}`; + + return { ip, baseDomain, resolves, url }; +}; + export const validateDomain = async ( domain: string, serverIp: string, diff --git a/apps/api/src/utils/secrets.ts b/apps/api/src/utils/secrets.ts new file mode 100644 index 0000000..e37e112 --- /dev/null +++ b/apps/api/src/utils/secrets.ts @@ -0,0 +1,16 @@ +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { randomBytes } from 'node:crypto'; + +export const loadOrCreateJwtSecret = async (dataDir: string): Promise => { + const path = join(dataDir, '.jwt_secret'); + try { + const existing = await readFile(path, 'utf8'); + return existing.trim(); + } catch { + await mkdir(dirname(path), { recursive: true }); + const secret = randomBytes(32).toString('hex'); + await writeFile(path, secret + '\n', { mode: 0o600 }); + return secret; + } +}; diff --git a/apps/docs/src/content/docs/auth.md b/apps/docs/src/content/docs/auth.md new file mode 100644 index 0000000..7bb9fad --- /dev/null +++ b/apps/docs/src/content/docs/auth.md @@ -0,0 +1,53 @@ +--- +title: Authentication & Access Control +category: Networking & Security +description: Secure your Dequel dashboard with PAM-based authentication, session management, and API keys. +slug: auth +--- + +Dequel authenticates users against the **Linux system user database** via PAM (Pluggable Authentication Modules). Only users who are members of the `dequel` group can sign in. + +## User Setup + +Create a Linux user and add them to the `dequel` group: + +```bash +sudo useradd -m -s /bin/bash +sudo passwd +sudo usermod -aG dequel +``` + +The user signs into the Dequel dashboard with their Linux username and password. + +## Session Management + +- **Access tokens**: Short-lived (15 min) HMAC-SHA256 JWTs stored in an `httpOnly` cookie. +- **Refresh tokens**: Long-lived (7 days) random tokens stored in the database. Automatically rotated on refresh. +- **Token blacklisting**: Logging out or refreshing invalidates the old refresh token immediately. +- **JWT secret**: Auto-generated on first API startup, stored at `data/.jwt_secret` (0600 permissions). Deleting this file invalidates all sessions. + +## API Keys + +For programmatic access (CI/CD, CLI tools), Dequel uses pre-shared API keys: + +1. Go to **Settings → API Keys** in the dashboard. +2. Create a new key — the full token is shown once. +3. Use it as a Bearer token in the `Authorization` header: + +```bash +curl -H "Authorization: Bearer dqk_" https://dequel.example.com/api/projects +``` + +API keys are scoped to the entire platform with the same permissions as the user who created them. + +## Endpoints + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `POST` | `/api/auth/login` | None | Sign in with username/password | +| `POST` | `/api/auth/logout` | Session | Sign out, blacklist refresh token | +| `POST` | `/api/auth/refresh` | Session | Rotate refresh token | +| `GET` | `/api/auth/me` | None | Check current session status | +| `GET` | `/api/health` | None | Health check | + +The web dashboard redirects to `/login` when no valid session is detected. diff --git a/apps/docs/src/content/docs/installation.md b/apps/docs/src/content/docs/installation.md index 5fda364..1dc0662 100644 --- a/apps/docs/src/content/docs/installation.md +++ b/apps/docs/src/content/docs/installation.md @@ -34,6 +34,18 @@ dequel start Open `http://localhost` to access the dashboard (or the configured `CADDY_BASE_DOMAIN` in production). +## Post-Installation: User Setup + +Dequel authenticates against Linux system users in the `dequel` group. After install, create a user: + +```bash +sudo useradd -m -s /bin/bash +sudo passwd +sudo usermod -aG dequel +``` + +See [Authentication & Access Control](/docs/auth) for details on session management and API keys. + ## Manual Setup (no install script) If the installer fails, set up the platform manually with just Docker Compose and the config files: diff --git a/apps/web/src/api/client.ts b/apps/web/src/api/client.ts index 05e1c67..8438ae2 100644 --- a/apps/web/src/api/client.ts +++ b/apps/web/src/api/client.ts @@ -340,7 +340,23 @@ export const deleteScalingPolicy = ( // Server export const getServerIp = () => - apiFetch<{ ip: string }>("/server/ip"); + apiFetch<{ ip: string; baseDomain: string; resolves: boolean; url: string }>("/server/ip"); + +// Auth +export const login = (username: string, password: string) => + apiFetch<{ ok: boolean; username: string; error?: string }>("/auth/login", { + method: "POST", + body: JSON.stringify({ username, password }), + }); + +export const logout = () => + apiFetch<{ ok: boolean }>("/auth/logout", { method: "POST" }); + +export const refreshSession = () => + apiFetch<{ ok: boolean; username: string }>("/auth/refresh", { method: "POST" }); + +export const getMe = () => + apiFetch<{ authenticated: boolean; username?: string }>("/auth/me"); // Prometheus export const queryPrometheus = (query: string) => diff --git a/apps/web/src/components/ConfigWarnings.tsx b/apps/web/src/components/ConfigWarnings.tsx index 3113b87..7a0b4f3 100644 --- a/apps/web/src/components/ConfigWarnings.tsx +++ b/apps/web/src/components/ConfigWarnings.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; import { useQuery } from '@tanstack/react-query'; import { Button } from './ui/button'; -import { Mail, GitBranch, type LucideIcon } from 'lucide-react'; +import { Globe, Mail, GitBranch, type LucideIcon } from 'lucide-react'; import * as api from '../api/client'; export function ConfigWarnings() { @@ -13,6 +13,9 @@ export function ConfigWarnings() { const github = useQuery({ queryKey: ['github-integration'], queryFn: () => api.getGithubIntegration(), }); + const serverInfo = useQuery({ + queryKey: ['server-ip'], queryFn: () => api.getServerIp(), + }); const missing: { key: string; icon: LucideIcon; title: string; desc: string }[] = []; if (smtp.data && !smtp.data.configured) { @@ -31,6 +34,14 @@ export function ConfigWarnings() { desc: 'Connect a GitHub OAuth App to enable the repo picker and auto-deploy.', }); } + if (serverInfo.data && !serverInfo.data.resolves) { + missing.push({ + key: 'base-domain', + icon: Globe, + title: 'Base domain misconfigured', + desc: `The base domain ${serverInfo.data.baseDomain} does not resolve to this server (${serverInfo.data.ip}). Access the dashboard at ${serverInfo.data.url}.`, + }); + } if (missing.length === 0) return null; diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index 542aeae..b9b79e6 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -11,6 +11,26 @@ import { NotificationBanner } from "./layout/NotificationBanner"; export function Layout({ children }: { children: React.ReactNode }) { const location = useLocation(); const navigate = useNavigate(); + + const { data: me, isLoading: authLoading } = useQuery({ + queryKey: ["auth", "me"], + queryFn: () => api.getMe(), + retry: false, + }); + + useEffect(() => { + if (authLoading) return; + if (location.pathname === "/login") { + if (me?.authenticated) { + navigate({ to: "/" }); + } + return; + } + if (!me?.authenticated) { + navigate({ to: "/login" }); + } + }, [me, authLoading, location.pathname, navigate]); + const { data: projects = [] } = useProjects(); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); @@ -69,10 +89,16 @@ export function Layout({ children }: { children: React.ReactNode }) { return () => clearTimeout(t); }, [notification]); + const isLoginPage = location.pathname === "/login"; + const match = location.pathname.match(/\/project\/([^/]+)/); const currentProjectId = match ? match[1] : null; const currentProject = projects.find((p) => p.id === currentProjectId); + if (isLoginPage) { + return
{children}
; + } + return (
{ + e.preventDefault(); + setError(''); + setLoading(true); + try { + const res = await api.login(username, password); + if (res.ok) { + window.location.href = '/'; + } else { + setError(res.error || 'Login failed'); + } + } catch (err: any) { + setError(err.message || 'Login failed'); + } finally { + setLoading(false); + } + }; + + return ( +
+ {/* Background glow using Dequel orange */} +
+ + {/* Center content container for closer alignment */} +
+ + {/* Left Column: Login Card & Heading */} +
+ {/* Logo Branding */} +
+ + + dequel + + + v0.1 + +
+ + {/* Form */} +
+
+

+ Sign in to Dequel +

+

+ Enter your credentials to access your self-hosted deployment engine. +

+
+ +
+
+ + + Security Gateway + + + + Active + +
+ +
+
+ +
+ + setUsername(e.target.value)} + className="w-full pl-9 pr-4 py-2 rounded-lg border border-[#222227] bg-[#121214] text-zinc-200 text-xs placeholder-zinc-700 focus:outline-none focus:border-orange-500 focus:ring-1 focus:ring-orange-500/20 transition-all" + placeholder="Linux username" + autoFocus + required + /> +
+
+ +
+ +
+ + setPassword(e.target.value)} + className="w-full pl-9 pr-4 py-2 rounded-lg border border-[#222227] bg-[#121214] text-zinc-200 text-xs placeholder-zinc-700 focus:outline-none focus:border-orange-500 focus:ring-1 focus:ring-orange-500/20 transition-all" + placeholder="Linux password" + required + /> +
+
+ + {error && ( +

+ {error} +

+ )} + + +
+
+
+ + {/* Footer Info */} +
+ © {new Date().getFullYear()} Dequel. +
+
+ + {/* Right Column: Clean, Low-contrast CSS Mockup of Dequel Dashboard */} +
+ {/* Header Mockup */} +
+
+ + + +
+
+ dequel.local/dashboard +
+
+
+ + {/* Content Layout Mockup */} +
+ {/* Sidebar Mockup */} +
+
+
+ + dequel +
+ +
+
+ + Overview +
+
+ + Logs +
+
+
+ + {/* Sidebar Footer Widget Mockup */} +
+
+ STATUS + + + Live + +
+
+ Services + 3 +
+
+
+ + {/* Main Area Mockup */} +
+
+
+

Overview

+

Manage and monitor cluster resources.

+
+
+ + New Project +
+
+ + {/* Metric stats card */} +
+
+
+
API Traffic
+
148 reqs
+
+ +
+ +
+
+
Deployments
+
3 active
+
+ +
+
+ + {/* Projects list */} +
+
+
+
+ A +
+
+
api-service
+
github.com/lftobs/api
+
+
+ + running + +
+
+
+
+
+ +
+
+ ); +} diff --git a/apps/web/src/routes/index.tsx b/apps/web/src/routes/index.tsx index e33483f..ffbfab3 100644 --- a/apps/web/src/routes/index.tsx +++ b/apps/web/src/routes/index.tsx @@ -1,6 +1,7 @@ import { createRootRoute, createRoute, createRouter, Outlet } from '@tanstack/react-router'; import { Layout } from '../components/Layout'; import { Dashboard } from './Dashboard'; +import { Login } from './Login'; import { Settings } from './Settings'; import { ProjectDetail } from './ProjectDetail'; @@ -18,6 +19,12 @@ const indexRoute = createRoute({ component: Dashboard, }); +const loginRoute = createRoute({ + getParentRoute: () => rootRoute, + path: '/login', + component: Login, +}); + const settingsRoute = createRoute({ getParentRoute: () => rootRoute, path: '/settings', @@ -38,7 +45,7 @@ const projectRoute = createRoute({ }), }); -const routeTree = rootRoute.addChildren([indexRoute, settingsRoute, projectRoute]); +const routeTree = rootRoute.addChildren([indexRoute, loginRoute, settingsRoute, projectRoute]); export const router = createRouter({ routeTree }); diff --git a/docker-compose.yml b/docker-compose.yml index 95f7051..6dbb4ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,9 @@ services: - ./infra/caddy/routes:/caddy/routes - /var/run/docker.sock:/var/run/docker.sock - railpack-cache:/tmp/railpack + - /etc/passwd:/etc/passwd:ro + - /etc/shadow:/etc/shadow:ro + - /etc/group:/etc/group:ro depends_on: buildkit: condition: service_started diff --git a/scripts/pam-verify.py b/scripts/pam-verify.py new file mode 100644 index 0000000..442b724 --- /dev/null +++ b/scripts/pam-verify.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Authenticate a Linux user via PAM and check dequel group membership. + +Reads JSON { username, password } from stdin. +Exits 0 with { "ok": true, "username": "..." } on success. +Exits 1 with { "ok": false, "error": "..." } on failure. +""" +import json +import sys +import ctypes +import ctypes.util +import subprocess + +GRP_NAME = "dequel" +SERVICE = "dequel" + +PAM_PROMPT_ECHO_OFF = 1 +PAM_SUCCESS = 0 +PAM_END = -1 + + +class PamMessage(ctypes.Structure): + _fields_ = [("msg_style", ctypes.c_int), ("msg", ctypes.c_char_p)] + + +class PamResponse(ctypes.Structure): + _fields_ = [("resp", ctypes.c_void_p), ("resp_retcode", ctypes.c_int)] + + +CONV_FUNC = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.POINTER(PamMessage)), + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, +) + + +class PamConv(ctypes.Structure): + _fields_ = [("conv", CONV_FUNC), ("appdata_ptr", ctypes.c_void_p)] + + +def verify(username: str, password: str) -> dict: + libpam = ctypes.cdll.LoadLibrary(ctypes.util.find_library("pam")) + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libpam.pam_start.restype = ctypes.c_int + libpam.pam_authenticate.restype = ctypes.c_int + libpam.pam_acct_mgmt.restype = ctypes.c_int + libpam.pam_end.restype = ctypes.c_int + libpam.pam_strerror.restype = ctypes.c_char_p + libpam.pam_strerror.argtypes = [ctypes.c_void_p, ctypes.c_int] + + password_cpy = [password] + + def conv(nmsg, msg, out_resp, appdata): + count = nmsg + resp_size = ctypes.sizeof(PamResponse) * count + buf = libc.malloc(resp_size) + ctypes.memset(buf, 0, resp_size) + arr = ctypes.cast(buf, ctypes.POINTER(PamResponse)) + for i in range(count): + pm = ctypes.cast(msg[i], ctypes.POINTER(PamMessage))[0] + if pm.msg_style == PAM_PROMPT_ECHO_OFF: + pw = password_cpy[0] + pw_buf = libc.strdup(pw.encode()) + arr[i].resp = pw_buf + arr[i].resp_retcode = 0 + else: + arr[i].resp = None + arr[i].resp_retcode = PAM_END + out_resp[0] = buf + return PAM_SUCCESS + + cb = CONV_FUNC(conv) + conv_struct = PamConv(cb, None) + handle = ctypes.c_void_p() + + ret = libpam.pam_start(SERVICE.encode(), username.encode(), ctypes.byref(conv_struct), ctypes.byref(handle)) + if ret != PAM_SUCCESS: + err = libpam.pam_strerror(handle, ret) + return {"ok": False, "error": f"PAM start failed: {(err or b'').decode()}"} + + ret = libpam.pam_authenticate(handle, 0) + if ret != PAM_SUCCESS: + libpam.pam_end(handle, ret) + return {"ok": False, "error": "Authentication failed"} + + ret = libpam.pam_acct_mgmt(handle, 0) + libpam.pam_end(handle, ret) + if ret != PAM_SUCCESS: + return {"ok": False, "error": "Account expired or disabled"} + + result = subprocess.run( + ["getent", "group", GRP_NAME], + capture_output=True, text=True, timeout=10, + ) + if result.returncode != 0: + return {"ok": False, "error": f"Group '{GRP_NAME}' does not exist"} + + members = result.stdout.strip().split(":")[-1].split(",") if ":" in result.stdout else [] + if username not in members: + return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} + + return {"ok": True, "username": username} + + +def main(): + try: + data = json.load(sys.stdin) + except json.JSONDecodeError as e: + print(json.dumps({"ok": False, "error": f"Invalid input: {e}"})) + sys.exit(1) + + username = data.get("username", "").strip() + password = data.get("password", "") + + if not username or not password: + print(json.dumps({"ok": False, "error": "Username and password required"})) + sys.exit(1) + + result = verify(username, password) + print(json.dumps(result)) + sys.exit(0 if result["ok"] else 1) + + +if __name__ == "__main__": + main() From caf25f2f5945ec43723ecf75bcf86f5e74940cd1 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 23 Jun 2026 06:10:58 +0100 Subject: [PATCH 11/43] refactor(api): switch PAM auth to HTTP service - Replace in-process PAM verification with HTTP PAM service at pam-auth - Use pam-auth:4567 for authentication requests - Rename and relocate PAM server script to scripts/auth/pam-server.py - Update tests to reflect new PAM flow and token handling --- .dockerignore | 20 ++ apps/api/src/api/auth/index.ts | 53 +++--- apps/api/src/db/__tests__/auth.test.ts | 6 +- apps/api/src/utils/__tests__/auth.test.ts | 218 ++++++++++++++-------- apps/api/src/utils/auth.ts | 7 +- docker-compose.yml | 25 ++- scripts/auth/pam-server.py | 167 +++++++++++++++++ scripts/{ => auth}/pam-verify.py | 26 ++- 8 files changed, 408 insertions(+), 114 deletions(-) create mode 100644 .dockerignore create mode 100644 scripts/auth/pam-server.py rename scripts/{ => auth}/pam-verify.py (79%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7ce11e4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.gitignore +.gitattributes +AGENTS.md +CHANGELOG.md +CONTRIBUTING.md +VERSION +*.md +node_modules +apps/web +apps/docs +data +workspace +infra +dist +.env +__pycache__ +*.log +docker-compose.yml +CLI.md diff --git a/apps/api/src/api/auth/index.ts b/apps/api/src/api/auth/index.ts index cab9e18..518f7c8 100644 --- a/apps/api/src/api/auth/index.ts +++ b/apps/api/src/api/auth/index.ts @@ -1,31 +1,32 @@ import { Elysia } from "elysia"; -import { spawn } from "node:child_process"; import { signAccessToken, verifyAccessToken, generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } from "../../utils/auth"; import { config } from "../../utils/config"; -import { join } from "node:path"; -const PAM_SCRIPT = "/app/scripts/pam-verify.py"; +const PAM_AUTH_URL = "http://pam-auth:4567"; -const callPam = (username: string, password: string): Promise<{ ok: boolean; username?: string; error?: string }> => - new Promise((resolve) => { - const proc = spawn("python3", [PAM_SCRIPT], { stdio: ["pipe", "pipe", "pipe"] }); - let stdout = ""; - let stderr = ""; - proc.stdout.on("data", (chunk) => { stdout += String(chunk); }); - proc.stderr.on("data", (chunk) => { stderr += String(chunk); }); - proc.on("close", (code) => { - if (code !== 0) { - try { resolve(JSON.parse(stdout)); } - catch { resolve({ ok: false, error: stderr.trim() || "Authentication failed" }); } - return; - } - try { resolve(JSON.parse(stdout)); } - catch { resolve({ ok: false, error: "Invalid response from auth helper" }); } +const callPam = async (username: string, password: string): Promise<{ ok: boolean; username?: string; error?: string }> => { + try { + const res = await fetch(`${PAM_AUTH_URL}/auth`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username, password }), }); - proc.stdin!.end(JSON.stringify({ username, password })); - }); + const data = await res.json(); + return data; + } catch (err) { + return { ok: false, error: "Auth service unavailable" }; + } +}; + +const SESSION_COOKIE_OPTS = { + path: "/", + httpOnly: true, + sameSite: "strict" as const, + secure: config.caddyBaseDomain !== "localhost", + maxAge: 900, +}; -const COOKIE_OPTS = { +const REFRESH_COOKIE_OPTS = { path: "/", httpOnly: true, sameSite: "strict" as const, @@ -49,9 +50,9 @@ export const authRoutes = new Elysia() const refreshToken = generateRefreshToken(); await storeRefreshToken(username, refreshToken); dequel_session.value = accessToken; - dequel_session.set(COOKIE_OPTS); + dequel_session.set(SESSION_COOKIE_OPTS); dequel_refresh.value = refreshToken; - dequel_refresh.set(COOKIE_OPTS); + dequel_refresh.set(REFRESH_COOKIE_OPTS); return { ok: true, username }; }) .post("/auth/logout", async ({ cookie: { dequel_session, dequel_refresh } }) => { @@ -74,14 +75,14 @@ export const authRoutes = new Elysia() set.status = 401; return { error: "Invalid or expired refresh token" }; } - try { await blacklistRefreshToken(rt); } catch {} + await blacklistRefreshToken(rt); const accessToken = await signAccessToken(username); const newRefreshToken = generateRefreshToken(); await storeRefreshToken(username, newRefreshToken); dequel_session.value = accessToken; - dequel_session.set(COOKIE_OPTS); + dequel_session.set(SESSION_COOKIE_OPTS); dequel_refresh.value = newRefreshToken; - dequel_refresh.set(COOKIE_OPTS); + dequel_refresh.set(REFRESH_COOKIE_OPTS); return { ok: true, username }; }) .get("/auth/me", async ({ cookie: { dequel_session } }) => { diff --git a/apps/api/src/db/__tests__/auth.test.ts b/apps/api/src/db/__tests__/auth.test.ts index 5b1c675..22dc2ef 100644 --- a/apps/api/src/db/__tests__/auth.test.ts +++ b/apps/api/src/db/__tests__/auth.test.ts @@ -74,11 +74,11 @@ describe('blacklistRefreshToken', () => { describe('cleanupExpiredTokens', () => { it('removes expired tokens', async () => { - const { generateRefreshToken, storeRefreshToken, validateRefreshToken, cleanupExpiredTokens, hashToken } = await import('../../utils/auth'); + const { generateRefreshToken, storeRefreshToken, cleanupExpiredTokens } = await import('../../utils/auth'); const token = generateRefreshToken(); await storeRefreshToken('testuser', token); - const tokenHash = hashToken(token); - db.run(`UPDATE refresh_tokens SET expires_at = '2000-01-01T00:00:00.000Z' WHERE token_hash = ?`, [tokenHash]); + const row = db.query('SELECT token_hash FROM refresh_tokens ORDER BY created_at DESC LIMIT 1').get() as { token_hash: string }; + db.run(`UPDATE refresh_tokens SET expires_at = '2000-01-01T00:00:00.000Z' WHERE token_hash = ?`, [row.token_hash]); await cleanupExpiredTokens(); const remaining = db.query('SELECT COUNT(*) as c FROM refresh_tokens').get() as { c: number }; expect(remaining.c).toBe(0); diff --git a/apps/api/src/utils/__tests__/auth.test.ts b/apps/api/src/utils/__tests__/auth.test.ts index af96521..62cb317 100644 --- a/apps/api/src/utils/__tests__/auth.test.ts +++ b/apps/api/src/utils/__tests__/auth.test.ts @@ -1,91 +1,155 @@ -import { describe, it, expect, mock, beforeAll, beforeEach, afterAll } from 'bun:test'; -import { Database } from 'bun:sqlite'; +import { + describe, + it, + expect, + mock, + beforeAll, + beforeEach, + afterAll, +} from "bun:test"; +import { Database } from "bun:sqlite"; -const TEST_SECRET = 'test-jwt-secret-for-testing-purposes-only'; +const TEST_SECRET = + "test-jwt-secret-for-testing-purposes-only"; -describe('JWT operations', () => { - beforeAll(async () => { - const { initAuth } = await import('../auth'); - initAuth(TEST_SECRET); - }); +describe("JWT operations", () => { + beforeAll(async () => { + const { initAuth } = + await import("../auth"); + initAuth(TEST_SECRET); + }); - it('signs and verifies an access token', async () => { - const { signAccessToken, verifyAccessToken } = await import('../auth'); - const token = await signAccessToken('testuser'); - expect(typeof token).toBe('string'); - expect(token.split('.').length).toBe(3); - const payload = await verifyAccessToken(token); - expect(payload).not.toBeNull(); - expect(payload!.sub).toBe('testuser'); - expect(payload!.iat).toBeGreaterThan(0); - expect(payload!.exp).toBe(payload!.iat + 900); - }); + it("signs and verifies an access token", async () => { + const { + signAccessToken, + verifyAccessToken, + } = await import("../auth"); + const token = + await signAccessToken("testuser"); + expect(typeof token).toBe("string"); + expect(token.split(".").length).toBe(3); + const payload = + await verifyAccessToken(token); + expect(payload).not.toBeNull(); + expect(payload!.sub).toBe("testuser"); + expect(payload!.iat).toBeGreaterThan(0); + expect(payload!.exp).toBe( + payload!.iat + 900, + ); + }); - it('rejects a tampered token', async () => { - const { signAccessToken, verifyAccessToken } = await import('../auth'); - const token = await signAccessToken('testuser'); - const parts = token.split('.'); - const tampered = ['bad', parts[1], parts[2]].join('.'); - expect(await verifyAccessToken(tampered)).toBeNull(); - }); + it("rejects a tampered token", async () => { + const { + signAccessToken, + verifyAccessToken, + } = await import("../auth"); + const token = + await signAccessToken("testuser"); + const parts = token.split("."); + const tampered = [ + "bad", + parts[1], + parts[2], + ].join("."); + expect( + await verifyAccessToken(tampered), + ).toBeNull(); + }); - it('rejects token with wrong secret', async () => { - const { signAccessToken, verifyAccessToken, initAuth } = await import('../auth'); - const token = await signAccessToken('testuser'); - initAuth('different-secret'); - const result = await verifyAccessToken(token); - expect(result).toBeNull(); - initAuth(TEST_SECRET); - }); + it("rejects token with wrong secret", async () => { + const { + signAccessToken, + verifyAccessToken, + initAuth, + } = await import("../auth"); + const token = + await signAccessToken("testuser"); + initAuth("different-secret"); + const result = + await verifyAccessToken(token); + expect(result).toBeNull(); + initAuth(TEST_SECRET); + }); - it('rejects malformed token', async () => { - const { verifyAccessToken } = await import('../auth'); - expect(await verifyAccessToken('not-a-jwt')).toBeNull(); - expect(await verifyAccessToken('only.two.parts.here')).toBeNull(); - expect(await verifyAccessToken('')).toBeNull(); - }); + it("rejects malformed token", async () => { + const { verifyAccessToken } = + await import("../auth"); + expect( + await verifyAccessToken("not-a-jwt"), + ).toBeNull(); + expect( + await verifyAccessToken( + "only.two.parts.here", + ), + ).toBeNull(); + expect( + await verifyAccessToken(""), + ).toBeNull(); + }); - it('rejects expired token', async () => { - const { verifyAccessToken } = await import('../auth'); - const parts = ['header', btoa(JSON.stringify({ - sub: 'testuser', - iat: 1000, - exp: 1, - })).replace(/=/g, ''), 'fakesig']; - const result = await verifyAccessToken(parts.join('.')); - expect(result).toBeNull(); - }); + it("rejects expired token", async () => { + const { signAccessToken, verifyAccessToken } = + await import("../auth"); + const token = await signAccessToken("testuser"); + const [, payloadPart] = token.split("."); + const padded = payloadPart + .replace(/-/g, "+") + .replace(/_/g, "/") + .padEnd( + Math.ceil(payloadPart.length / 4) * 4, + "=", + ); + const payload = JSON.parse( + atob(padded), + ) as { exp: number }; + + const originalNow = Date.now; + Date.now = () => (payload.exp + 1) * 1000; + try { + expect( + await verifyAccessToken(token), + ).toBeNull(); + } finally { + Date.now = originalNow; + } + }); }); -describe('hashToken', () => { - it('produces consistent hashes', async () => { - const { hashToken } = await import('../auth'); - const h1 = hashToken('test-token'); - const h2 = hashToken('test-token'); - expect(h1).toBe(h2); - expect(h1.length).toBe(64); - }); +describe("hashToken", () => { + it("produces consistent hashes", async () => { + const { hashToken } = + await import("../auth"); + const h1 = hashToken("test-token"); + const h2 = hashToken("test-token"); + expect(h1).toBe(h2); + expect(h1.length).toBe(64); + }); - it('produces different hashes for different tokens', async () => { - const { hashToken } = await import('../auth'); - const h1 = hashToken('token-a'); - const h2 = hashToken('token-b'); - expect(h1).not.toBe(h2); - }); + it("produces different hashes for different tokens", async () => { + const { hashToken } = + await import("../auth"); + const h1 = hashToken("token-a"); + const h2 = hashToken("token-b"); + expect(h1).not.toBe(h2); + }); }); -describe('generateRefreshToken', () => { - it('generates token with correct prefix', async () => { - const { generateRefreshToken } = await import('../auth'); - const token = generateRefreshToken(); - expect(token.startsWith('dqr_')).toBe(true); - expect(token.length).toBe(4 + 64); - }); +describe("generateRefreshToken", () => { + it("generates token with correct prefix", async () => { + const { generateRefreshToken } = + await import("../auth"); + const token = generateRefreshToken(); + expect(token.startsWith("dqr_")).toBe( + true, + ); + expect(token.length).toBe(4 + 64); + }); - it('generates unique tokens', async () => { - const { generateRefreshToken } = await import('../auth'); - const t1 = generateRefreshToken(); - const t2 = generateRefreshToken(); - expect(t1).not.toBe(t2); - }); + it("generates unique tokens", async () => { + const { generateRefreshToken } = + await import("../auth"); + const t1 = generateRefreshToken(); + const t2 = generateRefreshToken(); + expect(t1).not.toBe(t2); + }); }); diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.ts index fadbba0..31fc480 100644 --- a/apps/api/src/utils/auth.ts +++ b/apps/api/src/utils/auth.ts @@ -1,5 +1,6 @@ import { randomBytes } from 'node:crypto'; import { getDrizzle } from '../db/drizzle'; +import { getDb } from '../db/client'; import { eq, and, sql } from 'drizzle-orm'; import { refreshTokens } from '../db/schema'; @@ -94,7 +95,7 @@ export const validateRefreshToken = async (token: string): Promise datetime('now')`, + sql`datetime(${refreshTokens.expiresAt}) > datetime('now')`, )) .get(); return row?.username ?? null; @@ -111,6 +112,6 @@ export const blacklistRefreshToken = async (token: string): Promise => { }; export const cleanupExpiredTokens = async (): Promise => { - const drizzle = await getDrizzle(); - drizzle.run(sql`DELETE FROM refresh_tokens WHERE expires_at < datetime('now')`); + const db = await getDb(); + db.run(`DELETE FROM refresh_tokens WHERE expires_at < datetime('now')`); }; diff --git a/docker-compose.yml b/docker-compose.yml index 6dbb4ca..a6a5f82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock - railpack-cache:/tmp/railpack - /etc/passwd:/etc/passwd:ro - - /etc/shadow:/etc/shadow:ro - /etc/group:/etc/group:ro depends_on: buildkit: @@ -71,6 +70,30 @@ services: aliases: - api + pam-auth: + image: python:3-slim + entrypoint: [] + command: + - sh + - -c + - | + apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq libpam-modules > /dev/null 2>&1 && exec python3 -u /app/pam-server.py + volumes: + - ./scripts/auth/pam-server.py:/app/pam-server.py:ro + - /etc/passwd:/etc/passwd:ro + - /etc/shadow:/etc/shadow:ro + - /etc/group:/etc/group:ro + networks: + dequel: + aliases: + - pam-auth + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:4567/health').status == 200 else 1)"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + web: image: ghcr.io/lftobs/dequel/web:latest # build: diff --git a/scripts/auth/pam-server.py b/scripts/auth/pam-server.py new file mode 100644 index 0000000..98acd03 --- /dev/null +++ b/scripts/auth/pam-server.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""HTTP sidecar for PAM authentication. Replaces direct /etc/shadow access from the API container.""" +import json +import os +import ctypes +import ctypes.util +import subprocess +from http.server import HTTPServer, BaseHTTPRequestHandler + +GRP_NAME = "dequel" +SERVICE = "dequel" + +PAM_PROMPT_ECHO_OFF = 1 +PAM_SUCCESS = 0 +PAM_END = -1 + + +class PamMessage(ctypes.Structure): + _fields_ = [("msg_style", ctypes.c_int), ("msg", ctypes.c_char_p)] + + +class PamResponse(ctypes.Structure): + _fields_ = [("resp", ctypes.c_void_p), ("resp_retcode", ctypes.c_int)] + + +CONV_FUNC = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.POINTER(PamMessage)), + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, +) + + +class PamConv(ctypes.Structure): + _fields_ = [("conv", CONV_FUNC), ("appdata_ptr", ctypes.c_void_p)] + + +def verify(username: str, password: str) -> dict: + lib_path = ctypes.util.find_library("pam") + if not lib_path: + return {"ok": False, "error": "PAM library not found on system"} + libpam = ctypes.cdll.LoadLibrary(lib_path) + libc = ctypes.cdll.LoadLibrary("libc.so.6") + libpam.pam_start.restype = ctypes.c_int + libpam.pam_authenticate.restype = ctypes.c_int + libpam.pam_acct_mgmt.restype = ctypes.c_int + libpam.pam_end.restype = ctypes.c_int + libpam.pam_strerror.restype = ctypes.c_char_p + libpam.pam_strerror.argtypes = [ctypes.c_void_p, ctypes.c_int] + + password_cpy = [password] + + def conv(nmsg, msg, out_resp, appdata): + count = nmsg + resp_size = ctypes.sizeof(PamResponse) * count + buf = libc.malloc(resp_size) + ctypes.memset(buf, 0, resp_size) + arr = ctypes.cast(buf, ctypes.POINTER(PamResponse)) + for i in range(count): + pm = ctypes.cast(msg[i], ctypes.POINTER(PamMessage))[0] + if pm.msg_style == PAM_PROMPT_ECHO_OFF: + pw = password_cpy[0] + pw_buf = libc.strdup(pw.encode()) + arr[i].resp = pw_buf + arr[i].resp_retcode = 0 + else: + arr[i].resp = None + arr[i].resp_retcode = PAM_END + out_resp[0] = buf + return PAM_SUCCESS + + cb = CONV_FUNC(conv) + conv_struct = PamConv(cb, None) + handle = ctypes.c_void_p() + + ret = libpam.pam_start(SERVICE.encode(), username.encode(), ctypes.byref(conv_struct), ctypes.byref(handle)) + if ret != PAM_SUCCESS: + err = libpam.pam_strerror(handle, ret) + return {"ok": False, "error": f"PAM start failed: {(err or b'').decode()}"} + + ret = libpam.pam_authenticate(handle, 0) + if ret != PAM_SUCCESS: + libpam.pam_end(handle, ret) + return {"ok": False, "error": "Authentication failed"} + + ret = libpam.pam_acct_mgmt(handle, 0) + libpam.pam_end(handle, ret) + if ret != PAM_SUCCESS: + return {"ok": False, "error": "Account expired or disabled"} + + result = subprocess.run( + ["getent", "group", GRP_NAME], + capture_output=True, text=True, timeout=10, + ) + if result.returncode != 0: + return {"ok": False, "error": f"Group '{GRP_NAME}' does not exist"} + + group_parts = result.stdout.strip().split(":") + members = group_parts[-1].split(",") if len(group_parts) >= 4 else [] + gid = group_parts[2] if len(group_parts) >= 3 else None + + if username in members: + return {"ok": True, "username": username} + + if gid: + pw_result = subprocess.run( + ["getent", "passwd", username], + capture_output=True, text=True, timeout=10, + ) + if pw_result.returncode == 0: + user_gid = pw_result.stdout.strip().split(":")[3] if ":" in pw_result.stdout else None + if user_gid == gid: + return {"ok": True, "username": username} + + return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} + + +class PamHandler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + except json.JSONDecodeError: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"error": "Invalid JSON"}).encode()) + return + + username = data.get("username", "").strip() + password = data.get("password", "") + + if not username or not password: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"error": "Username and password required"}).encode()) + return + + result = verify(username, password) + status = 200 if result.get("ok") else 401 + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(result).encode()) + + def do_GET(self): + if self.path == "/health": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"ok": True}).encode()) + return + self.send_response(404) + self.end_headers() + + +def main(): + port = int(os.environ.get("PORT", "4567")) + server = HTTPServer(("0.0.0.0", port), PamHandler) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/pam-verify.py b/scripts/auth/pam-verify.py similarity index 79% rename from scripts/pam-verify.py rename to scripts/auth/pam-verify.py index 442b724..14e5e5e 100644 --- a/scripts/pam-verify.py +++ b/scripts/auth/pam-verify.py @@ -41,7 +41,10 @@ class PamConv(ctypes.Structure): def verify(username: str, password: str) -> dict: - libpam = ctypes.cdll.LoadLibrary(ctypes.util.find_library("pam")) + lib_path = ctypes.util.find_library("pam") + if not lib_path: + return {"ok": False, "error": "PAM library not found on system"} + libpam = ctypes.cdll.LoadLibrary(lib_path) libc = ctypes.cdll.LoadLibrary("libc.so.6") libpam.pam_start.restype = ctypes.c_int libpam.pam_authenticate.restype = ctypes.c_int @@ -97,9 +100,24 @@ def conv(nmsg, msg, out_resp, appdata): if result.returncode != 0: return {"ok": False, "error": f"Group '{GRP_NAME}' does not exist"} - members = result.stdout.strip().split(":")[-1].split(",") if ":" in result.stdout else [] - if username not in members: - return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} + group_parts = result.stdout.strip().split(":") + members = group_parts[-1].split(",") if len(group_parts) >= 4 else [] + gid = group_parts[2] if len(group_parts) >= 3 else None + + if username in members: + return {"ok": True, "username": username} + + if gid: + pw_result = subprocess.run( + ["getent", "passwd", username], + capture_output=True, text=True, timeout=10, + ) + if pw_result.returncode == 0: + user_gid = pw_result.stdout.strip().split(":")[3] if ":" in pw_result.stdout else None + if user_gid == gid: + return {"ok": True, "username": username} + + return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} return {"ok": True, "username": username} From afc28aa5032823f233471cdd7db2696feca22077 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 27 Jun 2026 03:08:03 +0100 Subject: [PATCH 12/43] refactor(auth): add timeout and libc fixes - Add AbortSignal.timeout(5000) to PAM authentication request in apps/api/src/api/auth/index.ts - Fetch projects only when authenticated and not on /login in apps/web/src/components/Layout.tsx - Update useProjects to accept options and pass through to useQuery in apps/web/src/hooks/useProjects.ts - Harden PAM Python scripts by loading libc via find_library and setting malloc/strdup restype in scripts/auth/pam-server.py - Symmetrically update pam-verify.py to use find_library for libc and set malloc/strdup restype - Remove premature success return from pam-verify.py to fix flow --- apps/api/src/api/auth/index.ts | 1 + apps/web/src/components/Layout.tsx | 2 +- apps/web/src/hooks/useProjects.ts | 4 ++-- scripts/auth/pam-server.py | 4 +++- scripts/auth/pam-verify.py | 6 +++--- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/apps/api/src/api/auth/index.ts b/apps/api/src/api/auth/index.ts index 518f7c8..8581577 100644 --- a/apps/api/src/api/auth/index.ts +++ b/apps/api/src/api/auth/index.ts @@ -10,6 +10,7 @@ const callPam = async (username: string, password: string): Promise<{ ok: boolea method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ username, password }), + signal: AbortSignal.timeout(5000), }); const data = await res.json(); return data; diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index b9b79e6..d47c4a9 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -31,7 +31,7 @@ export function Layout({ children }: { children: React.ReactNode }) { } }, [me, authLoading, location.pathname, navigate]); - const { data: projects = [] } = useProjects(); + const { data: projects = [] } = useProjects({ enabled: !!me?.authenticated && location.pathname !== "/login" }); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); const { data: metricsText } = useQuery({ diff --git a/apps/web/src/hooks/useProjects.ts b/apps/web/src/hooks/useProjects.ts index 23bd2ac..55bf629 100644 --- a/apps/web/src/hooks/useProjects.ts +++ b/apps/web/src/hooks/useProjects.ts @@ -2,8 +2,8 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { listProjects, createProject, deleteProject } from '../api/client'; import type { Project } from '../types'; -export function useProjects() { - return useQuery({ queryKey: ['projects'], queryFn: listProjects, refetchInterval: 10_000 }); +export function useProjects(options?: { enabled?: boolean }) { + return useQuery({ queryKey: ['projects'], queryFn: listProjects, refetchInterval: 10_000, ...options }); } export function useProject(id: string) { diff --git a/scripts/auth/pam-server.py b/scripts/auth/pam-server.py index 98acd03..4bb96d8 100644 --- a/scripts/auth/pam-server.py +++ b/scripts/auth/pam-server.py @@ -41,13 +41,15 @@ def verify(username: str, password: str) -> dict: if not lib_path: return {"ok": False, "error": "PAM library not found on system"} libpam = ctypes.cdll.LoadLibrary(lib_path) - libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c") or "libc.so.6") libpam.pam_start.restype = ctypes.c_int libpam.pam_authenticate.restype = ctypes.c_int libpam.pam_acct_mgmt.restype = ctypes.c_int libpam.pam_end.restype = ctypes.c_int libpam.pam_strerror.restype = ctypes.c_char_p libpam.pam_strerror.argtypes = [ctypes.c_void_p, ctypes.c_int] + libc.malloc.restype = ctypes.c_void_p + libc.strdup.restype = ctypes.c_void_p password_cpy = [password] diff --git a/scripts/auth/pam-verify.py b/scripts/auth/pam-verify.py index 14e5e5e..e7ca80b 100644 --- a/scripts/auth/pam-verify.py +++ b/scripts/auth/pam-verify.py @@ -45,13 +45,15 @@ def verify(username: str, password: str) -> dict: if not lib_path: return {"ok": False, "error": "PAM library not found on system"} libpam = ctypes.cdll.LoadLibrary(lib_path) - libc = ctypes.cdll.LoadLibrary("libc.so.6") + libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c") or "libc.so.6") libpam.pam_start.restype = ctypes.c_int libpam.pam_authenticate.restype = ctypes.c_int libpam.pam_acct_mgmt.restype = ctypes.c_int libpam.pam_end.restype = ctypes.c_int libpam.pam_strerror.restype = ctypes.c_char_p libpam.pam_strerror.argtypes = [ctypes.c_void_p, ctypes.c_int] + libc.malloc.restype = ctypes.c_void_p + libc.strdup.restype = ctypes.c_void_p password_cpy = [password] @@ -119,8 +121,6 @@ def conv(nmsg, msg, out_resp, appdata): return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} - return {"ok": True, "username": username} - def main(): try: From 295178ddf6b309b3f59a601bece23955ecd1ae9c Mon Sep 17 00:00:00 2001 From: lftobs Date: Thu, 2 Jul 2026 19:00:45 +0100 Subject: [PATCH 13/43] feat(deploy): support specific commit deployments and cache clearing - Add `clear_cache` column to `deployments` table - Allow specifying `commitSha` during deployment via API and UI - Update `prepareSourceWorkspace` to support explicit git checkout - Add unit tests for scaling engine - Modularize Docker utilities for the scaling engine --- apps/api/src/api/deployments/index.ts | 7 + apps/api/src/db/migrate.ts | 7 + apps/api/src/db/repo/deployments.ts | 2 + apps/api/src/db/repo/scaling.ts | 4 + apps/api/src/db/schema.ts | 1 + apps/api/src/orchestrator/pipeline.ts | 11 +- apps/api/src/orchestrator/railpack.ts | 8 +- apps/api/src/orchestrator/runtime.ts | 1 + apps/api/src/orchestrator/source.ts | 15 +- apps/api/src/scaling/__tests__/engine.test.ts | 274 +++++++++++++ apps/api/src/scaling/docker-utils.ts | 17 + apps/api/src/scaling/engine.ts | 18 +- apps/api/src/types.ts | 2 + .../project/deployments/DeploymentsTab.tsx | 369 ++++++++++++------ 14 files changed, 594 insertions(+), 142 deletions(-) create mode 100644 apps/api/src/scaling/__tests__/engine.test.ts create mode 100644 apps/api/src/scaling/docker-utils.ts diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index a8c4147..99b1cf0 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -62,6 +62,10 @@ export const deploymentsRoutes = new Elysia() const environment = String(form.get("environment") ?? "").trim() || undefined; + const commitSha = + String(form.get("commitSha") ?? "").trim() || + undefined; + const clearCache = form.get("clearCache") === "true"; if ( sourceType !== "git" && sourceType !== "upload" && @@ -86,6 +90,8 @@ export const deploymentsRoutes = new Elysia() sourceRef: gitUrl, branch, environment, + commitSha, + clearCache, }); orchestrator.enqueue(deployment.id); return deployment; @@ -118,6 +124,7 @@ export const deploymentsRoutes = new Elysia() sourceRef: uploadPath, branch, environment, + clearCache, }); orchestrator.enqueue(deployment.id); return deployment; diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 2c3682f..4a6b4c4 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -47,6 +47,13 @@ export const migrate = async () => { console.log("[Migrate] Added projects.source_type column"); } + const deploymentsTableInfo = sqlite.query("PRAGMA table_info('deployments')").all() as { name: string }[]; + const deploymentsColumns = deploymentsTableInfo.map(r => r.name); + if (!deploymentsColumns.includes('clear_cache')) { + sqlite.exec("ALTER TABLE deployments ADD COLUMN clear_cache integer NOT NULL DEFAULT 0"); + console.log("[Migrate] Added deployments.clear_cache column"); + } + await seedFromConfig(); }; diff --git a/apps/api/src/db/repo/deployments.ts b/apps/api/src/db/repo/deployments.ts index b4a983d..bdaf080 100644 --- a/apps/api/src/db/repo/deployments.ts +++ b/apps/api/src/db/repo/deployments.ts @@ -20,6 +20,7 @@ const mapDeployment = (row: typeof deployments.$inferSelect): Deployment => ({ replicas: row.replicas, environment: row.environment, failureReason: row.failureReason, + clearCache: Boolean(row.clearCache), createdAt: row.createdAt, updatedAt: row.updatedAt, }); @@ -38,6 +39,7 @@ export const createDeployment = async (input: CreateDeploymentInput): Promise => { await onLog( @@ -731,13 +732,18 @@ export const buildWithRailpack = async ( // Ensure builder exists (persists across builds) await ensureBuilder(); - const cacheKey = + let cacheKey = opts?.cacheKey ?? imageTag .split(":")[0] .replace(/-[0-9a-f]{8}$/i, "") // Strip unique deployment short ID suffix .replace(/[^a-zA-Z0-9_-]/g, "-"); + if (opts?.clearCache) { + cacheKey = `${cacheKey}-clear-${Date.now()}`; + await onLog(`Bypassing cache for clean build (cacheKey: ${cacheKey})`); + } + const cleanSourceDir = opts?.sourceDir ? opts.sourceDir.replace(/^\//, "") : null; diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 30e7f4b..8370f09 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -94,6 +94,7 @@ export const ensureContainerRunning = async (containerName: string) => { }; export const reloadCaddy = async () => { + if (process.env.NODE_ENV === 'test') return; const caddyContainer = await getCaddyContainer(); await run(dockerBin, ['exec', caddyContainer, 'caddy', 'reload', '--config', '/etc/caddy/Caddyfile']); }; diff --git a/apps/api/src/orchestrator/source.ts b/apps/api/src/orchestrator/source.ts index 2f05918..6d1d963 100644 --- a/apps/api/src/orchestrator/source.ts +++ b/apps/api/src/orchestrator/source.ts @@ -34,14 +34,19 @@ const run = (cmd: string, args: string[], cwd?: string) => }); }); -export const prepareSourceWorkspace = async (deploymentId: string, gitUrl: string, branch?: string) => { +export const prepareSourceWorkspace = async (deploymentId: string, gitUrl: string, branch?: string, commitSha?: string) => { const root = join(config.workspaceRoot, deploymentId); await rm(root, { recursive: true, force: true }); await mkdir(root, { recursive: true }); - const args = ['clone', '--depth', '1']; - if (branch) args.push('--branch', branch); - args.push(gitUrl, root); - await run('git', args); + if (commitSha) { + await run('git', ['clone', gitUrl, root]); + await run('git', ['checkout', commitSha], root); + } else { + const args = ['clone', '--depth', '1']; + if (branch) args.push('--branch', branch); + args.push(gitUrl, root); + await run('git', args); + } return root; }; diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts new file mode 100644 index 0000000..0145a9f --- /dev/null +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -0,0 +1,274 @@ +import { describe, it, expect, mock, beforeEach, afterAll } from 'bun:test'; + +const TEST_DEPLOYMENT = { id: 'dep-1', projectId: 'proj-1', containerName: 'deploy-dep-1' }; + +const TEST_POLICY = { + id: 'pol-1', projectId: 'proj-1', enabled: true, + minReplicas: 1, maxReplicas: 5, + cpuThresholdPercent: 70, memoryThresholdPercent: 85, + cooldownSeconds: 120, createdAt: '', updatedAt: '', +}; + +// Mock state variables +let mockRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; +let mockTryRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; +let mockReadFileContent: string | null = null; +let mockWriteFilePath: string | null = null; +let mockWriteContent: string | null = null; +let mockRedisData: Record = {}; +let mockGetScalingPolicyResult: ((id: string) => any) | null = null; +let mockListEnvVarsResult: any[] = []; + +beforeEach(() => { + mockRunImpl = null; + mockTryRunImpl = null; + mockReadFileContent = null; + mockWriteFilePath = null; + mockWriteContent = null; + mockRedisData = {}; + mockGetScalingPolicyResult = null; + mockListEnvVarsResult = []; +}); + +afterAll(() => { + mockRunImpl = null; + mockTryRunImpl = null; + mockReadFileContent = null; + mockWriteFilePath = null; + mockWriteContent = null; + mockRedisData = {}; + mockGetScalingPolicyResult = null; + mockListEnvVarsResult = []; +}); + +// Mock docker-utils +const fileUrl = (path: string) => new URL(path, import.meta.url).toString(); +mock.module(fileUrl('../docker-utils'), () => ({ + run: mock((cmd: string, args: string[]) => { + if (mockRunImpl) return mockRunImpl(cmd, args); + return Promise.resolve(''); + }), + tryRun: mock((cmd: string, args: string[]) => { + if (mockTryRunImpl) return mockTryRunImpl(cmd, args); + return Promise.resolve(''); + }), +})); + +// Mock fs +mock.module('node:fs/promises', () => { + const fs = require('node:fs'); + const pathMod = require('node:path'); + return { + readFile: mock(async (path: string, options?: any) => { + if (mockReadFileContent !== null) { + return mockReadFileContent; + } + if (fs.existsSync(path)) { + return fs.readFileSync(path, options || 'utf8'); + } + throw new Error(`ENOENT: no such file or directory, open '${path}'`); + }), + writeFile: mock(async (path: string, content: string, options?: any) => { + mockWriteFilePath = path; + mockWriteContent = content; + const dir = pathMod.dirname(path); + if (fs.existsSync(dir)) { + fs.writeFileSync(path, content, options || 'utf8'); + } + }), + }; +}); + +// Mock redis +mock.module('ioredis', () => ({ + default: class FakeRedis { + hgetall = mock(async () => mockRedisData); + hset = mock(async (_key: string, data: Record) => { + mockRedisData = { ...mockRedisData, ...data }; + }); + quit = mock(async () => {}); + }, +})); + +// Mock db/repo +mock.module(fileUrl('../../db/repo'), () => ({ + getProjectById: mock(() => Promise.resolve(null)), + listDeployments: mock(() => Promise.resolve([])), + getScalingPolicy: mock((id: string) => + mockGetScalingPolicyResult ? Promise.resolve(mockGetScalingPolicyResult(id)) : Promise.resolve(null) + ), + updateDeploymentStatus: mock(() => Promise.resolve()), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve(mockListEnvVarsResult)), + listDomains: mock(() => Promise.resolve([])), + updateDomainValidation: mock(() => Promise.resolve()), +})); + +// Mock docker-bin +mock.module(fileUrl('../../utils/docker-bin'), () => ({ dockerBin: '/usr/bin/docker' })); + +const { scalingEngine } = await import('../engine'); + +// --- Tests --- + +describe('parseMemToMb', () => { + it('converts GiB to MB', () => { + expect((scalingEngine as any).parseMemToMb('2GiB')).toBe(2048); + }); + + it('converts MiB directly', () => { + expect((scalingEngine as any).parseMemToMb('512MiB')).toBe(512); + }); + + it('converts KiB to MB', () => { + expect((scalingEngine as any).parseMemToMb('1024KiB')).toBe(1); + }); + + it('returns 0 for unrecognized format', () => { + expect((scalingEngine as any).parseMemToMb('')).toBe(0); + expect((scalingEngine as any).parseMemToMb('abc')).toBe(0); + }); + + it('handles MB format', () => { + expect((scalingEngine as any).parseMemToMb('256MB')).toBe(256); + }); +}); + +describe('getCurrentReplicas', () => { + it('returns 1 when no route file exists', async () => { + mockReadFileContent = null; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(1); + }); + + it('returns 1 when no reverse_proxy in file', async () => { + mockReadFileContent = 'proj-1.localhost:80 {\n log\n}\n'; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(1); + }); + + it('counts unique containers in reverse_proxy targets', async () => { + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(2); + }); + + it('ignores port suffixes when counting containers', async () => { + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy a:80 b:80 c:80\n}\n'; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(3); + }); +}); + +describe('evaluate', () => { + beforeEach(() => { + mockGetScalingPolicyResult = () => TEST_POLICY; + mockListEnvVarsResult = []; + mockReadFileContent = null; + mockRunImpl = null; + mockTryRunImpl = null; + }); + + it('returns early when deployment has no projectId', async () => { + await (scalingEngine as any).evaluate({ id: 'dep-1', projectId: null, containerName: null }); + }); + + it('returns early when no scaling policy', async () => { + mockGetScalingPolicyResult = null; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + }); + + it('returns early when policy is disabled', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, enabled: false }); + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + }); + + it('returns early when container stats fail', async () => { + mockRunImpl = async () => 'not-json'; + const result = await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(result).toBeUndefined(); + }); + + it('sets highCpuSince on first high CPU reading', async () => { + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.highCpuSince).toBeTruthy(); + }); + + it('does not scale up if highCpuSince is not old enough', async () => { + mockRedisData = { highCpuSince: String(Date.now() - 5000), lastScaleUp: '0' }; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it('scales up when CPU exceeds threshold past cooldown', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n'; + + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.some(a => a.includes('stats'))) return '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + if (args.some(a => a.includes('Image'))) return 'my-app:latest'; + if (args.some(a => a.includes('Env'))) return '["PORT=3000"]'; + if (args.some(a => a.includes('Mounts'))) return '[]'; + if (args[0] === 'run') return 'new-container-id'; + if (args[0] === 'ps') return 'caddy-abc123\n'; + if (args.some(a => a.includes('reload'))) return ''; + return ''; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain('deploy-dep-1-replica-2'); + }); + + it('does not scale up when already at max replicas', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it('does not scale down when CPU is moderate', async () => { + mockRunImpl = async () => '{"CPUPerc":"50%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it('scales down when CPU stays low for 5 min', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 1, cooldownSeconds: 0 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + mockTryRunImpl = async () => ''; + + let callCount = 0; + mockRunImpl = async (_cmd: string, args: string[]) => { + callCount++; + if (args.includes('stats')) return '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === 'stop') return ''; + if (args[0] === 'rm') return ''; + if (args[0] === 'ps') return 'caddy-abc123\n'; + if (args.includes('reload')) return ''; + return ''; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + + expect(mockWriteContent).toContain('deploy-dep-1'); + expect(mockWriteContent).not.toContain('replica-2'); + }); + + it('does not scale down below minReplicas', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 2, cooldownSeconds: 0 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + mockRunImpl = async () => '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); +}); diff --git a/apps/api/src/scaling/docker-utils.ts b/apps/api/src/scaling/docker-utils.ts new file mode 100644 index 0000000..44553cd --- /dev/null +++ b/apps/api/src/scaling/docker-utils.ts @@ -0,0 +1,17 @@ +import { spawn } from 'node:child_process'; + +export const run = (cmd: string, args: string[]) => + new Promise((resolve, reject) => { + const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += String(chunk); }); + child.stderr.on('data', (chunk) => { stderr += String(chunk); }); + child.on('close', (code) => { + if (code === 0) resolve((stdout + '\n' + stderr).trim()); + else reject(new Error(`${cmd} ${args.join(' ')} failed (${code}): ${stderr}`)); + }); + }); + +export const tryRun = (cmd: string, args: string[]) => + run(cmd, args).catch(() => ''); diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 510d5f3..16e544b 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -1,27 +1,11 @@ -import { spawn } from 'node:child_process'; import { readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import Redis from 'ioredis'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy } from '../db/repo'; -const run = (cmd: string, args: string[]) => - new Promise((resolve, reject) => { - const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => { stdout += String(chunk); }); - child.stderr.on('data', (chunk) => { stderr += String(chunk); }); - child.on('close', (code) => { - if (code === 0) resolve((stdout + '\n' + stderr).trim()); - else reject(new Error(`${cmd} ${args.join(' ')} failed (${code}): ${stderr}`)); - }); - }); - -const tryRun = (cmd: string, args: string[]) => - run(cmd, args).catch(() => ''); - interface ContainerStats { containerName: string; cpuPercent: number; diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index d315c31..e7bb643 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -221,6 +221,7 @@ export interface Deployment { replicas: number; environment: string | null; failureReason: string | null; + clearCache: boolean; createdAt: string; updatedAt: string; } @@ -239,6 +240,7 @@ export interface CreateDeploymentInput { branch?: string; commitSha?: string; environment?: string; + clearCache?: boolean; } export interface DeploymentLog { diff --git a/apps/web/src/components/project/deployments/DeploymentsTab.tsx b/apps/web/src/components/project/deployments/DeploymentsTab.tsx index 69bd4ee..53c0de1 100644 --- a/apps/web/src/components/project/deployments/DeploymentsTab.tsx +++ b/apps/web/src/components/project/deployments/DeploymentsTab.tsx @@ -246,6 +246,39 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { setShowGitSwitch(false); }; + const [showManualDeployDialog, setShowManualDeployDialog] = useState(false); + const [deployOption, setDeployOption] = useState<"latest" | "commit">("latest"); + const [commitSha, setCommitSha] = useState(""); + const [clearCache, setClearCache] = useState(false); + + const handleManualDeploy = async () => { + const form = new FormData(); + form.set("sourceType", "git"); + if (projectId) form.set("projectId", projectId); + if (project?.repoUrl) form.set("gitUrl", project.repoUrl); + if (project?.repoBranch) form.set("branch", project.repoBranch); + if (environment) form.set("environment", environment); + + if (deployOption === "commit") { + if (!commitSha.trim()) return; + form.set("commitSha", commitSha.trim()); + } + + if (clearCache) { + form.set("clearCache", "true"); + } + + try { + await createDeployment.mutateAsync(form); + setShowManualDeployDialog(false); + setCommitSha(""); + setDeployOption("latest"); + setClearCache(false); + } catch (err) { + console.error("Failed to start manual deployment:", err); + } + }; + const selectedDeployment = deployments.find( (d) => d.id === selectedId, ); @@ -288,132 +321,138 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) {
)} -
-
- {( - [ - "git", - "upload", - "compose", - ] as const - ).map((type) => ( - +
+
+
+ ) : ( + +
+ {( + [ + "git", + "upload", + "compose", + ] as const + ).map((type) => ( + - ))} -
- {sourceType === "git" ? ( + ? "Upload" + : "Compose"} + + ))} + + +
+ setClearCache(e.target.checked)} + className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" + /> + +
- setGitUrl( - e - .target + placeholder="Environment (e.g. production)" + value={ + environment + } + onChange={(e) => + setEnvironment( + e.target .value, ) } className="flex-1" - disabled /> - - setBranch( - e - .target - .value, - ) +
- ) : ( - - )} -
- - setEnvironment( - e.target - .value, - ) - } - className="flex-1" - /> - -
- {!canEditSource && ( -
- Source type locked - after first - deploy. Use Switch - to Git to change - source. + > + {createDeployment.isPending || + isAutoDeploying ? ( + "Deploying..." + ) : ( + <> + + Update + + )} +
- )} - + {!canEditSource && ( +
+ Source type locked + after first + deploy. Use Switch + to Git to change + source. +
+ )} + + )} @@ -751,6 +790,106 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { + + + + + + + Manual Deployment + + + Build and deploy a new version of {project?.name}. + + + +
+ {/* Deploy Option Selection */} +
+ +
+ + +
+
+ + {/* Commit Hash Input */} + {deployOption === "commit" && ( +
+ + setCommitSha(e.target.value)} + className="bg-[#121215] border-[#222227] text-zinc-200 focus:border-amber-500 text-xs font-mono h-9" + /> +
+ )} + + {/* Clear Cache Toggle */} +
+ setClearCache(e.target.checked)} + className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" + /> +
setClearCache(!clearCache)}> + + + Bypasses cached Buildkit stages to force a clean dependency fetch and build. + +
+
+
+ + + + + + + ); } From ef3fd36b45aed5ca5082f2d583e69049fd73588a Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 4 Jul 2026 11:48:05 +0100 Subject: [PATCH 14/43] feat(api): add drizzle migrations and deploy UI - Add drizzle-kit config for sqlite and generate migrations - Include initial migration snapshot and journal cleanup - Fix upsertScalingPolicy to store enabled as 0/1 and handle undef - Improve git deployment flow: validate commit SHAs and fetch FETCH_HEAD - Introduce frontend deployment UI: history and logs - Refactor deployments tab to use new components - Enable API build in docker-compose for local dev --- apps/api/drizzle.config.ts | 10 + apps/api/src/db/migrate.ts | 37 +- .../src/db/migrations/meta/0002_snapshot.json | 1296 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 2 +- apps/api/src/db/repo/scaling.ts | 2 +- apps/api/src/orchestrator/source.ts | 11 +- .../project/deployments/DeploymentsTab.tsx | 569 +------- .../deployments/deployment-history.tsx | 221 +++ .../project/deployments/deployment-logs.tsx | 134 ++ .../deployments/manual-deploy-dialog.tsx | 172 +++ docker-compose.yml | 21 +- 11 files changed, 1897 insertions(+), 578 deletions(-) create mode 100644 apps/api/drizzle.config.ts create mode 100644 apps/api/src/db/migrations/meta/0002_snapshot.json create mode 100644 apps/web/src/components/project/deployments/deployment-history.tsx create mode 100644 apps/web/src/components/project/deployments/deployment-logs.tsx create mode 100644 apps/web/src/components/project/deployments/manual-deploy-dialog.tsx diff --git a/apps/api/drizzle.config.ts b/apps/api/drizzle.config.ts new file mode 100644 index 0000000..7a85d99 --- /dev/null +++ b/apps/api/drizzle.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "drizzle-kit"; + +export default defineConfig({ + dialect: "sqlite", + schema: "./src/db/schema.ts", + out: "./src/db/migrations", + dbCredentials: { + url: "./data/dequel.db", + }, +}); diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 4a6b4c4..47506f6 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -1,8 +1,7 @@ -import { existsSync } from "node:fs"; -import { readdirSync } from "node:fs"; +import { existsSync, readdirSync } from "node:fs"; import { migrate as drizzleMigrate } from "drizzle-orm/bun-sqlite/migrator"; import { getDrizzle } from "./drizzle"; -import { getDb } from "./client"; +import { sql } from "drizzle-orm"; import { config } from "../utils/config"; import { getSmtpSettings, upsertSmtpSettings } from "./repo/settings"; import { getGithubIntegration, setGithubIntegration } from "./repo/github"; @@ -30,31 +29,17 @@ export const migrate = async () => { drizzleMigrate(db, { migrationsFolder }); - // Apply schema additions that Drizzle Kit migrations may miss - const sqlite = await getDb(); - const tableInfo = sqlite.query("PRAGMA table_info('projects')").all() as { name: string }[]; - const columns = tableInfo.map(r => r.name); - if (!columns.includes('port')) { - sqlite.exec("ALTER TABLE projects ADD COLUMN port integer"); - console.log("[Migrate] Added projects.port column"); - } - if (!columns.includes('source_dir')) { - sqlite.exec("ALTER TABLE projects ADD COLUMN source_dir text"); - console.log("[Migrate] Added projects.source_dir column"); - } - if (!columns.includes('source_type')) { - sqlite.exec("ALTER TABLE projects ADD COLUMN source_type text NOT NULL DEFAULT 'git'"); - console.log("[Migrate] Added projects.source_type column"); - } + await addClearCacheColumn(db); + await seedFromConfig(); +}; - const deploymentsTableInfo = sqlite.query("PRAGMA table_info('deployments')").all() as { name: string }[]; - const deploymentsColumns = deploymentsTableInfo.map(r => r.name); - if (!deploymentsColumns.includes('clear_cache')) { - sqlite.exec("ALTER TABLE deployments ADD COLUMN clear_cache integer NOT NULL DEFAULT 0"); - console.log("[Migrate] Added deployments.clear_cache column"); +const addClearCacheColumn = async (db: ReturnType) => { + try { + db.run(sql`ALTER TABLE deployments ADD COLUMN clear_cache integer NOT NULL DEFAULT 0`); + console.log("[Migrate] Added clear_cache column to deployments table"); + } catch { + // Column already exists — no-op } - - await seedFromConfig(); }; const seedFromConfig = async () => { diff --git a/apps/api/src/db/migrations/meta/0002_snapshot.json b/apps/api/src/db/migrations/meta/0002_snapshot.json new file mode 100644 index 0000000..fd83b7d --- /dev/null +++ b/apps/api/src/db/migrations/meta/0002_snapshot.json @@ -0,0 +1,1296 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "8d240493-b31a-41dc-bbe1-76c17a481530", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "alerts": { + "name": "alerts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "threshold": { + "name": "threshold", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_seconds": { + "name": "duration_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'email'" + }, + "destination": { + "name": "destination", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "alerts_project_id_projects_id_fk": { + "name": "alerts_project_id_projects_id_fk", + "tableFrom": "alerts", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "api_keys": { + "name": "api_keys", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'deploy:read'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "databases": { + "name": "databases", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "database_name": { + "name": "database_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "internal_host": { + "name": "internal_host", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "internal_port": { + "name": "internal_port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cpu_limit": { + "name": "cpu_limit", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_limit_mb": { + "name": "memory_limit_mb", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "connection_string": { + "name": "connection_string", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provisioning'" + }, + "container_name": { + "name": "container_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "databases_project_id_projects_id_fk": { + "name": "databases_project_id_projects_id_fk", + "tableFrom": "databases", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "deployment_logs": { + "name": "deployment_logs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_logs_dep_seq": { + "name": "idx_logs_dep_seq", + "columns": [ + "deployment_id", + "sequence" + ], + "isUnique": true + } + }, + "foreignKeys": { + "deployment_logs_deployment_id_deployments_id_fk": { + "name": "deployment_logs_deployment_id_deployments_id_fk", + "tableFrom": "deployment_logs", + "tableTo": "deployments", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "deployments": { + "name": "deployments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_ref": { + "name": "source_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "image_tag": { + "name": "image_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "container_name": { + "name": "container_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "route_path": { + "name": "route_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "live_url": { + "name": "live_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "replicas": { + "name": "replicas", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "clear_cache": { + "name": "clear_cache", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "domains": { + "name": "domains", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'custom'" + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "ssl_status": { + "name": "ssl_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "domains_project_id_projects_id_fk": { + "name": "domains_project_id_projects_id_fk", + "tableFrom": "domains", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environment_variables": { + "name": "environment_variables", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value_encrypted": { + "name": "value_encrypted", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "value_iv": { + "name": "value_iv", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "value_tag": { + "name": "value_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'production'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_env_vars_project": { + "name": "idx_env_vars_project", + "columns": [ + "project_id", + "environment" + ], + "isUnique": false + } + }, + "foreignKeys": { + "environment_variables_project_id_projects_id_fk": { + "name": "environment_variables_project_id_projects_id_fk", + "tableFrom": "environment_variables", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "github_integrations": { + "name": "github_integrations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "app_name": { + "name": "app_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Dequel'" + }, + "webhook_secret": { + "name": "webhook_secret", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repo_url": { + "name": "repo_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repo_branch": { + "name": "repo_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_domain": { + "name": "base_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_limit": { + "name": "cpu_limit", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_limit_mb": { + "name": "memory_limit_mb", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_dir": { + "name": "source_dir", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'git'" + }, + "github_token_encrypted": { + "name": "github_token_encrypted", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_token_iv": { + "name": "github_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_token_tag": { + "name": "github_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "refresh_tokens": { + "name": "refresh_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "blacklisted_at": { + "name": "blacklisted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "refresh_tokens_token_hash_unique": { + "name": "refresh_tokens_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scaling_policies": { + "name": "scaling_policies", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_replicas": { + "name": "min_replicas", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "max_replicas": { + "name": "max_replicas", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 5 + }, + "cpu_threshold_percent": { + "name": "cpu_threshold_percent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 70 + }, + "memory_threshold_percent": { + "name": "memory_threshold_percent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 85 + }, + "cooldown_seconds": { + "name": "cooldown_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 120 + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "scaling_policies_project_id_unique": { + "name": "scaling_policies_project_id_unique", + "columns": [ + "project_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "scaling_policies_project_id_projects_id_fk": { + "name": "scaling_policies_project_id_projects_id_fk", + "tableFrom": "scaling_policies", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "servers": { + "name": "servers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host": { + "name": "host", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 2375 + }, + "auth_token": { + "name": "auth_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "cpu_total": { + "name": "cpu_total", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_total_mb": { + "name": "memory_total_mb", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disk_total_mb": { + "name": "disk_total_mb", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cpu_used_percent": { + "name": "cpu_used_percent", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_used_mb": { + "name": "memory_used_mb", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_heartbeat": { + "name": "last_heartbeat", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "smtp_settings": { + "name": "smtp_settings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host": { + "name": "host", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 587 + }, + "user": { + "name": "user", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "pass_encrypted": { + "name": "pass_encrypted", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pass_iv": { + "name": "pass_iv", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pass_tag": { + "name": "pass_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "from_address": { + "name": "from_address", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'dequel@localhost'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "volumes": { + "name": "volumes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mount_path": { + "name": "mount_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'/app/data'" + }, + "size_mb": { + "name": "size_mb", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "docker_volume_name": { + "name": "docker_volume_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "volumes_project_id_projects_id_fk": { + "name": "volumes_project_id_projects_id_fk", + "tableFrom": "volumes", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 8d4d821..d7da1e3 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -17,4 +17,4 @@ "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/apps/api/src/db/repo/scaling.ts b/apps/api/src/db/repo/scaling.ts index 894e018..819527c 100644 --- a/apps/api/src/db/repo/scaling.ts +++ b/apps/api/src/db/repo/scaling.ts @@ -42,7 +42,7 @@ export const upsertScalingPolicy = async (input: CreateScalingPolicyInput): Prom cpuThresholdPercent: input.cpuThresholdPercent ?? 70, memoryThresholdPercent: input.memoryThresholdPercent ?? 85, cooldownSeconds: input.cooldownSeconds ?? 120, - enabled: input.enabled ? 1 : 1, + enabled: input.enabled !== undefined ? (input.enabled ? 1 : 0) : 1, createdAt: timestamp, updatedAt: timestamp, }).run(); diff --git a/apps/api/src/orchestrator/source.ts b/apps/api/src/orchestrator/source.ts index 6d1d963..f2f813f 100644 --- a/apps/api/src/orchestrator/source.ts +++ b/apps/api/src/orchestrator/source.ts @@ -34,13 +34,20 @@ const run = (cmd: string, args: string[], cwd?: string) => }); }); +const isValidSha = (s: string) => /^[0-9a-f]{7,40}$/i.test(s); + export const prepareSourceWorkspace = async (deploymentId: string, gitUrl: string, branch?: string, commitSha?: string) => { const root = join(config.workspaceRoot, deploymentId); await rm(root, { recursive: true, force: true }); await mkdir(root, { recursive: true }); if (commitSha) { - await run('git', ['clone', gitUrl, root]); - await run('git', ['checkout', commitSha], root); + if (!isValidSha(commitSha)) { + throw new Error(`Invalid commit SHA: ${commitSha}`); + } + await run('git', ['init'], root); + await run('git', ['remote', 'add', 'origin', gitUrl], root); + await run('git', ['fetch', '--depth', '1', 'origin', commitSha], root); + await run('git', ['checkout', 'FETCH_HEAD'], root); } else { const args = ['clone', '--depth', '1']; if (branch) args.push('--branch', branch); diff --git a/apps/web/src/components/project/deployments/DeploymentsTab.tsx b/apps/web/src/components/project/deployments/DeploymentsTab.tsx index 53c0de1..624430e 100644 --- a/apps/web/src/components/project/deployments/DeploymentsTab.tsx +++ b/apps/web/src/components/project/deployments/DeploymentsTab.tsx @@ -8,85 +8,15 @@ import { useCancelDeployment, useDeleteDeployment, } from "../../../hooks/useDeployments"; -import { - Dialog, - DialogContent, - DialogHeader, - DialogTitle, - DialogDescription, - DialogFooter, -} from "../../ui/dialog"; -import { useDeploymentLogs } from "../../../hooks/useDeploymentLogs"; -import { StatusBadge } from "../../StatusBadge"; import { Button } from "../../ui/button"; import { Input } from "../../ui/input"; -import { Badge } from "../../ui/badge"; import { Card, CardContent, CardHeader, CardTitle } from "../../ui/card"; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "../../ui/table"; -import { Rocket, Play, RefreshCw, RotateCcw, Terminal, ChevronLeft, ChevronRight, History } from "lucide-react"; -import { cn } from "../../../lib/utils"; - -function formatTimeAgo(dateStr: string) { - const diff = - Date.now() - new Date(dateStr).getTime(); - const mins = Math.floor(diff / 60000); - if (mins < 1) return "just now"; - if (mins < 60) return `${mins}m ago`; - const hours = Math.floor(mins / 60); - if (hours < 24) return `${hours}h ago`; - return `${Math.floor(hours / 24)}d ago`; -} - -function parseTimestamp(raw: string) { - if (!raw) return Date.now(); - const normalized = raw.includes(" ") && !raw.includes("T") ? raw.replace(" ", "T") : raw; - const d = new Date(normalized); - return Number.isNaN(d.getTime()) ? Date.now() : d.getTime(); -} - -function DeploymentDuration({ deployment }: { deployment: any }) { - const [duration, setDuration] = useState(""); - - useEffect(() => { - const calculate = () => { - const start = parseTimestamp(deployment.createdAt); - const status = deployment.status; - const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; - const end = isFinished ? parseTimestamp(deployment.updatedAt) : Date.now(); - - const diff = Math.max(0, end - start); - const secs = Math.floor(diff / 1000); - if (secs < 60) { - setDuration(`${secs}s`); - } else { - const mins = Math.floor(secs / 60); - const remainingSecs = secs % 60; - setDuration(`${mins}m ${remainingSecs}s`); - } - }; - - calculate(); - - const status = deployment.status; - const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; - if (isFinished) return; - - const interval = setInterval(calculate, 1000); - return () => clearInterval(interval); - }, [deployment.createdAt, deployment.updatedAt, deployment.status]); - - return {duration}; -} +import { Rocket, Play } from "lucide-react"; +import { ManualDeployDialog } from "./manual-deploy-dialog"; +import { DeploymentHistory } from "./deployment-history"; const PAGE_SIZE = 5; -function depDisplayName(projectName: string | undefined, depId: string) { - const short = depId.slice(0, 8); - if (!projectName) return short; - const slug = projectName.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63); - return `${slug}-${short}`; -} - interface DeploymentsTabProps { projectId: string; } @@ -106,12 +36,6 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { const redeploy = useRedeployDeployment(); const cancel = useCancelDeployment(); const deleteDep = useDeleteDeployment(); - const [deleteConfirmId, setDeleteConfirmId] = useState< - string | null - >(null); - const [cancelConfirmId, setCancelConfirmId] = useState< - string | null - >(null); const [selectedId, setSelectedId] = useState< string | null >(null); @@ -247,42 +171,13 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { }; const [showManualDeployDialog, setShowManualDeployDialog] = useState(false); - const [deployOption, setDeployOption] = useState<"latest" | "commit">("latest"); - const [commitSha, setCommitSha] = useState(""); - const [clearCache, setClearCache] = useState(false); - const handleManualDeploy = async () => { - const form = new FormData(); + const handleManualDeploy = async (form: FormData) => { form.set("sourceType", "git"); if (projectId) form.set("projectId", projectId); - if (project?.repoUrl) form.set("gitUrl", project.repoUrl); - if (project?.repoBranch) form.set("branch", project.repoBranch); - if (environment) form.set("environment", environment); - - if (deployOption === "commit") { - if (!commitSha.trim()) return; - form.set("commitSha", commitSha.trim()); - } - - if (clearCache) { - form.set("clearCache", "true"); - } - - try { - await createDeployment.mutateAsync(form); - setShowManualDeployDialog(false); - setCommitSha(""); - setDeployOption("latest"); - setClearCache(false); - } catch (err) { - console.error("Failed to start manual deployment:", err); - } + await createDeployment.mutateAsync(form); }; - const selectedDeployment = deployments.find( - (d) => d.id === selectedId, - ); - return (
@@ -525,442 +420,34 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { )} {totalDeployments > 0 && ( -
- - - - - Status - - - Deployment - - - Source - - - Branch - - - Duration - - - Age - - - - - - {deployments.map( - (dep) => ( - - setSelectedId( - selectedId === - dep.id - ? null - : dep.id, - ) - } - > - -
- - {dep.status === "running" && ( - - Active - - )} -
- - - {depDisplayName( - project?.name, - dep.id, - )} - - - { - dep.sourceType - } - - - {dep.branch ? ( - - { - dep.branch - } - - ) : ( - - — - - )} - - - - - - {formatTimeAgo( - dep.createdAt, - )} - - -
- {dep.status === "running" && dep.imageTag && dep.sourceType !== "image" && ( - - )} - {(dep.status === "pending" || dep.status === "building") && ( - - )} - {dep.imageTag && dep.status !== "running" && dep.status !== "pending" && dep.status !== "building" && dep.status !== "deploying" && ( - - )} - {dep.status !== "running" && ( - - )} -
- - - ), - )} - -
- {totalPages > 1 && ( -
- - {totalDeployments} total deployments - -
- - - {page + 1} / {totalPages} - - -
-
- )} -
- )} - - {selectedDeployment && ( - redeploy.mutate(id)} + onRollback={(id) => rollback.mutate(id)} + onCancel={(id) => cancel.mutate(id)} + onDelete={(id) => deleteDep.mutate(id)} /> )} - { if (!open) setCancelConfirmId(null); }}> - - - Cancel deployment? - - This will stop the current build/deploy process. The deployment will be marked as failed. This cannot be undone. - - - - - - - - - - { if (!open) setDeleteConfirmId(null); }}> - - - Delete deployment? - - This will remove the deployment record, its build logs, and stop its container. The Docker image is kept for potential rollback. This action cannot be undone. - - - - - - - - - - - - - - - Manual Deployment - - - Build and deploy a new version of {project?.name}. - - - -
- {/* Deploy Option Selection */} -
- -
- - -
-
- - {/* Commit Hash Input */} - {deployOption === "commit" && ( -
- - setCommitSha(e.target.value)} - className="bg-[#121215] border-[#222227] text-zinc-200 focus:border-amber-500 text-xs font-mono h-9" - /> -
- )} - - {/* Clear Cache Toggle */} -
- setClearCache(e.target.checked)} - className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" - /> -
setClearCache(!clearCache)}> - - - Bypasses cached Buildkit stages to force a clean dependency fetch and build. - -
-
-
- - - - - - - +
); } -function fmtLogTs(raw: string | undefined) { - if (!raw) return ""; - const d = new Date(raw); - if (Number.isNaN(d.getTime())) return raw; - const pad = (n: number) => String(n).padStart(2, "0"); - return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; -} -function DeploymentLogs({ - deployment, -}: { - deployment: any; -}) { - const { logs, isLoading } = useDeploymentLogs( - deployment.id, - ); - const endRef = useRef(null); - useEffect(() => { - endRef.current?.scrollIntoView({ - behavior: "smooth", - }); - }, [logs]); - - return ( - - - - - - Build Logs —{" "} - {deployment.id.slice(0, 8)} - - - Duration: - - - - - - {isLoading ? ( -
- Loading logs... -
- ) : logs.length === 0 ? ( -
- No build logs available - for this deployment. -
- ) : ( -
- {logs.map((log, i) => ( -
- - [{log.stage}]-[{fmtLogTs((log as any).timestamp || log.createdAt)}] - - - {log.message} - -
- ))} -
-
- )} - - - ); -} diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx new file mode 100644 index 0000000..49afcf0 --- /dev/null +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -0,0 +1,221 @@ +import React, { useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "../../ui/dialog"; +import { StatusBadge } from "../../StatusBadge"; +import { Button } from "../../ui/button"; +import { Badge } from "../../ui/badge"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "../../ui/table"; +import { RefreshCw, ChevronLeft, ChevronRight, History } from "lucide-react"; +import { cn } from "../../../lib/utils"; +import { formatTimeAgo, depDisplayName, DeploymentDuration, DeploymentLogs } from "./deployment-logs"; + +interface DeploymentHistoryProps { + deployments: any[]; + page: number; + totalPages: number; + totalDeployments: number; + projectName: string | undefined; + onPageChange: (page: number) => void; + onSelect: (id: string | null) => void; + selectedId: string | null; + onRedeploy: (id: string) => void; + onRollback: (id: string) => void; + onCancel: (id: string) => void; + onDelete: (id: string) => void; +} + +export function DeploymentHistory({ + deployments, + page, + totalPages, + totalDeployments, + projectName, + onPageChange, + onSelect, + selectedId, + onRedeploy, + onRollback, + onCancel, + onDelete, +}: DeploymentHistoryProps) { + const [deleteConfirmId, setDeleteConfirmId] = useState(null); + const [cancelConfirmId, setCancelConfirmId] = useState(null); + + const selectedDeployment = deployments.find((d) => d.id === selectedId); + + return ( + <> +
+ + + + Status + Deployment + Source + Branch + Duration + Age + + + + + {deployments.map((dep) => ( + + onSelect(selectedId === dep.id ? null : dep.id) + } + > + +
+ + {dep.status === "running" && ( + + Active + + )} +
+ + + {depDisplayName(projectName, dep.id)} + + + {dep.sourceType} + + + {dep.branch ? ( + {dep.branch} + ) : ( + — + )} + + + + + + {formatTimeAgo(dep.createdAt)} + + +
+ {dep.status === "running" && dep.imageTag && dep.sourceType !== "image" && ( + + )} + {(dep.status === "pending" || dep.status === "building") && ( + + )} + {dep.imageTag && dep.status !== "running" && dep.status !== "pending" && dep.status !== "building" && dep.status !== "deploying" && ( + + )} + {dep.status !== "running" && ( + + )} +
+ + + ))} + +
+ {totalPages > 1 && ( +
+ + {totalDeployments} total deployments + +
+ + + {page + 1} / {totalPages} + + +
+
+ )} +
+ + {selectedDeployment && } + + { if (!open) setCancelConfirmId(null); }}> + + + Cancel deployment? + + This will stop the current build/deploy process. The deployment will be marked as failed. This cannot be undone. + + + + + + + + + + { if (!open) setDeleteConfirmId(null); }}> + + + Delete deployment? + + This will remove the deployment record, its build logs, and stop its container. The Docker image is kept for potential rollback. This action cannot be undone. + + + + + + + + + + ); +} diff --git a/apps/web/src/components/project/deployments/deployment-logs.tsx b/apps/web/src/components/project/deployments/deployment-logs.tsx new file mode 100644 index 0000000..7eacf56 --- /dev/null +++ b/apps/web/src/components/project/deployments/deployment-logs.tsx @@ -0,0 +1,134 @@ +import React, { useState, useEffect, useRef } from "react"; +import { useDeploymentLogs } from "../../../hooks/useDeploymentLogs"; +import { Card, CardContent, CardHeader, CardTitle } from "../../ui/card"; +import { Terminal } from "lucide-react"; + +export function formatTimeAgo(dateStr: string) { + const diff = + Date.now() - new Date(dateStr).getTime(); + const mins = Math.floor(diff / 60000); + if (mins < 1) return "just now"; + if (mins < 60) return `${mins}m ago`; + const hours = Math.floor(mins / 60); + if (hours < 24) return `${hours}h ago`; + return `${Math.floor(hours / 24)}d ago`; +} + +export function parseTimestamp(raw: string) { + if (!raw) return Date.now(); + const normalized = raw.includes(" ") && !raw.includes("T") ? raw.replace(" ", "T") : raw; + const d = new Date(normalized); + return Number.isNaN(d.getTime()) ? Date.now() : d.getTime(); +} + +export function depDisplayName(projectName: string | undefined, depId: string) { + const short = depId.slice(0, 8); + if (!projectName) return short; + const slug = projectName.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63); + return `${slug}-${short}`; +} + +export function DeploymentDuration({ deployment }: { deployment: any }) { + const [duration, setDuration] = useState(""); + + useEffect(() => { + const calculate = () => { + const start = parseTimestamp(deployment.createdAt); + const status = deployment.status; + const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; + const end = isFinished ? parseTimestamp(deployment.updatedAt) : Date.now(); + + const diff = Math.max(0, end - start); + const secs = Math.floor(diff / 1000); + if (secs < 60) { + setDuration(`${secs}s`); + } else { + const mins = Math.floor(secs / 60); + const remainingSecs = secs % 60; + setDuration(`${mins}m ${remainingSecs}s`); + } + }; + + calculate(); + + const status = deployment.status; + const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; + if (isFinished) return; + + const interval = setInterval(calculate, 1000); + return () => clearInterval(interval); + }, [deployment.createdAt, deployment.updatedAt, deployment.status]); + + return {duration}; +} + +function fmtLogTs(raw: string | undefined) { + if (!raw) return ""; + const d = new Date(raw); + if (Number.isNaN(d.getTime())) return raw; + const pad = (n: number) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; +} + +export function DeploymentLogs({ + deployment, +}: { + deployment: any; +}) { + const { logs, isLoading } = useDeploymentLogs( + deployment.id, + ); + const endRef = useRef(null); + useEffect(() => { + endRef.current?.scrollIntoView({ + behavior: "smooth", + }); + }, [logs]); + + return ( + + + + + + Build Logs —{" "} + {deployment.id.slice(0, 8)} + + + Duration: + + + + + + {isLoading ? ( +
+ Loading logs... +
+ ) : logs.length === 0 ? ( +
+ No build logs available + for this deployment. +
+ ) : ( +
+ {logs.map((log, i) => ( +
+ + [{log.stage}]-[{fmtLogTs((log as any).timestamp || log.createdAt)}] + + + {log.message} + +
+ ))} +
+
+ )} + + + ); +} diff --git a/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx b/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx new file mode 100644 index 0000000..76b8f74 --- /dev/null +++ b/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx @@ -0,0 +1,172 @@ +import React, { useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "../../ui/dialog"; +import { Button } from "../../ui/button"; +import { Input } from "../../ui/input"; +import { Rocket } from "lucide-react"; +import { cn } from "../../../lib/utils"; + +interface ManualDeployDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + projectId: string; + projectName: string; + repoUrl: string | null | undefined; + repoBranch: string | null | undefined; + isPending: boolean; + onDeploy: (form: FormData) => Promise; +} + +export function ManualDeployDialog({ + open, + onOpenChange, + projectName, + repoUrl, + repoBranch, + isPending, + onDeploy, +}: ManualDeployDialogProps) { + const [deployOption, setDeployOption] = useState<"latest" | "commit">("latest"); + const [commitSha, setCommitSha] = useState(""); + const [clearCache, setClearCache] = useState(false); + const [error, setError] = useState(null); + + const handleDeploy = async () => { + const form = new FormData(); + form.set("sourceType", "git"); + if (repoUrl) form.set("gitUrl", repoUrl); + if (repoBranch) form.set("branch", repoBranch); + + if (deployOption === "commit") { + if (!commitSha.trim()) return; + form.set("commitSha", commitSha.trim()); + } + + if (clearCache) { + form.set("clearCache", "true"); + } + + try { + await onDeploy(form); + setDeployOption("latest"); + setCommitSha(""); + setClearCache(false); + setError(null); + onOpenChange(false); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to start deployment"); + } + }; + + return ( + { if (!v) setError(null); onOpenChange(v); }}> + + + + + Manual Deployment + + + Build and deploy a new version of {projectName}. + + + +
+
+ +
+ + +
+
+ + {deployOption === "commit" && ( +
+ + setCommitSha(e.target.value)} + className="bg-[#121215] border-[#222227] text-zinc-200 focus:border-amber-500 text-xs font-mono h-9" + /> +
+ )} + +
+ setClearCache(e.target.checked)} + className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" + /> +
setClearCache(!clearCache)}> + + + Bypasses cached Buildkit stages to force a clean dependency fetch and build. + +
+
+
+ + {error && ( +
+
+ {error} +
+
+ )} + + + + + + + ); +} diff --git a/docker-compose.yml b/docker-compose.yml index a6a5f82..f63cf1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,9 +27,10 @@ services: start_period: 5s api: - image: ghcr.io/lftobs/dequel/api:latest - # build: - # context: ./apps/api + # image: ghcr.io/lftobs/dequel/api:latest + build: + context: . + dockerfile: ./apps/api/Dockerfile environment: PORT: 3001 DATABASE_PATH: /app/data/dequel.db @@ -88,16 +89,22 @@ services: aliases: - pam-auth healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:4567/health').status == 200 else 1)"] + test: + [ + "CMD", + "python3", + "-c", + "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:4567/health').status == 200 else 1)", + ] interval: 5s timeout: 3s retries: 5 start_period: 10s web: - image: ghcr.io/lftobs/dequel/web:latest - # build: - # context: ./apps/web + # image: ghcr.io/lftobs/dequel/web:latest + build: + context: ./apps/web healthcheck: test: [ From ffb25573d04d1a4c43beaba7b3fe838c9cffecce Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 4 Jul 2026 12:01:21 +0100 Subject: [PATCH 15/43] feat(deployments): add ClearCacheToggle UI - Remove legacy migration snapshot at apps/api/src/db/migrations/meta/0002_snapshot.json to clean up migrations. - Add ClearCacheToggle component and integrate into DeploymentsTab and ManualDeployDialog. --- .../src/db/migrations/meta/0002_snapshot.json | 1296 ----------------- .../project/deployments/DeploymentsTab.tsx | 18 +- .../deployments/clear-cache-toggle.tsx | 37 + .../deployments/manual-deploy-dialog.tsx | 25 +- 4 files changed, 51 insertions(+), 1325 deletions(-) delete mode 100644 apps/api/src/db/migrations/meta/0002_snapshot.json create mode 100644 apps/web/src/components/project/deployments/clear-cache-toggle.tsx diff --git a/apps/api/src/db/migrations/meta/0002_snapshot.json b/apps/api/src/db/migrations/meta/0002_snapshot.json deleted file mode 100644 index fd83b7d..0000000 --- a/apps/api/src/db/migrations/meta/0002_snapshot.json +++ /dev/null @@ -1,1296 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "8d240493-b31a-41dc-bbe1-76c17a481530", - "prevId": "00000000-0000-0000-0000-000000000000", - "tables": { - "alerts": { - "name": "alerts", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "threshold": { - "name": "threshold", - "type": "real", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "duration_seconds": { - "name": "duration_seconds", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "channel": { - "name": "channel", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'email'" - }, - "destination": { - "name": "destination", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "enabled": { - "name": "enabled", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "alerts_project_id_projects_id_fk": { - "name": "alerts_project_id_projects_id_fk", - "tableFrom": "alerts", - "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "api_keys": { - "name": "api_keys", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "key_hash": { - "name": "key_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "permissions": { - "name": "permissions", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'deploy:read'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "last_used_at": { - "name": "last_used_at", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "databases": { - "name": "databases", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "version": { - "name": "version", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "database_name": { - "name": "database_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "username": { - "name": "username", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "internal_host": { - "name": "internal_host", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "internal_port": { - "name": "internal_port", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "cpu_limit": { - "name": "cpu_limit", - "type": "real", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "memory_limit_mb": { - "name": "memory_limit_mb", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "connection_string": { - "name": "connection_string", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'provisioning'" - }, - "container_name": { - "name": "container_name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "databases_project_id_projects_id_fk": { - "name": "databases_project_id_projects_id_fk", - "tableFrom": "databases", - "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "deployment_logs": { - "name": "deployment_logs", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "deployment_id": { - "name": "deployment_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sequence": { - "name": "sequence", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "stage": { - "name": "stage", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "message": { - "name": "message", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_logs_dep_seq": { - "name": "idx_logs_dep_seq", - "columns": [ - "deployment_id", - "sequence" - ], - "isUnique": true - } - }, - "foreignKeys": { - "deployment_logs_deployment_id_deployments_id_fk": { - "name": "deployment_logs_deployment_id_deployments_id_fk", - "tableFrom": "deployment_logs", - "tableTo": "deployments", - "columnsFrom": [ - "deployment_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "deployments": { - "name": "deployments", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "source_type": { - "name": "source_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source_ref": { - "name": "source_ref", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "image_tag": { - "name": "image_tag", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "container_name": { - "name": "container_name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "route_path": { - "name": "route_path", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "live_url": { - "name": "live_url", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "branch": { - "name": "branch", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "commit_sha": { - "name": "commit_sha", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "replicas": { - "name": "replicas", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "environment": { - "name": "environment", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "failure_reason": { - "name": "failure_reason", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "clear_cache": { - "name": "clear_cache", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "domains": { - "name": "domains", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "domain": { - "name": "domain", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'custom'" - }, - "validation_status": { - "name": "validation_status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "ssl_status": { - "name": "ssl_status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "domains_project_id_projects_id_fk": { - "name": "domains_project_id_projects_id_fk", - "tableFrom": "domains", - "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "environment_variables": { - "name": "environment_variables", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "key": { - "name": "key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "value": { - "name": "value", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "value_encrypted": { - "name": "value_encrypted", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "value_iv": { - "name": "value_iv", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "value_tag": { - "name": "value_tag", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "environment": { - "name": "environment", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'production'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_env_vars_project": { - "name": "idx_env_vars_project", - "columns": [ - "project_id", - "environment" - ], - "isUnique": false - } - }, - "foreignKeys": { - "environment_variables_project_id_projects_id_fk": { - "name": "environment_variables_project_id_projects_id_fk", - "tableFrom": "environment_variables", - "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "github_integrations": { - "name": "github_integrations", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "client_id": { - "name": "client_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "client_secret": { - "name": "client_secret", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "app_name": { - "name": "app_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Dequel'" - }, - "webhook_secret": { - "name": "webhook_secret", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "projects": { - "name": "projects", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "description": { - "name": "description", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "repo_url": { - "name": "repo_url", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "repo_branch": { - "name": "repo_branch", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "base_domain": { - "name": "base_domain", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "cpu_limit": { - "name": "cpu_limit", - "type": "real", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "memory_limit_mb": { - "name": "memory_limit_mb", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "port": { - "name": "port", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "source_dir": { - "name": "source_dir", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "source_type": { - "name": "source_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'git'" - }, - "github_token_encrypted": { - "name": "github_token_encrypted", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "github_token_iv": { - "name": "github_token_iv", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "github_token_tag": { - "name": "github_token_tag", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "refresh_tokens": { - "name": "refresh_tokens", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "username": { - "name": "username", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "token_hash": { - "name": "token_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blacklisted_at": { - "name": "blacklisted_at", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "refresh_tokens_token_hash_unique": { - "name": "refresh_tokens_token_hash_unique", - "columns": [ - "token_hash" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "scaling_policies": { - "name": "scaling_policies", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "min_replicas": { - "name": "min_replicas", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "max_replicas": { - "name": "max_replicas", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 5 - }, - "cpu_threshold_percent": { - "name": "cpu_threshold_percent", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 70 - }, - "memory_threshold_percent": { - "name": "memory_threshold_percent", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 85 - }, - "cooldown_seconds": { - "name": "cooldown_seconds", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 120 - }, - "enabled": { - "name": "enabled", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "scaling_policies_project_id_unique": { - "name": "scaling_policies_project_id_unique", - "columns": [ - "project_id" - ], - "isUnique": true - } - }, - "foreignKeys": { - "scaling_policies_project_id_projects_id_fk": { - "name": "scaling_policies_project_id_projects_id_fk", - "tableFrom": "scaling_policies", - "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "servers": { - "name": "servers", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "host": { - "name": "host", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "port": { - "name": "port", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 2375 - }, - "auth_token": { - "name": "auth_token", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "''" - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "cpu_total": { - "name": "cpu_total", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "memory_total_mb": { - "name": "memory_total_mb", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "disk_total_mb": { - "name": "disk_total_mb", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "cpu_used_percent": { - "name": "cpu_used_percent", - "type": "real", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "memory_used_mb": { - "name": "memory_used_mb", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "last_heartbeat": { - "name": "last_heartbeat", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "smtp_settings": { - "name": "smtp_settings", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "host": { - "name": "host", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "port": { - "name": "port", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 587 - }, - "user": { - "name": "user", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "''" - }, - "pass_encrypted": { - "name": "pass_encrypted", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "pass_iv": { - "name": "pass_iv", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "pass_tag": { - "name": "pass_tag", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "from_address": { - "name": "from_address", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'dequel@localhost'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "volumes": { - "name": "volumes", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "project_id": { - "name": "project_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "mount_path": { - "name": "mount_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'/app/data'" - }, - "size_mb": { - "name": "size_mb", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "docker_volume_name": { - "name": "docker_volume_name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "volumes_project_id_projects_id_fk": { - "name": "volumes_project_id_projects_id_fk", - "tableFrom": "volumes", - "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/apps/web/src/components/project/deployments/DeploymentsTab.tsx b/apps/web/src/components/project/deployments/DeploymentsTab.tsx index 624430e..c53ede8 100644 --- a/apps/web/src/components/project/deployments/DeploymentsTab.tsx +++ b/apps/web/src/components/project/deployments/DeploymentsTab.tsx @@ -14,6 +14,7 @@ import { Card, CardContent, CardHeader, CardTitle } from "../../ui/card"; import { Rocket, Play } from "lucide-react"; import { ManualDeployDialog } from "./manual-deploy-dialog"; import { DeploymentHistory } from "./deployment-history"; +import { ClearCacheToggle } from "./clear-cache-toggle"; const PAGE_SIZE = 5; @@ -292,18 +293,11 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { accept=".zip,.tar,.tar.gz,.tgz" className="file:mr-3 file:py-1 file:px-3 file:rounded-md file:border-0 file:text-xs file:font-medium file:bg-primary file:text-primary-foreground" /> -
- setClearCache(e.target.checked)} - className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" - /> - -
+
void; + id?: string; + label?: string; + description?: string; +} + +export function ClearCacheToggle({ + checked, + onChange, + id = "clearCache", + label = "Clear build cache for this deployment", + description, +}: ClearCacheToggleProps) { + return ( +
+ onChange(e.target.checked)} + className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" + /> +
onChange(!checked)}> + + {description && ( + + {description} + + )} +
+
+ ); +} diff --git a/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx b/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx index 76b8f74..aa7ce6f 100644 --- a/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx +++ b/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx @@ -11,6 +11,7 @@ import { Button } from "../../ui/button"; import { Input } from "../../ui/input"; import { Rocket } from "lucide-react"; import { cn } from "../../../lib/utils"; +import { ClearCacheToggle } from "./clear-cache-toggle"; interface ManualDeployDialogProps { open: boolean; @@ -122,23 +123,13 @@ export function ManualDeployDialog({
)} -
- setClearCache(e.target.checked)} - className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" - /> -
setClearCache(!clearCache)}> - - - Bypasses cached Buildkit stages to force a clean dependency fetch and build. - -
-
+
{error && ( From 5c49ed9b00cc7932305c363523fdbbaae2ac102b Mon Sep 17 00:00:00 2001 From: lftobs Date: Sun, 12 Jul 2026 00:54:00 +0100 Subject: [PATCH 16/43] fix: robust migration and UI log tweaks - Harden migration - Improve deployment logs UI --- apps/api/src/db/migrate.ts | 6 +- .../project/deployments/deployment-logs.tsx | 115 ++++++++++++++---- apps/web/src/index.css | 5 + 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 47506f6..9b59800 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -37,8 +37,10 @@ const addClearCacheColumn = async (db: ReturnType) => { try { db.run(sql`ALTER TABLE deployments ADD COLUMN clear_cache integer NOT NULL DEFAULT 0`); console.log("[Migrate] Added clear_cache column to deployments table"); - } catch { - // Column already exists — no-op + } catch (err) { + if (err instanceof Error && err.message.includes("duplicate column name")) return; + console.error("[Migrate] Failed to add clear_cache column:", err); + throw err; } }; diff --git a/apps/web/src/components/project/deployments/deployment-logs.tsx b/apps/web/src/components/project/deployments/deployment-logs.tsx index 7eacf56..1bf9e3e 100644 --- a/apps/web/src/components/project/deployments/deployment-logs.tsx +++ b/apps/web/src/components/project/deployments/deployment-logs.tsx @@ -1,6 +1,15 @@ -import React, { useState, useEffect, useRef } from "react"; +import React, { + useState, + useEffect, + useRef, +} from "react"; import { useDeploymentLogs } from "../../../hooks/useDeploymentLogs"; -import { Card, CardContent, CardHeader, CardTitle } from "../../ui/card"; +import { + Card, + CardContent, + CardHeader, + CardTitle, +} from "../../ui/card"; import { Terminal } from "lucide-react"; export function formatTimeAgo(dateStr: string) { @@ -16,57 +25,101 @@ export function formatTimeAgo(dateStr: string) { export function parseTimestamp(raw: string) { if (!raw) return Date.now(); - const normalized = raw.includes(" ") && !raw.includes("T") ? raw.replace(" ", "T") : raw; + const normalized = + raw.includes(" ") && !raw.includes("T") + ? raw.replace(" ", "T") + : raw; const d = new Date(normalized); - return Number.isNaN(d.getTime()) ? Date.now() : d.getTime(); + return Number.isNaN(d.getTime()) + ? Date.now() + : d.getTime(); } -export function depDisplayName(projectName: string | undefined, depId: string) { +export function depDisplayName( + projectName: string | undefined, + depId: string, +) { const short = depId.slice(0, 8); if (!projectName) return short; - const slug = projectName.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63); + const slug = projectName + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63); return `${slug}-${short}`; } -export function DeploymentDuration({ deployment }: { deployment: any }) { +export function DeploymentDuration({ + deployment, +}: { + deployment: any; +}) { const [duration, setDuration] = useState(""); useEffect(() => { const calculate = () => { - const start = parseTimestamp(deployment.createdAt); + const start = parseTimestamp( + deployment.createdAt, + ); const status = deployment.status; - const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; - const end = isFinished ? parseTimestamp(deployment.updatedAt) : Date.now(); - + const isFinished = + status !== "pending" && + status !== "building" && + status !== "deploying"; + const end = isFinished + ? parseTimestamp( + deployment.updatedAt, + ) + : Date.now(); + const diff = Math.max(0, end - start); const secs = Math.floor(diff / 1000); if (secs < 60) { setDuration(`${secs}s`); } else { - const mins = Math.floor(secs / 60); + const mins = Math.floor( + secs / 60, + ); const remainingSecs = secs % 60; - setDuration(`${mins}m ${remainingSecs}s`); + setDuration( + `${mins}m ${remainingSecs}s`, + ); } }; calculate(); - + const status = deployment.status; - const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; + const isFinished = + status !== "pending" && + status !== "building" && + status !== "deploying"; if (isFinished) return; - const interval = setInterval(calculate, 1000); + const interval = setInterval( + calculate, + 1000, + ); return () => clearInterval(interval); - }, [deployment.createdAt, deployment.updatedAt, deployment.status]); + }, [ + deployment.createdAt, + deployment.updatedAt, + deployment.status, + ]); - return {duration}; + return ( + + {duration} + + ); } function fmtLogTs(raw: string | undefined) { if (!raw) return ""; const d = new Date(raw); if (Number.isNaN(d.getTime())) return raw; - const pad = (n: number) => String(n).padStart(2, "0"); + const pad = (n: number) => + String(n).padStart(2, "0"); return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; } @@ -92,11 +145,18 @@ export function DeploymentLogs({ Build Logs —{" "} - {deployment.id.slice(0, 8)} + {deployment.id.slice( + 0, + 8, + )} Duration: - + @@ -115,10 +175,19 @@ export function DeploymentLogs({ {logs.map((log, i) => (
- [{log.stage}]-[{fmtLogTs((log as any).timestamp || log.createdAt)}] + [{log.stage} + ]-[ + {fmtLogTs( + ( + log as any + ) + .timestamp || + log.createdAt, + )} + ] {log.message} diff --git a/apps/web/src/index.css b/apps/web/src/index.css index 259e045..a1cff50 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -78,6 +78,11 @@ opacity: 0.4; } +.log-line.error { + color: hsl(0 72% 56%); + opacity: 0.7; +} + .log-stage { color: hsl(24 95% 53%); flex-shrink: 0; From 894a13579183d3195ebedc47b37ad0f7136b725e Mon Sep 17 00:00:00 2001 From: lftobs Date: Sun, 12 Jul 2026 01:43:20 +0100 Subject: [PATCH 17/43] Refactor(grafana): overhaul dashboards and logging - Improve migration error handling when adding clear_cache column - Pass projectId to Grafana dashboard creation and load project domains - Overhaul Grafana dashboards to system overview with metrics - Highlight CRITICAL and ERROR logs in deployment logs --- apps/api/src/db/migrate.ts | 3 +- apps/api/src/orchestrator/pipeline.ts | 1 + apps/api/src/utils/grafana.ts | 449 ++++++++++++++++-- .../project/deployments/deployment-logs.tsx | 2 +- .../grafana/dashboards/deployed-apps.json | 430 +++++++++++++++-- 5 files changed, 810 insertions(+), 75 deletions(-) diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 9b59800..70797f7 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -38,7 +38,8 @@ const addClearCacheColumn = async (db: ReturnType) => { db.run(sql`ALTER TABLE deployments ADD COLUMN clear_cache integer NOT NULL DEFAULT 0`); console.log("[Migrate] Added clear_cache column to deployments table"); } catch (err) { - if (err instanceof Error && err.message.includes("duplicate column name")) return; + const cause = err instanceof Error && "cause" in err ? err.cause : err; + if (cause instanceof Error && cause.message.includes("duplicate column name")) return; console.error("[Migrate] Failed to add clear_cache column:", err); throw err; } diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 3e75590..6cc4ec8 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -554,6 +554,7 @@ export class PipelineOrchestrator { .slice(0, 63); const containerRegex = `${dashSlug}-.*`; ensureProjectDashboard( + deployment.projectId!, projectName, containerRegex, ).catch((e) => diff --git a/apps/api/src/utils/grafana.ts b/apps/api/src/utils/grafana.ts index 08304bd..7559654 100644 --- a/apps/api/src/utils/grafana.ts +++ b/apps/api/src/utils/grafana.ts @@ -1,4 +1,5 @@ import { config } from "./config"; +import { listDomains } from "../db/repo"; interface GrafanaDashboard { dashboard: { @@ -62,6 +63,7 @@ async function grafanaPost( } export async function ensureProjectDashboard( + projectId: string, projectName: string, containerRegex: string, ): Promise { @@ -71,6 +73,19 @@ export async function ensureProjectDashboard( .replace(/^-+|-+$/g, "") .slice(0, 63); + const domains = [`${slug}.${config.caddyBaseDomain}`]; + try { + const projectDomains = await listDomains(projectId); + const verified = projectDomains.filter(d => d.validationStatus === 'verified'); + for (const d of verified) { + domains.push(d.domain); + } + } catch (e) { + console.warn("[Grafana] Failed to list domains for dashboard query:", e); + } + + const regexEscaped = domains.map(d => d.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\\\$&')).join('|'); + const dashboard: GrafanaDashboard = { dashboard: { title: `Dequel \u2014 ${projectName}`, @@ -79,20 +94,400 @@ export async function ensureProjectDashboard( schemaVersion: 39, version: 1, timezone: "browser", - refresh: "30s", + refresh: "10s", panels: [ { type: "row", - title: "Resource Usage", + title: "Overview", collapsed: false, gridPos: { h: 1, w: 24, x: 0, y: 0 }, }, + { + id: 5, + type: "stat", + title: "Requests (period)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 0, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "blue" + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__range]))`, + refId: "A" + } + ] + }, + { + id: 6, + type: "stat", + title: "% Success", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 3, y: 1 }, + fieldConfig: { + defaults: { + unit: "percent", + color: { mode: "thresholds" }, + thresholds: { + mode: "absolute", + steps: [ + { color: "red", value: null }, + { color: "yellow", value: 90 }, + { color: "green", value: 95 } + ] + } + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `(sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 400 [$__range])) / sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__range])) * 100) or 0`, + refId: "A" + } + ] + }, + { + id: 7, + type: "stat", + title: "Avg Latency", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 6, y: 1 }, + fieldConfig: { + defaults: { + unit: "s", + color: { mode: "thresholds" }, + thresholds: { + mode: "absolute", + steps: [ + { color: "green", value: null }, + { color: "yellow", value: 0.5 }, + { color: "red", value: 2.0 } + ] + } + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `avg_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__range])`, + refId: "A" + } + ] + }, + { + id: 8, + type: "stat", + title: "Reqs (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 9, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "blue" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [2m]))`, + refId: "A" + } + ] + }, + { + id: 9, + type: "stat", + title: "% Success (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 12, y: 1 }, + fieldConfig: { + defaults: { + unit: "percent", + color: { mode: "thresholds" }, + thresholds: { + mode: "absolute", + steps: [ + { color: "red", value: null }, + { color: "yellow", value: 90 }, + { color: "green", value: 95 } + ] + } + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `(sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 400 [2m])) / sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [2m])) * 100) or 0`, + refId: "A" + } + ] + }, + { + id: 10, + type: "stat", + title: "HTTP 1/2xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2, x: 15, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "green" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 300 [2m]))`, + refId: "A" + } + ] + }, + { + id: 11, + type: "stat", + title: "HTTP 3xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2, x: 17, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "orange" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 300 and status < 400 [2m]))`, + refId: "A" + } + ] + }, + { + id: 12, + type: "stat", + title: "HTTP 4xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2.5, x: 19, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "yellow" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 400 and status < 500 [2m]))`, + refId: "A" + } + ] + }, + { + id: 13, + type: "stat", + title: "HTTP 5xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2.5, x: 21.5, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "red" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 500 [2m]))`, + refId: "A" + } + ] + }, + { + type: "row", + title: "HTTP Ingress Performance", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 4 }, + }, + { + id: 14, + type: "timeseries", + title: "HTTP Requests / Ingress", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 8, x: 0, y: 5 }, + fieldConfig: { + defaults: { + unit: "reqps", + custom: { fillOpacity: 10, lineWidth: 1.5 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(rate({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval])) by (request_host)`, + legendFormat: "{{request_host}}", + refId: "A" + } + ] + }, + { + id: 15, + type: "timeseries", + title: "HTTP Status Codes", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 8, x: 8, y: 5 }, + fieldConfig: { + defaults: { + unit: "reqps", + custom: { fillOpacity: 10, lineWidth: 1.5 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(rate({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval])) by (status)`, + legendFormat: "HTTP {{status}}", + refId: "A" + } + ] + }, + { + id: 16, + type: "timeseries", + title: "Total HTTP Requests", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 8, x: 16, y: 5 }, + fieldConfig: { + defaults: { + unit: "none", + custom: { fillOpacity: 25, lineWidth: 1 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval]))`, + legendFormat: "Requests", + refId: "A" + } + ] + }, + { + type: "row", + title: "Latency", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 13 }, + }, + { + id: 17, + type: "timeseries", + title: "Latency (Average Percentiles)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 12, x: 0, y: 14 }, + fieldConfig: { + defaults: { + unit: "s", + custom: { fillOpacity: 10, lineWidth: 1.5 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `quantile_over_time(0.99, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "p99", + refId: "A" + }, + { + datasource: { type: "loki", uid: "loki" }, + expr: `quantile_over_time(0.95, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "p95", + refId: "B" + }, + { + datasource: { type: "loki", uid: "loki" }, + expr: `quantile_over_time(0.50, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "p50 (median)", + refId: "C" + }, + { + datasource: { type: "loki", uid: "loki" }, + expr: `avg_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "Average", + refId: "D" + } + ] + }, + { + id: 18, + type: "heatmap", + title: "Latency Heatmap", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 12, x: 12, y: 14 }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `log_histogram({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + refId: "A" + } + ] + }, + { + type: "row", + title: "Resource Usage", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 22 }, + }, { id: 1, type: "timeseries", title: "CPU Usage", datasource: { type: "prometheus", uid: "prometheus" }, - gridPos: { h: 9, w: 12, x: 0, y: 1 }, + gridPos: { h: 8, w: 12, x: 0, y: 23 }, fieldConfig: { defaults: { unit: "short", @@ -126,7 +521,7 @@ export async function ensureProjectDashboard( type: "timeseries", title: "Memory Usage", datasource: { type: "prometheus", uid: "prometheus" }, - gridPos: { h: 9, w: 12, x: 12, y: 1 }, + gridPos: { h: 8, w: 12, x: 12, y: 23 }, fieldConfig: { defaults: { unit: "bytes", @@ -157,55 +552,37 @@ export async function ensureProjectDashboard( }, { type: "row", - title: "Request Metrics", + title: "Logs & Troubleshooting", collapsed: false, - gridPos: { h: 1, w: 24, x: 0, y: 10 }, + gridPos: { h: 1, w: 24, x: 0, y: 31 }, }, { - id: 4, - type: "timeseries", - title: "Request Rate", + id: 19, + type: "logs", + title: "HTTP Request Error Logs", datasource: { type: "loki", uid: "loki" }, - gridPos: { h: 9, w: 24, x: 0, y: 11 }, - fieldConfig: { - defaults: { - unit: "reqps", - custom: { - fillOpacity: 30, - lineWidth: 1, - }, - }, - overrides: [], - }, + gridPos: { h: 10, w: 12, x: 0, y: 32 }, options: { - legend: { - displayMode: "table", - placement: "right", - showLegend: true, - }, - tooltip: { mode: "multi" }, + showLabels: true, + showTime: true, + wrapLogMessage: true, + enableLogDetails: true, + dedupStrategy: "none", }, targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum by(host) (count_over_time({container=~"${containerRegex}"} | json [5m]))`, - legendFormat: "{{host}}", + expr: `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 400`, refId: "A", }, ], }, - { - type: "row", - title: "Logs", - collapsed: false, - gridPos: { h: 1, w: 24, x: 0, y: 20 }, - }, { id: 3, type: "logs", - title: "Container Logs", + title: "Application Container Logs", datasource: { type: "loki", uid: "loki" }, - gridPos: { h: 12, w: 24, x: 0, y: 21 }, + gridPos: { h: 10, w: 12, x: 12, y: 32 }, options: { showLabels: true, showTime: true, diff --git a/apps/web/src/components/project/deployments/deployment-logs.tsx b/apps/web/src/components/project/deployments/deployment-logs.tsx index 1bf9e3e..fe5396a 100644 --- a/apps/web/src/components/project/deployments/deployment-logs.tsx +++ b/apps/web/src/components/project/deployments/deployment-logs.tsx @@ -175,7 +175,7 @@ export function DeploymentLogs({ {logs.map((log, i) => (
[{log.stage} diff --git a/infra/monitoring/grafana/dashboards/deployed-apps.json b/infra/monitoring/grafana/dashboards/deployed-apps.json index dc9fc72..9fd599f 100644 --- a/infra/monitoring/grafana/dashboards/deployed-apps.json +++ b/infra/monitoring/grafana/dashboards/deployed-apps.json @@ -1,11 +1,11 @@ { - "title": "Dequel — Deployed Apps", + "title": "Dequel — System & Apps Overview", "uid": "dequel-deployed-apps", "tags": ["dequel"], "schemaVersion": 39, - "version": 1, + "version": 2, "timezone": "browser", - "refresh": "30s", + "refresh": "10s", "templating": { "list": [ { @@ -36,37 +36,403 @@ "panels": [ { "type": "row", - "title": "Resource Usage", + "title": "Key Metrics", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } }, { "id": 1, + "type": "gauge", + "title": "CPU Busy", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 85 } + ] + } + } + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[5m])) * 100 / max(machine_cpu_cores) or sum(rate(container_cpu_usage_seconds_total{name=~\".*\"}[5m])) * 100 / max(machine_cpu_cores)", + "refId": "A" + } + ] + }, + { + "id": 2, + "type": "gauge", + "title": "Used RAM Memory", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + } + } + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_working_set_bytes{id=\"/\"} * 100 / machine_memory_bytes or sum(container_memory_working_set_bytes{name=~\".*\"}) * 100 / max(machine_memory_bytes)", + "refId": "A" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Incoming Data (Total)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 4, "x": 12, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed" + }, + "fixedColor": "green" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_network_receive_bytes_total)", + "refId": "A" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Outgoing Data (Total)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 4, "x": 16, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed" + }, + "fixedColor": "green" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_network_transmit_bytes_total)", + "refId": "A" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Total Transferred Data", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 4, "x": 20, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed" + }, + "fixedColor": "green" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_network_receive_bytes_total) + sum(container_network_transmit_bytes_total)", + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "System Statistics / Ingress", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } + }, + { + "id": 6, + "type": "stat", + "title": "Bitrate Download", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "bps", + "color": { + "mode": "fixed" + }, + "fixedColor": "blue" + } + }, + "options": { + "graphMode": "line", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_network_receive_bytes_total[5m])) * 8", + "refId": "A" + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Bitrate Upload", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "bps", + "color": { + "mode": "fixed" + }, + "fixedColor": "blue" + } + }, + "options": { + "graphMode": "line", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_network_transmit_bytes_total[5m])) * 8", + "refId": "A" + } + ] + }, + { + "id": 8, + "type": "stat", + "title": "Running Projects", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "none", + "color": { + "mode": "fixed" + }, + "fixedColor": "orange" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "dequel_active_deployments", + "refId": "A" + } + ] + }, + { + "id": 9, + "type": "stat", + "title": "API Uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "fixed" + }, + "fixedColor": "blue" + } + }, + "options": { + "graphMode": "none", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "dequel_uptime_seconds", + "refId": "A" + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "Total API Requests", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "none", + "color": { + "mode": "fixed" + }, + "fixedColor": "purple" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "dequel_requests_total", + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Resource History", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 } + }, + { + "id": 11, "type": "timeseries", - "title": "CPU Usage", + "title": "CPU Usage (Cores)", "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "fieldConfig": { "defaults": { "unit": "short", "custom": { - "stacking": { "mode": "normal" }, - "fillOpacity": 30, - "lineWidth": 1 + "fillOpacity": 20, + "lineWidth": 1.5 } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { "mode": "multi" } + } }, "targets": [ { @@ -81,32 +447,22 @@ ] }, { - "id": 2, + "id": 12, "type": "timeseries", "title": "Memory Usage", "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 1 }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { - "stacking": { "mode": "normal" }, - "fillOpacity": 30, - "lineWidth": 1 + "fillOpacity": 20, + "lineWidth": 1.5 } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { "mode": "multi" } + } }, "targets": [ { @@ -124,17 +480,17 @@ "type": "row", "title": "Logs", "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 } + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 } }, { - "id": 3, + "id": 13, "type": "logs", "title": "Container Logs", "datasource": { "type": "loki", "uid": "loki" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 11 }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 21 }, "options": { "showLabels": true, "showTime": true, From 0a56270b33d9ea7fb30f78b99d076e9edf62f8ac Mon Sep 17 00:00:00 2001 From: lftobs Date: Sun, 12 Jul 2026 02:00:40 +0100 Subject: [PATCH 18/43] fix(api): fix project deletion --- apps/api/src/api/projects/index.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index 0c5045d..66eb821 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -6,7 +6,7 @@ import { listProjects, getProjectById, updateProject, - deleteProject, + deleteProjectCascade, listDomains, } from "../../db/repo"; import { tryRun, reloadCaddy } from "../../orchestrator/runtime"; @@ -79,7 +79,6 @@ export const projectsRoutes = new Elysia() return { error: "Project not found" }; } - // Docker containers cleanup for (const name of info.deploymentContainerNames) { await tryRun(dockerBin, ['stop', '-t', '5', name]); await tryRun(dockerBin, ['rm', '-f', name]); @@ -89,22 +88,18 @@ export const projectsRoutes = new Elysia() await tryRun(dockerBin, ['rm', '-f', name]); } - // Docker volumes cleanup for (const name of [...info.databaseVolumeNames, ...info.volumeDockerNames]) { await tryRun(dockerBin, ['volume', 'rm', '-f', name]); } - // Docker images cleanup for (const tag of info.deploymentImageTags) { if (tag) await tryRun(dockerBin, ['rmi', '-f', tag]); } - // Remove domains from Caddy route file for (const { domain, projectName } of info.domains) { await removeFromCaddyRoute(domain, id, projectName); } - // Delete the Caddy route file for this project's slug const caddyFilePath = join(config.caddyRoutesDir, `${info.slug}.caddy`); await unlink(caddyFilePath).catch(() => {}); await reloadCaddy().catch(() => {}); @@ -206,14 +201,14 @@ export const projectsRoutes = new Elysia() } const regexEscaped = domains.map(d => d.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\\\$&')).join('|'); - + // Loki metric query for request rate const query = `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [5m]))`; - + const end = Math.floor(Date.now() / 1000); const start = end - (6 * 60 * 60); // 6 hours ago const step = "60s"; // 1 min resolution - + try { const response = await fetch( `http://loki:3100/loki/api/v1/query_range?query=${encodeURIComponent(query)}&start=${start}&end=${end}&step=${step}` @@ -304,8 +299,8 @@ export const projectsRoutes = new Elysia() ws.onmessage = (event) => { if (closed) return; try { - const dataStr = typeof event.data === "string" - ? event.data + const dataStr = typeof event.data === "string" + ? event.data : new TextDecoder().decode(event.data as any); const data = JSON.parse(dataStr) as any; if (data.streams) { From f87ee8b75443c63d3fe2a748813b4443a6989769 Mon Sep 17 00:00:00 2001 From: lftobs Date: Mon, 13 Jul 2026 03:50:28 +0100 Subject: [PATCH 19/43] refactor(docs): reorder docs menu and polish UI - Add orderedSlugs and sort doc groups to enforce docs order - Polish hero UI: tweak line breaks and button markup - Polish HowItWorks UI: replace arrow with pulse indicator - Remove Next CTA blocks from docs to streamline navigation - Minor whitespace fixes in auth.md and installation.md --- apps/docs/src/components/Hero.astro | 29 ++++-- apps/docs/src/components/HowItWorks.astro | 24 +++-- apps/docs/src/content/docs/auth.md | 1 + apps/docs/src/content/docs/changelog.md | 1 + apps/docs/src/content/docs/configuration.md | 6 -- apps/docs/src/content/docs/databases.md | 6 -- apps/docs/src/content/docs/deployments.md | 6 -- apps/docs/src/content/docs/domains.md | 6 -- apps/docs/src/content/docs/env-vars.md | 6 -- apps/docs/src/content/docs/index.md | 7 +- apps/docs/src/content/docs/installation.md | 1 + apps/docs/src/content/docs/quickstart.md | 6 -- apps/docs/src/content/docs/scaling.md | 6 -- apps/docs/src/content/docs/ssl.md | 5 - apps/docs/src/content/docs/system-config.md | 1 + apps/docs/src/content/docs/volumes.md | 6 -- apps/docs/src/layouts/Layout.astro | 106 ++++++++++++++++++++ 17 files changed, 149 insertions(+), 74 deletions(-) diff --git a/apps/docs/src/components/Hero.astro b/apps/docs/src/components/Hero.astro index 0a7c8d8..359b666 100644 --- a/apps/docs/src/components/Hero.astro +++ b/apps/docs/src/components/Hero.astro @@ -10,7 +10,7 @@ const { tag = "Private Beta", titleNormal = "Deploy anything.", titleItalic = "Scale everything.", - subText = "The self-hosted cloud platform for developers who want Railway-grade deployment workflows on their own infrastructure. Container orchestration, observability, and scaling — all from one dashboard." + subText = "The self-hosted cloud platform for developers who want Railway-grade deployment workflows on their own infrastructure. Container orchestration, observability, and scaling — all from one dashboard.", } = Astro.props; --- @@ -21,7 +21,7 @@ const {

- {titleNormal}
+ {titleNormal}
{titleItalic}

@@ -33,15 +33,28 @@ const { Read the docs → - -
diff --git a/apps/docs/src/components/HowItWorks.astro b/apps/docs/src/components/HowItWorks.astro index 2ea0527..5dcc486 100644 --- a/apps/docs/src/components/HowItWorks.astro +++ b/apps/docs/src/components/HowItWorks.astro @@ -60,13 +60,18 @@
Client Request
my-app.com
- -
→
+ +
+ + +
Git Push
Repo Webhook
-
→
+
+ +
@@ -79,18 +84,23 @@ Auto SSL / Proxy PORT 80/443 -
→
+
+ + +
Dequel API Docker Orchestrator -
- ↓ +
+
-
→
+
+ +
diff --git a/apps/docs/src/content/docs/auth.md b/apps/docs/src/content/docs/auth.md index 7bb9fad..704e7fa 100644 --- a/apps/docs/src/content/docs/auth.md +++ b/apps/docs/src/content/docs/auth.md @@ -51,3 +51,4 @@ API keys are scoped to the entire platform with the same permissions as the user | `GET` | `/api/health` | None | Health check | The web dashboard redirects to `/login` when no valid session is detected. + diff --git a/apps/docs/src/content/docs/changelog.md b/apps/docs/src/content/docs/changelog.md index fdc5455..da489ff 100644 --- a/apps/docs/src/content/docs/changelog.md +++ b/apps/docs/src/content/docs/changelog.md @@ -37,3 +37,4 @@ slug: changelog ### Fixed - Railpack build timeout handling and log scrolling + diff --git a/apps/docs/src/content/docs/configuration.md b/apps/docs/src/content/docs/configuration.md index ac38575..26fbc0a 100644 --- a/apps/docs/src/content/docs/configuration.md +++ b/apps/docs/src/content/docs/configuration.md @@ -44,9 +44,3 @@ app.listen(PORT, '0.0.0.0', () => { }); -
- - Next: Deployments - - -
diff --git a/apps/docs/src/content/docs/databases.md b/apps/docs/src/content/docs/databases.md index 5c0cd14..c907adb 100644 --- a/apps/docs/src/content/docs/databases.md +++ b/apps/docs/src/content/docs/databases.md @@ -35,9 +35,3 @@ To inject this secret into your container runtime, navigate to **Environment Var DATABASE_URL=postgresql://postgres:random-password@db-my-web-app.dequel.local:5432/postgres -
- - Next: Persistent Volumes - - -
diff --git a/apps/docs/src/content/docs/deployments.md b/apps/docs/src/content/docs/deployments.md index 7d5a4c0..811d51f 100644 --- a/apps/docs/src/content/docs/deployments.md +++ b/apps/docs/src/content/docs/deployments.md @@ -29,9 +29,3 @@ Each deployment creates an isolated, immutable build artifact. - **Build Logs:** View the live compiler steps as Docker pulls layers, executes commands, and builds cache images. - **Zero-Downtime Rollbacks:** Instantly switch traffic back to a previous build artifact by clicking the **Promote** button on any previously successful deployment. -
- - Next: Environment Variables - - -
diff --git a/apps/docs/src/content/docs/domains.md b/apps/docs/src/content/docs/domains.md index 1605aee..dd3e3e7 100644 --- a/apps/docs/src/content/docs/domains.md +++ b/apps/docs/src/content/docs/domains.md @@ -29,9 +29,3 @@ In your DNS registrar's control panel (Cloudflare, GoDaddy, Namecheap, etc.), ad - **Verified:** DNS records resolve correctly. The proxy router is configured. - **Failed:** Resolving record targets did not match the required target targets. Check your DNS values. -
- - Next: SSL Certificates - - -
diff --git a/apps/docs/src/content/docs/env-vars.md b/apps/docs/src/content/docs/env-vars.md index 28422b0..fbb0ce3 100644 --- a/apps/docs/src/content/docs/env-vars.md +++ b/apps/docs/src/content/docs/env-vars.md @@ -26,9 +26,3 @@ Because environment variables are injected during container startup, modifying o Dequel displays a **Redeployment Warning Banner** at the top of the tab whenever variables are mutated. Click the **Redeploy Now** button on the banner to execute a zero-downtime rolling update with the new configurations. -
- - Next: Scaling Policies - - -
diff --git a/apps/docs/src/content/docs/index.md b/apps/docs/src/content/docs/index.md index c283f8a..47318df 100644 --- a/apps/docs/src/content/docs/index.md +++ b/apps/docs/src/content/docs/index.md @@ -27,9 +27,4 @@ When you connect a repository or upload a folder: 3. The container image is pushed to the local cluster registry and distributed across running nodes. 4. Caddy routing endpoints are automatically updated to direct traffic to the newly created instance ports. -
- - Next: Quickstart Guide - - -
+ diff --git a/apps/docs/src/content/docs/installation.md b/apps/docs/src/content/docs/installation.md index 1dc0662..41c2a61 100644 --- a/apps/docs/src/content/docs/installation.md +++ b/apps/docs/src/content/docs/installation.md @@ -162,3 +162,4 @@ dequel update ``` This pulls the latest images from GitHub Container Registry and recreates the services. + diff --git a/apps/docs/src/content/docs/quickstart.md b/apps/docs/src/content/docs/quickstart.md index 00ea75d..201720b 100644 --- a/apps/docs/src/content/docs/quickstart.md +++ b/apps/docs/src/content/docs/quickstart.md @@ -51,9 +51,3 @@ On the deployments page, you can choose between connecting your GitHub repositor Once the deployment changes status from `BUILDING` to `READY`, click the external link button near your custom domain or the auto-generated base domain to see your running web container. -
- - Next: Configuration - - -
diff --git a/apps/docs/src/content/docs/scaling.md b/apps/docs/src/content/docs/scaling.md index c9b241d..cafb7d8 100644 --- a/apps/docs/src/content/docs/scaling.md +++ b/apps/docs/src/content/docs/scaling.md @@ -28,9 +28,3 @@ Auto-scaling lets you scale compute bounds in response to real-time resource dem To disable auto-scaling policies or switch back to static scaling, click **Delete Policy** in the scaling panel. Confirm the change in the custom Radix confirmation dialog to update the scaling configuration safely. -
- - Next: Managed Databases - - -
diff --git a/apps/docs/src/content/docs/ssl.md b/apps/docs/src/content/docs/ssl.md index d79d3ff..07724db 100644 --- a/apps/docs/src/content/docs/ssl.md +++ b/apps/docs/src/content/docs/ssl.md @@ -32,8 +32,3 @@ If your domain status is stuck on `PENDING_SSL`: - Verify that your DNS CNAME/A records have fully propagated. - Ensure that no CDN service (e.g. Cloudflare proxy) is blocking HTTP ACME challenge endpoints (`/.well-known/acme-challenge/`). -
- - Back to Intro - -
diff --git a/apps/docs/src/content/docs/system-config.md b/apps/docs/src/content/docs/system-config.md index 989c4af..5851c4f 100644 --- a/apps/docs/src/content/docs/system-config.md +++ b/apps/docs/src/content/docs/system-config.md @@ -106,3 +106,4 @@ Config file equivalent: "envEncryptionKey": "your-secure-key-here" } ``` + diff --git a/apps/docs/src/content/docs/volumes.md b/apps/docs/src/content/docs/volumes.md index 304c424..7583e1b 100644 --- a/apps/docs/src/content/docs/volumes.md +++ b/apps/docs/src/content/docs/volumes.md @@ -26,9 +26,3 @@ Under the **Volumes** tab, click **Add Volume**: Because data is tied directly to cluster block volumes, data persists even if you update environment variables, scale container replicas, or deploy code rollbacks. -
- - Next: Custom Domains - - -
diff --git a/apps/docs/src/layouts/Layout.astro b/apps/docs/src/layouts/Layout.astro index 41f0636..4156d78 100644 --- a/apps/docs/src/layouts/Layout.astro +++ b/apps/docs/src/layouts/Layout.astro @@ -34,6 +34,22 @@ const description = ""; const allDocs = await getCollection("docs"); +const orderedSlugs = [ + "docs", // Introduction + "installation", // Installation + "quickstart", // Quickstart Guide + "configuration", // Configuration + "deployments", // Deployments + "env-vars", // Environment Variables + "scaling", // Scaling Policies + "system-config", // System Configuration + "databases", // Managed Databases + "volumes", // Persistent Volumes + "auth", // Authentication & Access Control + "domains", // Custom Domains + "ssl", // SSL Certificates + "changelog", // Changelog +]; const categoryOrder = [ "Getting Started", "Core Architecture", @@ -51,6 +67,15 @@ for (const entry of allDocs) { slug: entry.data.slug, }); } + +for (const cat in grouped) { + grouped[cat].sort((a, b) => { + const indexA = orderedSlugs.indexOf(a.slug); + const indexB = orderedSlugs.indexOf(b.slug); + return indexA - indexB; + }); +} + const docMenu = categoryOrder .filter((cat) => grouped[cat]) .map((cat) => ({ title: cat, items: grouped[cat] })); @@ -477,6 +502,87 @@ const docMenu = categoryOrder } + + { + (() => { + const currentIndex = orderedSlugs.indexOf(currentSlug); + if (currentIndex === -1) return null; + + const prevSlug = currentIndex > 0 ? orderedSlugs[currentIndex - 1] : null; + const nextSlug = currentIndex < orderedSlugs.length - 1 ? orderedSlugs[currentIndex + 1] : null; + + const prevDoc = prevSlug ? allDocs.find((d) => d.data.slug === prevSlug) : null; + const nextDoc = nextSlug ? allDocs.find((d) => d.data.slug === nextSlug) : null; + + if (!prevDoc && !nextDoc) return null; + + return ( +
+ {prevDoc ? ( + + + + +
+ + Previous + + + {prevDoc.data.title} + +
+ + ) : ( + From 9ea08480b7792d2f7bcdb71ecd073537559d4a9c Mon Sep 17 00:00:00 2001 From: lftobs Date: Mon, 13 Jul 2026 04:24:41 +0100 Subject: [PATCH 20/43] chore: pending changelog for v0.2.0 --- .tegami/2026-07-13-changes-since-v0.1.1.md | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .tegami/2026-07-13-changes-since-v0.1.1.md diff --git a/.tegami/2026-07-13-changes-since-v0.1.1.md b/.tegami/2026-07-13-changes-since-v0.1.1.md new file mode 100644 index 0000000..34a9b01 --- /dev/null +++ b/.tegami/2026-07-13-changes-since-v0.1.1.md @@ -0,0 +1,27 @@ +--- +packages: + npm:dequel-api: minor + npm:dequel-web: minor + npm:dequel-docs: minor +--- + +## What's Changed + +### New Features + +- feat(api): add PAM auth and token utilities +- feat(deploy): support specific commit deployments and cache clearing +- feat(api): add drizzle migrations and deploy UI +- feat(deployments): add ClearCacheToggle UI + +### Improvements + +- refactor(api): switch PAM auth to HTTP service +- refactor(auth): add timeout and libc fixes +- refactor(grafana): overhaul dashboards and logging +- chore(scripts): improve shell compatibility in installer + +### Bug Fixes + +- fix: robust migration and UI log tweaks +- fix(api): fix project deletion cascade From f5162f98b9c445676b13691001b52be542128bee Mon Sep 17 00:00:00 2001 From: rm -rf <106287048+Lftobs@users.noreply.github.com> Date: Mon, 13 Jul 2026 05:24:48 +0100 Subject: [PATCH 21/43] Delete .tegami/2026-07-13-changes-since-v0.1.1.md --- .tegami/2026-07-13-changes-since-v0.1.1.md | 27 ---------------------- 1 file changed, 27 deletions(-) delete mode 100644 .tegami/2026-07-13-changes-since-v0.1.1.md diff --git a/.tegami/2026-07-13-changes-since-v0.1.1.md b/.tegami/2026-07-13-changes-since-v0.1.1.md deleted file mode 100644 index 34a9b01..0000000 --- a/.tegami/2026-07-13-changes-since-v0.1.1.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -packages: - npm:dequel-api: minor - npm:dequel-web: minor - npm:dequel-docs: minor ---- - -## What's Changed - -### New Features - -- feat(api): add PAM auth and token utilities -- feat(deploy): support specific commit deployments and cache clearing -- feat(api): add drizzle migrations and deploy UI -- feat(deployments): add ClearCacheToggle UI - -### Improvements - -- refactor(api): switch PAM auth to HTTP service -- refactor(auth): add timeout and libc fixes -- refactor(grafana): overhaul dashboards and logging -- chore(scripts): improve shell compatibility in installer - -### Bug Fixes - -- fix: robust migration and UI log tweaks -- fix(api): fix project deletion cascade From 8501dd369305dcef7da1e07e143ced0e3fc031c9 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 14 Jul 2026 23:58:37 +0100 Subject: [PATCH 22/43] feat(orchestrator): implement cleanup system for builds and containers - Add `DEQUEL_MANAGED_LABEL` for Docker resource tracking - Implement periodic Docker and dead-letter queue garbage collection - Add `cleanupFailedDeployment` to remove artifacts on pipeline failure - Improve worker connection management in deployment queue --- apps/api/src/databases/manager.ts | 2 + apps/api/src/git/watcher.ts | 57 -------- apps/api/src/index.ts | 4 +- .../orchestrator/__tests__/cleanup.test.ts | 86 ++++++++++++ .../__tests__/pipeline-cleanup.test.ts | 124 ++++++++++++++++++ apps/api/src/orchestrator/cleanup.ts | 49 +++++++ apps/api/src/orchestrator/pipeline.ts | 30 ++++- apps/api/src/orchestrator/queue.ts | 37 ++++-- apps/api/src/orchestrator/runtime.ts | 21 +++ apps/api/src/scaling/engine.ts | 2 + apps/api/src/utils/dequel-labels.ts | 1 + 11 files changed, 338 insertions(+), 75 deletions(-) delete mode 100644 apps/api/src/git/watcher.ts create mode 100644 apps/api/src/orchestrator/__tests__/cleanup.test.ts create mode 100644 apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts create mode 100644 apps/api/src/orchestrator/cleanup.ts create mode 100644 apps/api/src/utils/dequel-labels.ts diff --git a/apps/api/src/databases/manager.ts b/apps/api/src/databases/manager.ts index 6821bda..8c02b04 100644 --- a/apps/api/src/databases/manager.ts +++ b/apps/api/src/databases/manager.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import type { Database } from '../types'; import { updateDatabaseStatus } from '../db/repo'; @@ -52,6 +53,7 @@ export const provisionDatabase = async (dbRecord: Database): Promise => { '--name', containerName, '--network', config.dockerNetwork, '--network-alias', containerName, + '-l', DEQUEL_MANAGED_LABEL, ...(dbRecord.cpuLimit ? ['--cpus', String(dbRecord.cpuLimit)] : []), ...(dbRecord.memoryLimitMb ? ['--memory', `${Math.round(dbRecord.memoryLimitMb)}m`] : []), '-v', `${volumeName}:/var/lib/${dbRecord.type === 'mysql' ? 'mysql' : 'postgresql/data'}`, diff --git a/apps/api/src/git/watcher.ts b/apps/api/src/git/watcher.ts deleted file mode 100644 index a6da875..0000000 --- a/apps/api/src/git/watcher.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { getDb } from '../db/client'; -import { listProjects } from '../db/repo'; -import { createDeployment } from '../db/repo'; -import { orchestrator } from '../orchestrator'; -import { getRemoteSha } from '../orchestrator/source'; - -const POLL_INTERVAL_MS = 60_000; - -let interval: ReturnType | null = null; - -export const startGitWatcher = () => { - if (interval) return; - interval = setInterval(poll, POLL_INTERVAL_MS); - console.log('[GitWatcher] Started (poll every 60s)'); -}; - -export const stopGitWatcher = () => { - if (interval) { clearInterval(interval); interval = null; } -}; - -async function poll() { - try { - const db = await getDb(); - const projects = db.query( - "SELECT * FROM projects WHERE repo_url IS NOT NULL AND repo_url != ''", - ).all() as any[]; - - for (const project of projects) { - const branch = project.repo_branch || 'main'; - - const latest = db.query( - 'SELECT * FROM deployments WHERE project_id = ? AND source_type = ? ORDER BY created_at DESC LIMIT 1', - ).get(project.id, 'git') as any; - - if (!latest?.commit_sha) continue; - - const remoteSha = await getRemoteSha(project.repo_url, branch); - if (!remoteSha) continue; - - if (remoteSha === latest.commit_sha) continue; - - console.log(`[GitWatcher] New commit detected for ${project.name} (${branch}): ${remoteSha.slice(0, 7)}`); - - const dep = await createDeployment({ - projectId: project.id, - sourceType: 'git', - sourceRef: project.repo_url, - branch, - commitSha: remoteSha, - }); - - orchestrator.enqueue(dep.id); - } - } catch (err) { - console.error('[GitWatcher] Poll error:', err); - } -} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5fad630..e63a333 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,11 +9,11 @@ import { config } from './utils/config'; import { getDb } from './db/client'; import { scalingEngine } from './scaling/engine'; import { serverManager } from './servers/manager'; -import { startGitWatcher } from './git/watcher'; import { startDomainPolling } from './utils/domain-verifier'; import { alertEvaluator } from './monitoring/evaluator'; import { loadOrCreateJwtSecret } from './utils/secrets'; import { initAuth, cleanupExpiredTokens } from './utils/auth'; +import { startBuildCleanup } from './orchestrator/cleanup'; const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); @@ -27,9 +27,9 @@ const bootstrap = async () => { orchestrator.startWorker(); scalingEngine.start(); serverManager.start(); - startGitWatcher(); startDomainPolling(); alertEvaluator.start(); + startBuildCleanup(); setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts new file mode 100644 index 0000000..0986179 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const tryRunCalls: { cmd: string; args: string[] }[] = []; +const mockTryRun = mock(async (cmd: string, args: string[]) => { + tryRunCalls.push({ cmd, args }); +}); +const mockRedisDel = mock(async () => {}); +const mockRedisLlLen = mock(async () => 0); +const mockRedisQuit = mock(async () => {}); + +beforeEach(() => { + tryRunCalls.length = 0; + mockTryRun.mockClear(); + mockRedisDel.mockClear(); + mockRedisLlLen.mockClear(); + mockRedisQuit.mockClear(); +}); + +mock.module(fileUrl('../runtime'), () => ({ + tryRun: mockTryRun, +})); + +mock.module(fileUrl('../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + }, +})); + +mock.module(fileUrl('../utils/docker-bin'), () => ({ + dockerBin: '/usr/bin/docker', +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + llen = mockRedisLlLen; + del = mockRedisDel; + quit = mockRedisQuit; + }, +})); + +const { pruneDocker, pruneDlq } = await import('../cleanup'); + +describe('pruneDocker', () => { + beforeEach(() => { + tryRunCalls.length = 0; + }); + + it('prunes containers with dequel label filter only', async () => { + await pruneDocker(); + + const containerPrune = tryRunCalls.find( + c => c.args[0] === 'container' && c.args[1] === 'prune', + ); + expect(containerPrune).toBeDefined(); + expect(containerPrune!.args).toContain('-f'); + expect(containerPrune!.args).toContain('--filter'); + expect(containerPrune!.args.some(a => a.includes('com.dequel.managed'))).toBe(true); + }); + + it('prunes dangling images without -a flag (preserves cache)', async () => { + await pruneDocker(); + + const imagePrune = tryRunCalls.find( + c => c.args[0] === 'image' && c.args[1] === 'prune', + ); + expect(imagePrune).toBeDefined(); + expect(imagePrune!.args).not.toContain('-a'); + + const buildxPrune = tryRunCalls.find( + c => c.args[0] === 'buildx' && c.args[1] === 'prune', + ); + expect(buildxPrune).toBeDefined(); + expect(buildxPrune!.args).not.toContain('-a'); + }); +}); + +describe('pruneDlq', () => { + it('does not need a Redis connection per tick (shared connection)', async () => { + await pruneDlq(); + + expect(mockRedisLlLen).toHaveBeenCalled(); + expect(mockRedisQuit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts new file mode 100644 index 0000000..0e8a0b9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +let cleanupFailedDeploymentCalled = false; +let buildShouldFail = false; + +beforeEach(() => { + cleanupFailedDeploymentCalled = false; + buildShouldFail = false; +}); + +const mockDb = { + getDeploymentById: mock(() => Promise.resolve({ + id: 'dep-1', + projectId: 'proj-1', + sourceType: 'git', + sourceRef: 'https://github.com/test/repo.git', + branch: 'main', + status: 'pending', + commitSha: null, + clearCache: false, + environment: null, + imageTag: null, + containerName: null, + })), + getProjectById: mock(() => Promise.resolve({ + id: 'proj-1', + name: 'Test Project', + sourceDir: null, + port: 3000, + cpuLimit: null, + memoryLimitMb: null, + repoUrl: 'https://github.com/test/repo.git', + })), + updateDeploymentStatus: mock(() => Promise.resolve()), + updateDeploymentCommitSha: mock(() => Promise.resolve()), + appendLog: mock(async (_depId: string, stage: string, message: string) => { + if (stage === 'system' && message === 'Deployment is running') { + throw new Error('Simulated DB write failure'); + } + return { sequence: 1 }; + }), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), + listVolumes: mock(() => Promise.resolve([])), + listDeployments: mock(() => Promise.resolve([])), + listAllDatabases: mock(() => Promise.resolve([])), + deleteDeploymentAndLogs: mock(() => Promise.resolve()), + getScalingPolicy: mock(() => Promise.resolve(null)), + updateDomainValidation: mock(() => Promise.resolve()), + listDomains: mock(() => Promise.resolve([])), +}; + +mock.module(fileUrl('../../db/repo'), () => mockDb); + +mock.module(fileUrl('../runtime'), () => ({ + deployContainer: mock(() => Promise.resolve({ + containerName: 'test-project-abc12345', + liveUrl: 'http://test-project.localhost', + })), + cleanupFailedDeployment: mock(async () => { + cleanupFailedDeploymentCalled = true; + }), + ensureContainerRunning: mock(() => Promise.resolve()), + reloadCaddy: mock(() => Promise.resolve()), + tryRun: mock(() => Promise.resolve('')), +})); + +mock.module(fileUrl('../railpack'), () => ({ + buildWithRailpack: mock(async ( + _workspace: string, + _imageTag: string, + _onLog: (line: string) => Promise, + _opts?: unknown, + ) => { + if (buildShouldFail) throw new Error('Build failed'); + }), + CancelledError: class CancelledError extends Error { + constructor() { super('Cancelled'); } + }, +})); + +mock.module(fileUrl('../source'), () => ({ + prepareSourceWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + prepareUploadWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + cleanupWorkspace: mock(() => Promise.resolve()), + getHeadSha: mock(() => Promise.resolve('abc123def456')), +})); + +mock.module(fileUrl('../../utils/grafana'), () => ({ + ensureProjectDashboard: mock(() => Promise.resolve()), +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + queue: string[] = []; + rpush = mock(async (_key: string, ...values: string[]) => { this.queue.push(...values); }); + blpop = mock(async () => { const v = this.queue.shift(); return v ? ['queue', v] : null; }); + lrem = mock(async () => {}); + zadd = mock(async () => {}); + zrem = mock(async () => {}); + zrangebyscore = mock(async () => []); + quit = mock(async () => {}); + }, +})); + +const { PipelineOrchestrator } = await import('../pipeline'); + +describe('runDeployment cleanup behavior', () => { + it('does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds', async () => { + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(false); + }); + + it('calls cleanupFailedDeployment when failure occurs during build (before deployContainer)', async () => { + buildShouldFail = true; + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(true); + }); +}); diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts new file mode 100644 index 0000000..6bf3adc --- /dev/null +++ b/apps/api/src/orchestrator/cleanup.ts @@ -0,0 +1,49 @@ +import Redis from 'ioredis'; +import { config } from '../utils/config'; +import { dockerBin } from '../utils/docker-bin'; +import { tryRun } from './runtime'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; + +const DLQ_KEY = 'dequel:deploy:dlq'; +const GC_INTERVAL_MS = 1_800_000; + +let interval: ReturnType | null = null; +let redis: Redis | null = null; + +const getRedis = () => { + if (!redis) redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + return redis; +}; + +export const pruneDocker = async () => { + await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]); + await tryRun(dockerBin, ['image', 'prune', '-f']); + await tryRun(dockerBin, ['buildx', 'prune', '-f']); +}; + +export const pruneDlq = async () => { + try { + const r = getRedis(); + const size = await r.llen(DLQ_KEY); + if (size > 0) { + await r.del(DLQ_KEY); + console.log(`[Cleanup] Purged ${size} items from dead letter queue`); + } + } catch (e) { + console.warn('[Cleanup] DLQ prune failed:', e); + } +}; + +export const startBuildCleanup = () => { + if (interval) return; + console.log('[Cleanup] Docker garbage collector started (every 30min)'); + interval = setInterval(async () => { + await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e)); + await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e)); + }, GC_INTERVAL_MS); +}; + +export const stopBuildCleanup = () => { + if (interval) { clearInterval(interval); interval = null; } + if (redis) { redis.quit(); redis = null; } +}; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 3e75590..91e8b77 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -21,6 +21,7 @@ import { getHeadSha, } from "./source"; import { + cleanupFailedDeployment, deployContainer, ensureContainerRunning, reloadCaddy, @@ -158,6 +159,10 @@ export class PipelineOrchestrator { ]); } + if (deployment.imageTag) { + await tryRun("docker", ["rmi", "-f", deployment.imageTag]); + } + await deleteDeploymentAndLogs( deploymentId, ); @@ -290,6 +295,9 @@ export class PipelineOrchestrator { let workspacePath = ""; let uploadedArchivePath: string | null = null; + let imageTag: string | undefined; + let projectName: string | undefined; + let deployed = false; try { await emitLog( @@ -298,7 +306,7 @@ export class PipelineOrchestrator { "Deployment enqueued", ); - const imageTag = + imageTag = await this.resolveImageTag( deployment, ); @@ -484,7 +492,6 @@ export class PipelineOrchestrator { await this.checkCancelled(deploymentId); - let projectName: string | undefined; let cpuLimit: | number | null @@ -546,6 +553,8 @@ export class PipelineOrchestrator { }, ); + deployed = true; + if (projectName) { const dashSlug = projectName .toLowerCase() @@ -622,6 +631,23 @@ export class PipelineOrchestrator { "failed", { failureReason: message }, ); + + if (!deployed) { + await emitLog( + deploymentId, + "system", + "Cleaning up Docker resources from failed deployment", + ); + await cleanupFailedDeployment( + deploymentId, + imageTag, + projectName, + deployment.projectId, + ).catch(e => + console.warn(`[Cleanup] Failed to clean deployment ${deploymentId}:`, e), + ); + } + if (deployment.projectId) { try { const dbs = diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index 3fdfe51..2b8ecb3 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -22,12 +22,14 @@ const decodeJob = (raw: string): JobPayload | null => { } }; +const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + export class DeploymentQueue { private redis: Redis; private shuttingDown = false; constructor() { - this.redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + this.redis = createRedis(); } async enqueue(deploymentId: string) { @@ -41,7 +43,9 @@ export class DeploymentQueue { encodeJob({ id: deploymentId, attempt: i }), ); await this.redis.zrem(RETRY_KEY, ...allAttempts); - await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); + for (let i = 0; i <= config.queueRetryMax + 1; i++) { + await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: i })); + } } async start(handler: (deploymentId: string) => Promise) { @@ -55,21 +59,26 @@ export class DeploymentQueue { } private async runWorker(workerId: number, handler: (deploymentId: string) => Promise) { - while (!this.shuttingDown) { - await this.requeueDueJobs(); + const workerRedis = createRedis(); + try { + while (!this.shuttingDown) { + await this.requeueDueJobs(); - const item = await this.redis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); - if (!item) continue; - const job = decodeJob(item[1]); - if (!job) continue; + const item = await workerRedis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); + if (!item) continue; + const job = decodeJob(item[1]); + if (!job) continue; - try { - const ok = await handler(job.id); - if (!ok) await this.retryOrDlq(job); - } catch (err) { - console.error(`[Queue] Worker ${workerId} handler error:`, err); - await this.retryOrDlq(job); + try { + const ok = await handler(job.id); + if (!ok) await this.retryOrDlq(job); + } catch (err) { + console.error(`[Queue] Worker ${workerId} handler error:`, err); + await this.retryOrDlq(job); + } } + } finally { + await workerRedis.quit(); } } diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 8370f09..f1b82e1 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; const slugify = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63); @@ -99,6 +100,25 @@ export const reloadCaddy = async () => { await run(dockerBin, ['exec', caddyContainer, 'caddy', 'reload', '--config', '/etc/caddy/Caddyfile']); }; +export const getContainerName = (deploymentId: string, projectName?: string, projectId?: string) => { + const slug = slugify(projectName || projectId || deploymentId); + return `${slug}-${deploymentId.slice(0, 8)}`; +}; + +export const cleanupFailedDeployment = async ( + deploymentId: string, + imageTag?: string | null, + projectName?: string, + projectId?: string, +) => { + const containerName = getContainerName(deploymentId, projectName, projectId); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await tryRun(dockerBin, ['rm', '-f', containerName]); + if (imageTag) { + await tryRun(dockerBin, ['rmi', '-f', imageTag]); + } +}; + export const deployContainer = async ( deploymentId: string, imageTag: string, @@ -117,6 +137,7 @@ export const deployContainer = async ( 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, '-e', `PORT=${opts.appPort ?? config.appInternalPort}`, ]; diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 16e544b..afdc173 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import Redis from 'ioredis'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy } from '../db/repo'; @@ -201,6 +202,7 @@ class ScalingEngine { await run(dockerBin, [ 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, ...volumes, ...envVars, imageTag, ]); diff --git a/apps/api/src/utils/dequel-labels.ts b/apps/api/src/utils/dequel-labels.ts new file mode 100644 index 0000000..cac7a70 --- /dev/null +++ b/apps/api/src/utils/dequel-labels.ts @@ -0,0 +1 @@ +export const DEQUEL_MANAGED_LABEL = 'com.dequel.managed=true'; From 3b73df998a25231e4ae678dad3955738320ee662 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:14:29 +0100 Subject: [PATCH 23/43] feat(web): add mobile-responsive navigation and layouts - Implement mobile sidebar toggle - Make dashboard tables and tabs scrollable on small screens - Set restart policies for docker services - Improve CLI installation and path resolution --- apps/web/src/components/Layout.tsx | 14 ++- apps/web/src/components/layout/Header.tsx | 14 ++- .../components/layout/NotificationBanner.tsx | 2 +- apps/web/src/components/layout/Sidebar.tsx | 95 ++++++++++++------- .../deployments/deployment-history.tsx | 4 +- .../components/project/domains/DomainsTab.tsx | 4 +- .../components/project/envtab/EnvVarTable.tsx | 4 +- .../components/project/volumes/VolumesTab.tsx | 4 +- apps/web/src/routes/ProjectDetail.tsx | 20 ++-- apps/web/src/routes/Settings.tsx | 10 +- docker-compose.yml | 11 +++ scripts/dequel | 12 ++- scripts/install.sh | 13 +++ 13 files changed, 143 insertions(+), 64 deletions(-) diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index d47c4a9..29c4c82 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -33,6 +33,11 @@ export function Layout({ children }: { children: React.ReactNode }) { const { data: projects = [] } = useProjects({ enabled: !!me?.authenticated && location.pathname !== "/login" }); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); + const [sidebarOpen, setSidebarOpen] = useState(false); + + useEffect(() => { + setSidebarOpen(false); + }, [location.pathname, location.search]); const { data: metricsText } = useQuery({ queryKey: ["metrics"], @@ -100,7 +105,7 @@ export function Layout({ children }: { children: React.ReactNode }) { } return ( -
+
-
+
setNotification(null)} /> -
{children}
+
{children}
); diff --git a/apps/web/src/components/layout/Header.tsx b/apps/web/src/components/layout/Header.tsx index 7b686c5..cce94ce 100644 --- a/apps/web/src/components/layout/Header.tsx +++ b/apps/web/src/components/layout/Header.tsx @@ -1,19 +1,29 @@ import { Link } from "@tanstack/react-router"; -import { ChevronRight } from "lucide-react"; +import { ChevronRight, Menu } from "lucide-react"; interface HeaderProps { currentProject: { name: string } | undefined; currentProjectId: string | null; location: { pathname: string; search: any }; + setSidebarOpen: (open: boolean) => void; } export function Header({ currentProject, currentProjectId, location, + setSidebarOpen, }: HeaderProps) { return ( -
+
+ + Workspace diff --git a/apps/web/src/components/layout/NotificationBanner.tsx b/apps/web/src/components/layout/NotificationBanner.tsx index 35755d6..cb1c6ec 100644 --- a/apps/web/src/components/layout/NotificationBanner.tsx +++ b/apps/web/src/components/layout/NotificationBanner.tsx @@ -18,7 +18,7 @@ export function NotificationBanner({ const Icon = notification.type === "success" ? CheckCircle2 : AlertCircle; return ( -
+
void; + sidebarOpen: boolean; + setSidebarOpen: (open: boolean) => void; } export function Sidebar({ @@ -29,43 +33,70 @@ export function Sidebar({ metrics, location, navigate, + sidebarOpen, + setSidebarOpen, }: SidebarProps) { return ( - + ); } diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx index 49afcf0..d3e8d3c 100644 --- a/apps/web/src/components/project/deployments/deployment-history.tsx +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -51,8 +51,8 @@ export function DeploymentHistory({ return ( <> -
- +
+
Status diff --git a/apps/web/src/components/project/domains/DomainsTab.tsx b/apps/web/src/components/project/domains/DomainsTab.tsx index 3cef41e..c2ab828 100644 --- a/apps/web/src/components/project/domains/DomainsTab.tsx +++ b/apps/web/src/components/project/domains/DomainsTab.tsx @@ -366,8 +366,8 @@ export function DomainsTab({ )} -
-
+
+
diff --git a/apps/web/src/components/project/envtab/EnvVarTable.tsx b/apps/web/src/components/project/envtab/EnvVarTable.tsx index f6c4314..2c5dd58 100644 --- a/apps/web/src/components/project/envtab/EnvVarTable.tsx +++ b/apps/web/src/components/project/envtab/EnvVarTable.tsx @@ -77,8 +77,8 @@ export function EnvVarTable({ -
-
+
+
diff --git a/apps/web/src/components/project/volumes/VolumesTab.tsx b/apps/web/src/components/project/volumes/VolumesTab.tsx index d269a75..f67d4d4 100644 --- a/apps/web/src/components/project/volumes/VolumesTab.tsx +++ b/apps/web/src/components/project/volumes/VolumesTab.tsx @@ -233,8 +233,8 @@ export function VolumesTab({ -
-
+
+
diff --git a/apps/web/src/routes/ProjectDetail.tsx b/apps/web/src/routes/ProjectDetail.tsx index f1e69db..ee878f3 100644 --- a/apps/web/src/routes/ProjectDetail.tsx +++ b/apps/web/src/routes/ProjectDetail.tsx @@ -106,32 +106,32 @@ export function ProjectDetail({ navigate({ search: { tab: val } as any })} className="space-y-4"> - - + + Deployments - + Env Vars - + Volumes - + Databases - + Domains - + Scaling - + Alerts - + Observability - + Logs diff --git a/apps/web/src/routes/Settings.tsx b/apps/web/src/routes/Settings.tsx index fb16b15..466b44c 100644 --- a/apps/web/src/routes/Settings.tsx +++ b/apps/web/src/routes/Settings.tsx @@ -72,8 +72,8 @@ function ServersSection() { {servers.length > 0 && ( -
-
+
+
NameHostStatus @@ -322,7 +322,7 @@ function ApiKeysSection() { {newKey} )} - +
setName(e.target.value)} className="w-56" /> @@ -330,8 +330,8 @@ function ApiKeysSection() { {apiKeys.length > 0 && ( -
-
+
+
NameKey HashCreated diff --git a/docker-compose.yml b/docker-compose.yml index f63cf1d..0f22018 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: buildkit: image: moby/buildkit:latest + restart: unless-stopped command: [ "--addr", @@ -28,6 +29,7 @@ services: api: # image: ghcr.io/lftobs/dequel/api:latest + restart: unless-stopped build: context: . dockerfile: ./apps/api/Dockerfile @@ -73,6 +75,7 @@ services: pam-auth: image: python:3-slim + restart: unless-stopped entrypoint: [] command: - sh @@ -103,6 +106,7 @@ services: web: # image: ghcr.io/lftobs/dequel/web:latest + restart: unless-stopped build: context: ./apps/web healthcheck: @@ -124,6 +128,7 @@ services: caddy: image: caddy:2.8-alpine + restart: unless-stopped command: [ "caddy", @@ -153,6 +158,7 @@ services: redis: image: redis:7-alpine + restart: unless-stopped command: redis-server --appendonly yes volumes: - redis-data:/data @@ -168,6 +174,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest + restart: unless-stopped volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -197,6 +204,7 @@ services: prometheus: image: prom/prometheus:latest + restart: unless-stopped stop_grace_period: 60s entrypoint: [] command: @@ -229,6 +237,7 @@ services: loki: image: grafana/loki:3.0.0 + restart: unless-stopped command: - -config.file=/etc/loki/loki-config.yml volumes: @@ -254,6 +263,7 @@ services: promtail: image: grafana/promtail:3.0.0 + restart: unless-stopped command: - -config.file=/etc/promtail/promtail-config.yml volumes: @@ -269,6 +279,7 @@ services: grafana: image: grafana/grafana:latest + restart: unless-stopped environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin diff --git a/scripts/dequel b/scripts/dequel index 811a722..3e67244 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -DEQUEL_HOME="${DEQUEL_HOME:-$HOME/.dequel}" +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" +DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" VERSION="0.1.0" @@ -27,7 +28,12 @@ cmd() { cmd_start() { header "Starting Dequel" cmd up -d - success "Dequel is running at http://localhost" + local domain="${CADDY_BASE_DOMAIN:-localhost}" + if [ "$domain" = "localhost" ]; then + success "Dequel is running at http://localhost" + else + success "Dequel is running at https://$domain" + fi } cmd_stop() { @@ -130,7 +136,7 @@ cmd_help() { echo " --help Show this help" echo "" echo "Environment:" - echo " DEQUEL_HOME Config directory (default: \$HOME/.dequel)" + echo " DEQUEL_HOME Config directory (default: auto-detected from install path)" } case "${1:-}" in diff --git a/scripts/install.sh b/scripts/install.sh index dd9508a..d40e504 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -40,6 +40,19 @@ check_prerequisites() { fail "Docker is not installed. See https://docs.docker.com/engine/install/" fi + if docker info >/dev/null 2>&1; then + success "Docker socket accessible" + else + warn "Docker socket not accessible — attempting to add user to docker group" + if sudo usermod -aG docker "$USER" 2>/dev/null; then + success "Added $USER to docker group" + warn "Log out and back in (or run 'newgrp docker') for the change to take effect" + else + warn "Could not add user to docker group automatically" + warn "Run this manually: sudo usermod -aG docker $USER && newgrp docker" + fi + fi + if docker compose version >/dev/null 2>&1; then success "Docker Compose found: $(docker compose version)" COMPOSE_CMD="docker compose" From dfb54e5550f90c7ddf8c4f6c51bd2a1e4ec317da Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:28:49 +0100 Subject: [PATCH 24/43] feat(api): simplify repository fetching to owner affiliation --- apps/api/src/api/github/index.ts | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b2b6e83..f295140 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -151,24 +151,9 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 401; return { error: "Not authenticated" }; } - const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&type=all", token); - const user = await fetchGitHub("/user", token) as { login: string }; - const orgs = await fetchGitHub("/user/orgs", token) as { login: string }[]; + const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&affiliation=owner", token); const allRepos = Array.isArray(repos) ? repos : []; - const orgRepos: any[] = []; - for (const org of Array.isArray(orgs) ? orgs : []) { - try { - const orgR = await fetchGitHub(`/orgs/${org.login}/repos?per_page=100&sort=updated&type=all`, token); - if (Array.isArray(orgR)) orgRepos.push(...orgR); - } catch {} - } - const seen = new Set(); - const merged = [...allRepos, ...orgRepos].filter((r) => { - if (seen.has(r.id)) return false; - seen.add(r.id); - return true; - }); - return merged.map((r: any) => ({ + return allRepos.map((r: any) => ({ id: r.id, name: r.name, fullName: r.full_name, From e6d0e69df2d9e638be09d7b6237d1798301ed2b6 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:36:33 +0100 Subject: [PATCH 25/43] feat(scripts): implement update cmd for dequel --- scripts/dequel | 78 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/scripts/dequel b/scripts/dequel index 3e67244..fc3942d 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,10 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" -DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" +SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || realpath "$0" 2>/dev/null || echo "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +DEQUEL_HOME="${DEQUEL_HOME:-$SCRIPT_DIR}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" -VERSION="0.1.0" +INSTALLED_VERSION="$(cat "$DEQUEL_HOME/VERSION" 2>/dev/null || echo "0.1.0")" BOLD='\033[1m' DIM='\033[2m' @@ -59,9 +60,74 @@ cmd_logs() { cmd_update() { header "Updating Dequel" + + local repo="Lftobs/dequel" + local tag="" + local base_url="" + + info "Checking for latest release..." + tag=$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest" \ + | grep '"tag_name"' | head -1 | sed -E 's/.*"([^"]+)".*/\1/') || true + + if [ -z "$tag" ]; then + warn "Could not determine latest release, using main branch" + base_url="https://raw.githubusercontent.com/$repo/main" + else + base_url="https://raw.githubusercontent.com/$repo/$tag" + success "Latest release: $tag" + fi + + local tmp_dir + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" EXIT + + info "Downloading updated configuration..." + curl -fsSL "$base_url/docker-compose.yml" -o "$tmp_dir/docker-compose.yml" + curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" + curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" + curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/$f" + done + + for f in loki.yml prometheus.yml; do + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/$f" + done + + for f in dashboards.yml deployed-apps.json; do + curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/$f" + done + + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" + mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + mkdir -p "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/$f" + done + for f in loki.yml prometheus.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + done + for f in dashboards.yml deployed-apps.json; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + done + + mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" + chmod +x "$tmp_dir/dequel" + mv "$tmp_dir/dequel" "$SCRIPT_PATH" + + success "Configuration updated" + + header "Pulling Docker images" cmd pull + + header "Recreating services" cmd up -d - success "Dequel updated." + + success "Dequel updated to $tag" } cmd_uninstall() { @@ -116,7 +182,7 @@ cmd_uninstall() { } cmd_version() { - echo "dequel v$VERSION" + echo "dequel v$INSTALLED_VERSION" } cmd_help() { @@ -130,7 +196,7 @@ cmd_help() { echo " restart Restart all Dequel services" echo " status Show service status" echo " logs Follow service logs" - echo " update Pull latest images and recreate services" + echo " update Download latest config, pull images, and recreate services" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help" From 052b3ab57e6254d7248ebedac53455f8b87976f3 Mon Sep 17 00:00:00 2001 From: lftobs Date: Tue, 14 Jul 2026 23:58:37 +0100 Subject: [PATCH 26/43] feat(orchestrator): implement cleanup system for builds and containers - Add `DEQUEL_MANAGED_LABEL` for Docker resource tracking - Implement periodic Docker and dead-letter queue garbage collection - Add `cleanupFailedDeployment` to remove artifacts on pipeline failure - Improve worker connection management in deployment queue --- apps/api/src/databases/manager.ts | 2 + apps/api/src/git/watcher.ts | 57 -------- apps/api/src/index.ts | 4 +- .../orchestrator/__tests__/cleanup.test.ts | 86 ++++++++++++ .../__tests__/pipeline-cleanup.test.ts | 124 ++++++++++++++++++ apps/api/src/orchestrator/cleanup.ts | 49 +++++++ apps/api/src/orchestrator/pipeline.ts | 30 ++++- apps/api/src/orchestrator/queue.ts | 37 ++++-- apps/api/src/orchestrator/runtime.ts | 21 +++ apps/api/src/scaling/engine.ts | 2 + apps/api/src/utils/dequel-labels.ts | 1 + 11 files changed, 338 insertions(+), 75 deletions(-) delete mode 100644 apps/api/src/git/watcher.ts create mode 100644 apps/api/src/orchestrator/__tests__/cleanup.test.ts create mode 100644 apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts create mode 100644 apps/api/src/orchestrator/cleanup.ts create mode 100644 apps/api/src/utils/dequel-labels.ts diff --git a/apps/api/src/databases/manager.ts b/apps/api/src/databases/manager.ts index 6821bda..8c02b04 100644 --- a/apps/api/src/databases/manager.ts +++ b/apps/api/src/databases/manager.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import type { Database } from '../types'; import { updateDatabaseStatus } from '../db/repo'; @@ -52,6 +53,7 @@ export const provisionDatabase = async (dbRecord: Database): Promise => { '--name', containerName, '--network', config.dockerNetwork, '--network-alias', containerName, + '-l', DEQUEL_MANAGED_LABEL, ...(dbRecord.cpuLimit ? ['--cpus', String(dbRecord.cpuLimit)] : []), ...(dbRecord.memoryLimitMb ? ['--memory', `${Math.round(dbRecord.memoryLimitMb)}m`] : []), '-v', `${volumeName}:/var/lib/${dbRecord.type === 'mysql' ? 'mysql' : 'postgresql/data'}`, diff --git a/apps/api/src/git/watcher.ts b/apps/api/src/git/watcher.ts deleted file mode 100644 index a6da875..0000000 --- a/apps/api/src/git/watcher.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { getDb } from '../db/client'; -import { listProjects } from '../db/repo'; -import { createDeployment } from '../db/repo'; -import { orchestrator } from '../orchestrator'; -import { getRemoteSha } from '../orchestrator/source'; - -const POLL_INTERVAL_MS = 60_000; - -let interval: ReturnType | null = null; - -export const startGitWatcher = () => { - if (interval) return; - interval = setInterval(poll, POLL_INTERVAL_MS); - console.log('[GitWatcher] Started (poll every 60s)'); -}; - -export const stopGitWatcher = () => { - if (interval) { clearInterval(interval); interval = null; } -}; - -async function poll() { - try { - const db = await getDb(); - const projects = db.query( - "SELECT * FROM projects WHERE repo_url IS NOT NULL AND repo_url != ''", - ).all() as any[]; - - for (const project of projects) { - const branch = project.repo_branch || 'main'; - - const latest = db.query( - 'SELECT * FROM deployments WHERE project_id = ? AND source_type = ? ORDER BY created_at DESC LIMIT 1', - ).get(project.id, 'git') as any; - - if (!latest?.commit_sha) continue; - - const remoteSha = await getRemoteSha(project.repo_url, branch); - if (!remoteSha) continue; - - if (remoteSha === latest.commit_sha) continue; - - console.log(`[GitWatcher] New commit detected for ${project.name} (${branch}): ${remoteSha.slice(0, 7)}`); - - const dep = await createDeployment({ - projectId: project.id, - sourceType: 'git', - sourceRef: project.repo_url, - branch, - commitSha: remoteSha, - }); - - orchestrator.enqueue(dep.id); - } - } catch (err) { - console.error('[GitWatcher] Poll error:', err); - } -} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 5fad630..e63a333 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,11 +9,11 @@ import { config } from './utils/config'; import { getDb } from './db/client'; import { scalingEngine } from './scaling/engine'; import { serverManager } from './servers/manager'; -import { startGitWatcher } from './git/watcher'; import { startDomainPolling } from './utils/domain-verifier'; import { alertEvaluator } from './monitoring/evaluator'; import { loadOrCreateJwtSecret } from './utils/secrets'; import { initAuth, cleanupExpiredTokens } from './utils/auth'; +import { startBuildCleanup } from './orchestrator/cleanup'; const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); @@ -27,9 +27,9 @@ const bootstrap = async () => { orchestrator.startWorker(); scalingEngine.start(); serverManager.start(); - startGitWatcher(); startDomainPolling(); alertEvaluator.start(); + startBuildCleanup(); setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts new file mode 100644 index 0000000..0986179 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const tryRunCalls: { cmd: string; args: string[] }[] = []; +const mockTryRun = mock(async (cmd: string, args: string[]) => { + tryRunCalls.push({ cmd, args }); +}); +const mockRedisDel = mock(async () => {}); +const mockRedisLlLen = mock(async () => 0); +const mockRedisQuit = mock(async () => {}); + +beforeEach(() => { + tryRunCalls.length = 0; + mockTryRun.mockClear(); + mockRedisDel.mockClear(); + mockRedisLlLen.mockClear(); + mockRedisQuit.mockClear(); +}); + +mock.module(fileUrl('../runtime'), () => ({ + tryRun: mockTryRun, +})); + +mock.module(fileUrl('../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + }, +})); + +mock.module(fileUrl('../utils/docker-bin'), () => ({ + dockerBin: '/usr/bin/docker', +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + llen = mockRedisLlLen; + del = mockRedisDel; + quit = mockRedisQuit; + }, +})); + +const { pruneDocker, pruneDlq } = await import('../cleanup'); + +describe('pruneDocker', () => { + beforeEach(() => { + tryRunCalls.length = 0; + }); + + it('prunes containers with dequel label filter only', async () => { + await pruneDocker(); + + const containerPrune = tryRunCalls.find( + c => c.args[0] === 'container' && c.args[1] === 'prune', + ); + expect(containerPrune).toBeDefined(); + expect(containerPrune!.args).toContain('-f'); + expect(containerPrune!.args).toContain('--filter'); + expect(containerPrune!.args.some(a => a.includes('com.dequel.managed'))).toBe(true); + }); + + it('prunes dangling images without -a flag (preserves cache)', async () => { + await pruneDocker(); + + const imagePrune = tryRunCalls.find( + c => c.args[0] === 'image' && c.args[1] === 'prune', + ); + expect(imagePrune).toBeDefined(); + expect(imagePrune!.args).not.toContain('-a'); + + const buildxPrune = tryRunCalls.find( + c => c.args[0] === 'buildx' && c.args[1] === 'prune', + ); + expect(buildxPrune).toBeDefined(); + expect(buildxPrune!.args).not.toContain('-a'); + }); +}); + +describe('pruneDlq', () => { + it('does not need a Redis connection per tick (shared connection)', async () => { + await pruneDlq(); + + expect(mockRedisLlLen).toHaveBeenCalled(); + expect(mockRedisQuit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts new file mode 100644 index 0000000..0e8a0b9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +let cleanupFailedDeploymentCalled = false; +let buildShouldFail = false; + +beforeEach(() => { + cleanupFailedDeploymentCalled = false; + buildShouldFail = false; +}); + +const mockDb = { + getDeploymentById: mock(() => Promise.resolve({ + id: 'dep-1', + projectId: 'proj-1', + sourceType: 'git', + sourceRef: 'https://github.com/test/repo.git', + branch: 'main', + status: 'pending', + commitSha: null, + clearCache: false, + environment: null, + imageTag: null, + containerName: null, + })), + getProjectById: mock(() => Promise.resolve({ + id: 'proj-1', + name: 'Test Project', + sourceDir: null, + port: 3000, + cpuLimit: null, + memoryLimitMb: null, + repoUrl: 'https://github.com/test/repo.git', + })), + updateDeploymentStatus: mock(() => Promise.resolve()), + updateDeploymentCommitSha: mock(() => Promise.resolve()), + appendLog: mock(async (_depId: string, stage: string, message: string) => { + if (stage === 'system' && message === 'Deployment is running') { + throw new Error('Simulated DB write failure'); + } + return { sequence: 1 }; + }), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), + listVolumes: mock(() => Promise.resolve([])), + listDeployments: mock(() => Promise.resolve([])), + listAllDatabases: mock(() => Promise.resolve([])), + deleteDeploymentAndLogs: mock(() => Promise.resolve()), + getScalingPolicy: mock(() => Promise.resolve(null)), + updateDomainValidation: mock(() => Promise.resolve()), + listDomains: mock(() => Promise.resolve([])), +}; + +mock.module(fileUrl('../../db/repo'), () => mockDb); + +mock.module(fileUrl('../runtime'), () => ({ + deployContainer: mock(() => Promise.resolve({ + containerName: 'test-project-abc12345', + liveUrl: 'http://test-project.localhost', + })), + cleanupFailedDeployment: mock(async () => { + cleanupFailedDeploymentCalled = true; + }), + ensureContainerRunning: mock(() => Promise.resolve()), + reloadCaddy: mock(() => Promise.resolve()), + tryRun: mock(() => Promise.resolve('')), +})); + +mock.module(fileUrl('../railpack'), () => ({ + buildWithRailpack: mock(async ( + _workspace: string, + _imageTag: string, + _onLog: (line: string) => Promise, + _opts?: unknown, + ) => { + if (buildShouldFail) throw new Error('Build failed'); + }), + CancelledError: class CancelledError extends Error { + constructor() { super('Cancelled'); } + }, +})); + +mock.module(fileUrl('../source'), () => ({ + prepareSourceWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + prepareUploadWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + cleanupWorkspace: mock(() => Promise.resolve()), + getHeadSha: mock(() => Promise.resolve('abc123def456')), +})); + +mock.module(fileUrl('../../utils/grafana'), () => ({ + ensureProjectDashboard: mock(() => Promise.resolve()), +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + queue: string[] = []; + rpush = mock(async (_key: string, ...values: string[]) => { this.queue.push(...values); }); + blpop = mock(async () => { const v = this.queue.shift(); return v ? ['queue', v] : null; }); + lrem = mock(async () => {}); + zadd = mock(async () => {}); + zrem = mock(async () => {}); + zrangebyscore = mock(async () => []); + quit = mock(async () => {}); + }, +})); + +const { PipelineOrchestrator } = await import('../pipeline'); + +describe('runDeployment cleanup behavior', () => { + it('does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds', async () => { + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(false); + }); + + it('calls cleanupFailedDeployment when failure occurs during build (before deployContainer)', async () => { + buildShouldFail = true; + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(true); + }); +}); diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts new file mode 100644 index 0000000..6bf3adc --- /dev/null +++ b/apps/api/src/orchestrator/cleanup.ts @@ -0,0 +1,49 @@ +import Redis from 'ioredis'; +import { config } from '../utils/config'; +import { dockerBin } from '../utils/docker-bin'; +import { tryRun } from './runtime'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; + +const DLQ_KEY = 'dequel:deploy:dlq'; +const GC_INTERVAL_MS = 1_800_000; + +let interval: ReturnType | null = null; +let redis: Redis | null = null; + +const getRedis = () => { + if (!redis) redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + return redis; +}; + +export const pruneDocker = async () => { + await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]); + await tryRun(dockerBin, ['image', 'prune', '-f']); + await tryRun(dockerBin, ['buildx', 'prune', '-f']); +}; + +export const pruneDlq = async () => { + try { + const r = getRedis(); + const size = await r.llen(DLQ_KEY); + if (size > 0) { + await r.del(DLQ_KEY); + console.log(`[Cleanup] Purged ${size} items from dead letter queue`); + } + } catch (e) { + console.warn('[Cleanup] DLQ prune failed:', e); + } +}; + +export const startBuildCleanup = () => { + if (interval) return; + console.log('[Cleanup] Docker garbage collector started (every 30min)'); + interval = setInterval(async () => { + await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e)); + await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e)); + }, GC_INTERVAL_MS); +}; + +export const stopBuildCleanup = () => { + if (interval) { clearInterval(interval); interval = null; } + if (redis) { redis.quit(); redis = null; } +}; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 6cc4ec8..c672e3d 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -21,6 +21,7 @@ import { getHeadSha, } from "./source"; import { + cleanupFailedDeployment, deployContainer, ensureContainerRunning, reloadCaddy, @@ -158,6 +159,10 @@ export class PipelineOrchestrator { ]); } + if (deployment.imageTag) { + await tryRun("docker", ["rmi", "-f", deployment.imageTag]); + } + await deleteDeploymentAndLogs( deploymentId, ); @@ -290,6 +295,9 @@ export class PipelineOrchestrator { let workspacePath = ""; let uploadedArchivePath: string | null = null; + let imageTag: string | undefined; + let projectName: string | undefined; + let deployed = false; try { await emitLog( @@ -298,7 +306,7 @@ export class PipelineOrchestrator { "Deployment enqueued", ); - const imageTag = + imageTag = await this.resolveImageTag( deployment, ); @@ -484,7 +492,6 @@ export class PipelineOrchestrator { await this.checkCancelled(deploymentId); - let projectName: string | undefined; let cpuLimit: | number | null @@ -546,6 +553,8 @@ export class PipelineOrchestrator { }, ); + deployed = true; + if (projectName) { const dashSlug = projectName .toLowerCase() @@ -623,6 +632,23 @@ export class PipelineOrchestrator { "failed", { failureReason: message }, ); + + if (!deployed) { + await emitLog( + deploymentId, + "system", + "Cleaning up Docker resources from failed deployment", + ); + await cleanupFailedDeployment( + deploymentId, + imageTag, + projectName, + deployment.projectId, + ).catch(e => + console.warn(`[Cleanup] Failed to clean deployment ${deploymentId}:`, e), + ); + } + if (deployment.projectId) { try { const dbs = diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index 3fdfe51..2b8ecb3 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -22,12 +22,14 @@ const decodeJob = (raw: string): JobPayload | null => { } }; +const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + export class DeploymentQueue { private redis: Redis; private shuttingDown = false; constructor() { - this.redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + this.redis = createRedis(); } async enqueue(deploymentId: string) { @@ -41,7 +43,9 @@ export class DeploymentQueue { encodeJob({ id: deploymentId, attempt: i }), ); await this.redis.zrem(RETRY_KEY, ...allAttempts); - await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); + for (let i = 0; i <= config.queueRetryMax + 1; i++) { + await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: i })); + } } async start(handler: (deploymentId: string) => Promise) { @@ -55,21 +59,26 @@ export class DeploymentQueue { } private async runWorker(workerId: number, handler: (deploymentId: string) => Promise) { - while (!this.shuttingDown) { - await this.requeueDueJobs(); + const workerRedis = createRedis(); + try { + while (!this.shuttingDown) { + await this.requeueDueJobs(); - const item = await this.redis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); - if (!item) continue; - const job = decodeJob(item[1]); - if (!job) continue; + const item = await workerRedis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); + if (!item) continue; + const job = decodeJob(item[1]); + if (!job) continue; - try { - const ok = await handler(job.id); - if (!ok) await this.retryOrDlq(job); - } catch (err) { - console.error(`[Queue] Worker ${workerId} handler error:`, err); - await this.retryOrDlq(job); + try { + const ok = await handler(job.id); + if (!ok) await this.retryOrDlq(job); + } catch (err) { + console.error(`[Queue] Worker ${workerId} handler error:`, err); + await this.retryOrDlq(job); + } } + } finally { + await workerRedis.quit(); } } diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 8370f09..f1b82e1 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; const slugify = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63); @@ -99,6 +100,25 @@ export const reloadCaddy = async () => { await run(dockerBin, ['exec', caddyContainer, 'caddy', 'reload', '--config', '/etc/caddy/Caddyfile']); }; +export const getContainerName = (deploymentId: string, projectName?: string, projectId?: string) => { + const slug = slugify(projectName || projectId || deploymentId); + return `${slug}-${deploymentId.slice(0, 8)}`; +}; + +export const cleanupFailedDeployment = async ( + deploymentId: string, + imageTag?: string | null, + projectName?: string, + projectId?: string, +) => { + const containerName = getContainerName(deploymentId, projectName, projectId); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await tryRun(dockerBin, ['rm', '-f', containerName]); + if (imageTag) { + await tryRun(dockerBin, ['rmi', '-f', imageTag]); + } +}; + export const deployContainer = async ( deploymentId: string, imageTag: string, @@ -117,6 +137,7 @@ export const deployContainer = async ( 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, '-e', `PORT=${opts.appPort ?? config.appInternalPort}`, ]; diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 16e544b..afdc173 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import Redis from 'ioredis'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy } from '../db/repo'; @@ -201,6 +202,7 @@ class ScalingEngine { await run(dockerBin, [ 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, ...volumes, ...envVars, imageTag, ]); diff --git a/apps/api/src/utils/dequel-labels.ts b/apps/api/src/utils/dequel-labels.ts new file mode 100644 index 0000000..cac7a70 --- /dev/null +++ b/apps/api/src/utils/dequel-labels.ts @@ -0,0 +1 @@ +export const DEQUEL_MANAGED_LABEL = 'com.dequel.managed=true'; From b9d0a965f893789819e22dcd47d482922ed73091 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:14:29 +0100 Subject: [PATCH 27/43] feat(web): add mobile-responsive navigation and layouts - Implement mobile sidebar toggle - Make dashboard tables and tabs scrollable on small screens - Set restart policies for docker services - Improve CLI installation and path resolution --- apps/web/src/components/Layout.tsx | 14 ++- apps/web/src/components/layout/Header.tsx | 14 ++- .../components/layout/NotificationBanner.tsx | 2 +- apps/web/src/components/layout/Sidebar.tsx | 95 ++++++++++++------- .../deployments/deployment-history.tsx | 4 +- .../components/project/domains/DomainsTab.tsx | 4 +- .../components/project/envtab/EnvVarTable.tsx | 4 +- .../components/project/volumes/VolumesTab.tsx | 4 +- apps/web/src/routes/ProjectDetail.tsx | 20 ++-- apps/web/src/routes/Settings.tsx | 10 +- docker-compose.yml | 11 +++ scripts/dequel | 12 ++- scripts/install.sh | 13 +++ 13 files changed, 143 insertions(+), 64 deletions(-) diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index d47c4a9..29c4c82 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -33,6 +33,11 @@ export function Layout({ children }: { children: React.ReactNode }) { const { data: projects = [] } = useProjects({ enabled: !!me?.authenticated && location.pathname !== "/login" }); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); + const [sidebarOpen, setSidebarOpen] = useState(false); + + useEffect(() => { + setSidebarOpen(false); + }, [location.pathname, location.search]); const { data: metricsText } = useQuery({ queryKey: ["metrics"], @@ -100,7 +105,7 @@ export function Layout({ children }: { children: React.ReactNode }) { } return ( -
+
-
+
setNotification(null)} /> -
{children}
+
{children}
); diff --git a/apps/web/src/components/layout/Header.tsx b/apps/web/src/components/layout/Header.tsx index 7b686c5..cce94ce 100644 --- a/apps/web/src/components/layout/Header.tsx +++ b/apps/web/src/components/layout/Header.tsx @@ -1,19 +1,29 @@ import { Link } from "@tanstack/react-router"; -import { ChevronRight } from "lucide-react"; +import { ChevronRight, Menu } from "lucide-react"; interface HeaderProps { currentProject: { name: string } | undefined; currentProjectId: string | null; location: { pathname: string; search: any }; + setSidebarOpen: (open: boolean) => void; } export function Header({ currentProject, currentProjectId, location, + setSidebarOpen, }: HeaderProps) { return ( -
+
+ + Workspace diff --git a/apps/web/src/components/layout/NotificationBanner.tsx b/apps/web/src/components/layout/NotificationBanner.tsx index 35755d6..cb1c6ec 100644 --- a/apps/web/src/components/layout/NotificationBanner.tsx +++ b/apps/web/src/components/layout/NotificationBanner.tsx @@ -18,7 +18,7 @@ export function NotificationBanner({ const Icon = notification.type === "success" ? CheckCircle2 : AlertCircle; return ( -
+
void; + sidebarOpen: boolean; + setSidebarOpen: (open: boolean) => void; } export function Sidebar({ @@ -29,43 +33,70 @@ export function Sidebar({ metrics, location, navigate, + sidebarOpen, + setSidebarOpen, }: SidebarProps) { return ( - + ); } diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx index 49afcf0..d3e8d3c 100644 --- a/apps/web/src/components/project/deployments/deployment-history.tsx +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -51,8 +51,8 @@ export function DeploymentHistory({ return ( <> -
-
+
+
Status diff --git a/apps/web/src/components/project/domains/DomainsTab.tsx b/apps/web/src/components/project/domains/DomainsTab.tsx index 3cef41e..c2ab828 100644 --- a/apps/web/src/components/project/domains/DomainsTab.tsx +++ b/apps/web/src/components/project/domains/DomainsTab.tsx @@ -366,8 +366,8 @@ export function DomainsTab({ )} -
-
+
+
diff --git a/apps/web/src/components/project/envtab/EnvVarTable.tsx b/apps/web/src/components/project/envtab/EnvVarTable.tsx index f6c4314..2c5dd58 100644 --- a/apps/web/src/components/project/envtab/EnvVarTable.tsx +++ b/apps/web/src/components/project/envtab/EnvVarTable.tsx @@ -77,8 +77,8 @@ export function EnvVarTable({ -
-
+
+
diff --git a/apps/web/src/components/project/volumes/VolumesTab.tsx b/apps/web/src/components/project/volumes/VolumesTab.tsx index d269a75..f67d4d4 100644 --- a/apps/web/src/components/project/volumes/VolumesTab.tsx +++ b/apps/web/src/components/project/volumes/VolumesTab.tsx @@ -233,8 +233,8 @@ export function VolumesTab({ -
-
+
+
diff --git a/apps/web/src/routes/ProjectDetail.tsx b/apps/web/src/routes/ProjectDetail.tsx index f1e69db..ee878f3 100644 --- a/apps/web/src/routes/ProjectDetail.tsx +++ b/apps/web/src/routes/ProjectDetail.tsx @@ -106,32 +106,32 @@ export function ProjectDetail({ navigate({ search: { tab: val } as any })} className="space-y-4"> - - + + Deployments - + Env Vars - + Volumes - + Databases - + Domains - + Scaling - + Alerts - + Observability - + Logs diff --git a/apps/web/src/routes/Settings.tsx b/apps/web/src/routes/Settings.tsx index fb16b15..466b44c 100644 --- a/apps/web/src/routes/Settings.tsx +++ b/apps/web/src/routes/Settings.tsx @@ -72,8 +72,8 @@ function ServersSection() { {servers.length > 0 && ( -
-
+
+
NameHostStatus @@ -322,7 +322,7 @@ function ApiKeysSection() { {newKey} )} - +
setName(e.target.value)} className="w-56" /> @@ -330,8 +330,8 @@ function ApiKeysSection() { {apiKeys.length > 0 && ( -
-
+
+
NameKey HashCreated diff --git a/docker-compose.yml b/docker-compose.yml index f63cf1d..0f22018 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: buildkit: image: moby/buildkit:latest + restart: unless-stopped command: [ "--addr", @@ -28,6 +29,7 @@ services: api: # image: ghcr.io/lftobs/dequel/api:latest + restart: unless-stopped build: context: . dockerfile: ./apps/api/Dockerfile @@ -73,6 +75,7 @@ services: pam-auth: image: python:3-slim + restart: unless-stopped entrypoint: [] command: - sh @@ -103,6 +106,7 @@ services: web: # image: ghcr.io/lftobs/dequel/web:latest + restart: unless-stopped build: context: ./apps/web healthcheck: @@ -124,6 +128,7 @@ services: caddy: image: caddy:2.8-alpine + restart: unless-stopped command: [ "caddy", @@ -153,6 +158,7 @@ services: redis: image: redis:7-alpine + restart: unless-stopped command: redis-server --appendonly yes volumes: - redis-data:/data @@ -168,6 +174,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest + restart: unless-stopped volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -197,6 +204,7 @@ services: prometheus: image: prom/prometheus:latest + restart: unless-stopped stop_grace_period: 60s entrypoint: [] command: @@ -229,6 +237,7 @@ services: loki: image: grafana/loki:3.0.0 + restart: unless-stopped command: - -config.file=/etc/loki/loki-config.yml volumes: @@ -254,6 +263,7 @@ services: promtail: image: grafana/promtail:3.0.0 + restart: unless-stopped command: - -config.file=/etc/promtail/promtail-config.yml volumes: @@ -269,6 +279,7 @@ services: grafana: image: grafana/grafana:latest + restart: unless-stopped environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin diff --git a/scripts/dequel b/scripts/dequel index 811a722..3e67244 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -DEQUEL_HOME="${DEQUEL_HOME:-$HOME/.dequel}" +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" +DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" VERSION="0.1.0" @@ -27,7 +28,12 @@ cmd() { cmd_start() { header "Starting Dequel" cmd up -d - success "Dequel is running at http://localhost" + local domain="${CADDY_BASE_DOMAIN:-localhost}" + if [ "$domain" = "localhost" ]; then + success "Dequel is running at http://localhost" + else + success "Dequel is running at https://$domain" + fi } cmd_stop() { @@ -130,7 +136,7 @@ cmd_help() { echo " --help Show this help" echo "" echo "Environment:" - echo " DEQUEL_HOME Config directory (default: \$HOME/.dequel)" + echo " DEQUEL_HOME Config directory (default: auto-detected from install path)" } case "${1:-}" in diff --git a/scripts/install.sh b/scripts/install.sh index dd9508a..d40e504 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -40,6 +40,19 @@ check_prerequisites() { fail "Docker is not installed. See https://docs.docker.com/engine/install/" fi + if docker info >/dev/null 2>&1; then + success "Docker socket accessible" + else + warn "Docker socket not accessible — attempting to add user to docker group" + if sudo usermod -aG docker "$USER" 2>/dev/null; then + success "Added $USER to docker group" + warn "Log out and back in (or run 'newgrp docker') for the change to take effect" + else + warn "Could not add user to docker group automatically" + warn "Run this manually: sudo usermod -aG docker $USER && newgrp docker" + fi + fi + if docker compose version >/dev/null 2>&1; then success "Docker Compose found: $(docker compose version)" COMPOSE_CMD="docker compose" From a3037ad6d92723c9eb0df84c2d873205daa93d93 Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:28:49 +0100 Subject: [PATCH 28/43] feat(api): simplify repository fetching to owner affiliation --- apps/api/src/api/github/index.ts | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b2b6e83..f295140 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -151,24 +151,9 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 401; return { error: "Not authenticated" }; } - const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&type=all", token); - const user = await fetchGitHub("/user", token) as { login: string }; - const orgs = await fetchGitHub("/user/orgs", token) as { login: string }[]; + const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&affiliation=owner", token); const allRepos = Array.isArray(repos) ? repos : []; - const orgRepos: any[] = []; - for (const org of Array.isArray(orgs) ? orgs : []) { - try { - const orgR = await fetchGitHub(`/orgs/${org.login}/repos?per_page=100&sort=updated&type=all`, token); - if (Array.isArray(orgR)) orgRepos.push(...orgR); - } catch {} - } - const seen = new Set(); - const merged = [...allRepos, ...orgRepos].filter((r) => { - if (seen.has(r.id)) return false; - seen.add(r.id); - return true; - }); - return merged.map((r: any) => ({ + return allRepos.map((r: any) => ({ id: r.id, name: r.name, fullName: r.full_name, From 875eabd186ee2e68f2821598e739fb9dd1398bfa Mon Sep 17 00:00:00 2001 From: lftobs Date: Wed, 15 Jul 2026 02:36:33 +0100 Subject: [PATCH 29/43] feat(scripts): implement update cmd for dequel --- scripts/dequel | 78 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/scripts/dequel b/scripts/dequel index 3e67244..fc3942d 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,10 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd 2>/dev/null)" -DEQUEL_HOME="${DEQUEL_HOME:-${SCRIPT_DIR:-$HOME/.dequel}}" +SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || realpath "$0" 2>/dev/null || echo "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +DEQUEL_HOME="${DEQUEL_HOME:-$SCRIPT_DIR}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" -VERSION="0.1.0" +INSTALLED_VERSION="$(cat "$DEQUEL_HOME/VERSION" 2>/dev/null || echo "0.1.0")" BOLD='\033[1m' DIM='\033[2m' @@ -59,9 +60,74 @@ cmd_logs() { cmd_update() { header "Updating Dequel" + + local repo="Lftobs/dequel" + local tag="" + local base_url="" + + info "Checking for latest release..." + tag=$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest" \ + | grep '"tag_name"' | head -1 | sed -E 's/.*"([^"]+)".*/\1/') || true + + if [ -z "$tag" ]; then + warn "Could not determine latest release, using main branch" + base_url="https://raw.githubusercontent.com/$repo/main" + else + base_url="https://raw.githubusercontent.com/$repo/$tag" + success "Latest release: $tag" + fi + + local tmp_dir + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" EXIT + + info "Downloading updated configuration..." + curl -fsSL "$base_url/docker-compose.yml" -o "$tmp_dir/docker-compose.yml" + curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" + curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" + curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/$f" + done + + for f in loki.yml prometheus.yml; do + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/$f" + done + + for f in dashboards.yml deployed-apps.json; do + curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/$f" + done + + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" + mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + mkdir -p "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/$f" + done + for f in loki.yml prometheus.yml; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + done + for f in dashboards.yml deployed-apps.json; do + mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + done + + mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" + chmod +x "$tmp_dir/dequel" + mv "$tmp_dir/dequel" "$SCRIPT_PATH" + + success "Configuration updated" + + header "Pulling Docker images" cmd pull + + header "Recreating services" cmd up -d - success "Dequel updated." + + success "Dequel updated to $tag" } cmd_uninstall() { @@ -116,7 +182,7 @@ cmd_uninstall() { } cmd_version() { - echo "dequel v$VERSION" + echo "dequel v$INSTALLED_VERSION" } cmd_help() { @@ -130,7 +196,7 @@ cmd_help() { echo " restart Restart all Dequel services" echo " status Show service status" echo " logs Follow service logs" - echo " update Pull latest images and recreate services" + echo " update Download latest config, pull images, and recreate services" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help" From 15a8b9ba2f712a11452785e6252c66dfcef07a71 Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 11:44:57 +0100 Subject: [PATCH 30/43] feat: improve auth, GitHub integration, and project management - Update session cookie to `lax` and validate GitHub tokens via API - Prevent duplicate project creation by repository URL - Optimize Docker pruning and pipeline cleanup - Inject versioning into frontend and build processes - Enhance Caddy proxy trust configuration and install script feedback --- apps/api/src/api/auth/index.ts | 2 +- apps/api/src/api/github/index.ts | 246 +++++++++++++----- apps/api/src/api/index.ts | 2 +- apps/api/src/api/projects/index.ts | 20 ++ apps/api/src/db/repo/github.ts | 2 +- .../orchestrator/__tests__/cleanup.test.ts | 4 +- apps/api/src/orchestrator/cleanup.ts | 4 +- apps/api/src/orchestrator/pipeline.ts | 8 +- apps/api/src/orchestrator/queue.ts | 24 +- apps/api/src/orchestrator/runtime.ts | 3 +- apps/api/src/scaling/__tests__/engine.test.ts | 12 +- apps/web/Dockerfile | 3 + apps/web/src/components/Layout.tsx | 2 +- apps/web/src/components/github/RepoPicker.tsx | 23 +- apps/web/src/components/layout/Sidebar.tsx | 8 +- .../project/deployments/DeploymentsTab.tsx | 91 ++++++- apps/web/src/routes/Login.tsx | 2 +- apps/web/src/routes/ProjectDetail.tsx | 15 +- apps/web/vite-env.d.ts | 1 + apps/web/vite.config.ts | 7 + docker-compose.yml | 2 + infra/caddy/Caddyfile | 8 +- scripts/dequel | 15 +- scripts/install.sh | 1 + 24 files changed, 381 insertions(+), 124 deletions(-) create mode 100644 apps/web/vite-env.d.ts diff --git a/apps/api/src/api/auth/index.ts b/apps/api/src/api/auth/index.ts index 8581577..b362544 100644 --- a/apps/api/src/api/auth/index.ts +++ b/apps/api/src/api/auth/index.ts @@ -22,7 +22,7 @@ const callPam = async (username: string, password: string): Promise<{ ok: boolea const SESSION_COOKIE_OPTS = { path: "/", httpOnly: true, - sameSite: "strict" as const, + sameSite: "lax" as const, secure: config.caddyBaseDomain !== "localhost", maxAge: 900, }; diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index f295140..b0cfef5 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -1,18 +1,53 @@ import { Elysia } from "elysia"; +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; import { getGithubIntegration, setGithubIntegration, createDeployment, listProjects } from "../../db/repo"; import { orchestrator } from "../../orchestrator"; import { config } from "../../utils/config"; -const SESSIONS = new Map(); -const SESSION_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours +const SESSIONS_FILE = join(process.env.DATA_DIR ?? "./data", ".github-sessions.json"); -const getSession = (cookie: string | null): string | null => { +let SESSIONS = new Map(); + +const loadSessions = () => { + try { + const raw = readFileSync(SESSIONS_FILE, "utf-8"); + const entries: [string, { token: string }][] = JSON.parse(raw); + SESSIONS = new Map(entries); + } catch {} +}; + +const saveSessions = () => { + try { + const dir = SESSIONS_FILE.substring(0, SESSIONS_FILE.lastIndexOf("/")); + mkdirSync(dir, { recursive: true }); + writeFileSync(SESSIONS_FILE, JSON.stringify([...SESSIONS]), "utf-8"); + } catch {} +}; + +loadSessions(); + +const validateToken = async (token: string): Promise => { + try { + const res = await fetch("https://api.github.com/user", { + headers: { Authorization: `Bearer ${token}`, "User-Agent": "dequel" }, + }); + return res.ok; + } catch { + return true; + } +}; + +const getSession = async (cookie: string | null): Promise => { if (!cookie) return null; const match = cookie.match(/github_session=([^;]+)/); if (!match) return null; const session = SESSIONS.get(match[1]); - if (!session || session.expiresAt < Date.now()) { - if (session) SESSIONS.delete(match[1]); + if (!session) return null; + const valid = await validateToken(session.token); + if (!valid) { + SESSIONS.delete(match[1]); + saveSessions(); return null; } return session.token; @@ -20,10 +55,31 @@ const getSession = (cookie: string | null): string | null => { const createSession = (token: string): string => { const id = crypto.randomUUID(); - SESSIONS.set(id, { token, expiresAt: Date.now() + SESSION_TTL_MS }); + SESSIONS.set(id, { token }); + saveSessions(); return id; }; +const publicUrl = (request: Request): URL => { + const proto = request.headers.get("x-forwarded-proto") || "http"; + const host = request.headers.get("x-forwarded-host") || request.headers.get("host") || "localhost"; + return new URL(`${proto}://${host}${new URL(request.url).pathname}`); +}; + +const describeGithubError = (err: unknown): { status: number; message: string } => { + const raw = err instanceof Error ? err.message : String(err); + const match = raw.match(/^GitHub API error (\d+): (.*)$/s); + if (!match) return { status: 502, message: raw }; + const status = Number(match[1]); + if (status === 403 && /not accessible by integration/i.test(match[2])) { + return { + status: 502, + message: "GitHub App is missing the 'Webhooks' repository permission. In your GitHub App settings, go to Permissions & events and set Webhooks to Read and write, approve the new permission, then try again.", + }; + } + return { status: 502, message: `GitHub API error ${status}: ${match[2]}` }; +}; + const fetchGitHub = async (path: string, token: string) => { const res = await fetch(`https://api.github.com${path}`, { headers: { @@ -84,7 +140,7 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 400; return { error: "GitHub integration not configured" }; } - const url = new URL(request.url); + const url = publicUrl(request); const redirectUri = `${url.protocol}//${url.host}/api/github/callback`; const state = crypto.randomUUID(); const params = new URLSearchParams({ @@ -107,7 +163,8 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 400; return { error: "GitHub integration not configured" }; } - const origin = new URL(request.url).origin; + const url = publicUrl(request); + const origin = url.origin; const redirectUri = `${origin}/api/github/callback`; const res = await fetch("https://github.com/login/oauth/access_token", { @@ -132,12 +189,12 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) } const sessionId = createSession(data.access_token); set.status = 302; - set.headers["Set-Cookie"] = `github_session=${sessionId}; Path=/; HttpOnly; SameSite=Lax; Max-Age=${SESSION_TTL_MS / 1000}`; + set.headers["Set-Cookie"] = `github_session=${sessionId}; Path=/; HttpOnly; SameSite=Lax; Max-Age=315360000`; set.headers["Location"] = `${origin}/?github=connected`; }) .get("/user", async ({ request, set }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; @@ -146,13 +203,19 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) }) .get("/repos", async ({ request, set }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&affiliation=owner", token); - const allRepos = Array.isArray(repos) ? repos : []; + const allRepos: any[] = []; + let page = 1; + while (page <= 10) { + const repos = await fetchGitHub(`/user/repos?per_page=100&page=${page}&sort=updated&affiliation=owner`, token); + if (!Array.isArray(repos) || repos.length === 0) break; + allRepos.push(...repos); + page++; + } return allRepos.map((r: any) => ({ id: r.id, name: r.name, @@ -168,82 +231,119 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) }) .get("/repos/:owner/:repo/hooks", async ({ request, set, params }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); - return Array.isArray(hooks) ? hooks.map((h: any) => ({ id: h.id, url: h.config.url, active: h.active, events: h.events })) : []; + try { + const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + return Array.isArray(hooks) ? hooks.map((h: any) => ({ id: h.id, url: h.config.url, active: h.active, events: h.events })) : []; + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; + } }) .post("/repos/:owner/:repo/hook", async ({ request, set, params }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const baseUrl = config.caddyBaseDomain === 'localhost' ? 'http://localhost' : `https://${config.caddyBaseDomain}`; - const webhookUrl = `${baseUrl}/api/github/webhook`; + const webhookUrl = `${publicUrl(request).origin}/api/github/webhook`; const integration = await getGithubIntegration(); const secret = integration?.webhookSecret || config.githubWebhookSecret; - const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); - const existing = Array.isArray(hooks) ? hooks.find((h: any) => h.config?.url === webhookUrl) : null; - - if (existing) { - return { id: existing.id, created: false, url: webhookUrl }; + try { + const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + const existing = Array.isArray(hooks) ? hooks.find((h: any) => h.config?.url === webhookUrl) : null; + + if (existing) { + return { id: existing.id, created: false, url: webhookUrl }; + } + + // Remove stale Dequel webhooks left over from a previous tunnel/domain + const stale = Array.isArray(hooks) + ? hooks.filter((h: any) => typeof h.config?.url === "string" && h.config.url.endsWith("/api/github/webhook") && h.config.url !== webhookUrl) + : []; + for (const h of stale) { + await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks/${h.id}`, token, "DELETE").catch(() => {}); + } + + const hook = await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks`, token, "POST", { + name: "web", + active: true, + events: ["push"], + config: { + url: webhookUrl, + content_type: "json", + secret, + insecure_ssl: "0", + }, + }); + return { id: hook.id, created: true, url: webhookUrl }; + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; } - - const hook = await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks`, token, "POST", { - name: "web", - active: true, - events: ["push"], - config: { - url: webhookUrl, - content_type: "json", - secret, - insecure_ssl: "0", - }, - }); - return { id: hook.id, created: true, url: webhookUrl }; }) .delete("/repos/:owner/:repo/hook", async ({ request, set, params }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const baseUrl = config.caddyBaseDomain === 'localhost' ? 'http://localhost' : `https://${config.caddyBaseDomain}`; - const webhookUrl = `${baseUrl}/api/github/webhook`; - const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + const webhookUrl = `${publicUrl(request).origin}/api/github/webhook`; + let hooks: any; + try { + hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; + } const existing = Array.isArray(hooks) ? hooks.find((h: any) => h.config?.url === webhookUrl) : null; if (!existing) { return { ok: true, removed: false }; } - await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks/${existing.id}`, token, "DELETE"); - return { ok: true, removed: true }; + try { + await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks/${existing.id}`, token, "DELETE"); + return { ok: true, removed: true }; + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; + } }) .post("/disconnect", async ({ set, request }) => { const cookie = request.headers.get("cookie"); const match = cookie?.match(/github_session=([^;]+)/); - if (match) SESSIONS.delete(match[1]); + if (match) { + SESSIONS.delete(match[1]); + saveSessions(); + } set.headers["Set-Cookie"] = "github_session=; Path=/; Max-Age=0"; return { ok: true }; }) .post("/webhook", async ({ request, set }) => { const integration = await getGithubIntegration(); + const event = request.headers.get("x-github-event") || ""; + const delivery = request.headers.get("x-github-delivery") || ""; + console.log(`[GitWebhook] received event=${event} delivery=${delivery}`); + if (!integration?.webhookSecret) { + console.log("[GitWebhook] rejected: no webhook secret configured on this Dequel instance"); set.status = 400; return { error: "GitHub webhook not configured" }; } const signature = request.headers.get("x-hub-signature-256") || ""; - const event = request.headers.get("x-github-event") || ""; - const delivery = request.headers.get("x-github-delivery") || ""; const rawBody = await request.text(); const encoder = new TextEncoder(); @@ -258,15 +358,18 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) const expectedSig = "sha256=" + Array.from(new Uint8Array(expectedSigRaw)).map(b => b.toString(16).padStart(2, "0")).join(""); if (signature.length !== expectedSig.length) { + console.log(`[GitWebhook] rejected delivery=${delivery}: signature length mismatch (check webhook secret matches Dequel settings)`); set.status = 401; return { error: "Invalid signature" }; } if (!crypto.timingSafeEqual(Buffer.from(expectedSig), Buffer.from(signature))) { + console.log(`[GitWebhook] rejected delivery=${delivery}: signature mismatch (check webhook secret matches Dequel settings)`); set.status = 401; return { error: "Invalid signature" }; } if (event !== "push") { + console.log(`[GitWebhook] ignored delivery=${delivery}: unsupported event ${event}`); return { ok: true, ignored: `unsupported event: ${event}` }; } @@ -287,37 +390,40 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) const branch = payload?.ref?.replace("refs/heads/", "") || "main"; const commitSha = payload?.after || ""; + if (!commitSha || commitSha === "0000000000000000000000000000000000000000") { + console.log(`[GitWebhook] ignored delivery=${delivery}: deletion event, no commit to deploy`); + return { ok: true, ignored: "deletion event, no commit to deploy" }; + } + const projects = await listProjects(); - const project = projects.find(p => - p.repoUrl && ( - p.repoUrl === repoUrl || - p.repoUrl.replace(/\.git$/, "") === repoUrl.replace(/\.git$/, "") - ) - ); + const normalize = (u: string) => u.replace(/\.git$/, "").replace(/\/+$/, "").toLowerCase(); + const matching = projects.filter(p => p.repoUrl && normalize(p.repoUrl) === normalize(repoUrl)); - if (!project) { + if (matching.length === 0) { + console.log(`[GitWebhook] ignored delivery=${delivery}: no project found for repo ${repoUrl}. Known project repos: ${projects.map(p => p.repoUrl).filter(Boolean).join(", ") || "(none)"}`); return { ok: true, ignored: `no project found for repo: ${repoUrl}` }; } - if (project.repoBranch && project.repoBranch !== branch) { - return { ok: true, ignored: `branch "${branch}" does not match project branch "${project.repoBranch}"` }; - } + const targets = matching.filter(p => !p.repoBranch || p.repoBranch === branch); - if (!commitSha || commitSha === "0000000000000000000000000000000000000000") { - return { ok: true, ignored: "deletion event, no commit to deploy" }; + if (targets.length === 0) { + console.log(`[GitWebhook] ignored delivery=${delivery}: branch "${branch}" does not match any project watching ${repoUrl} (branches: ${matching.map(p => p.repoBranch ?? "any").join(", ")})`); + return { ok: true, ignored: `branch "${branch}" does not match any watching project's branch` }; } - const dep = await createDeployment({ - projectId: project.id, - sourceType: "git", - sourceRef: repoUrl, - branch, - commitSha, - }); - - orchestrator.enqueue(dep.id); - - console.log(`[GitWebhook] Auto-deploy triggered for ${project.name} (${branch}) — commit ${commitSha.slice(0, 7)}`); + const deploymentIds: string[] = []; + for (const project of targets) { + const dep = await createDeployment({ + projectId: project.id, + sourceType: "git", + sourceRef: repoUrl, + branch, + commitSha, + }); + orchestrator.enqueue(dep.id); + deploymentIds.push(dep.id); + console.log(`[GitWebhook] Auto-deploy triggered for ${project.name} (${branch}) — commit ${commitSha.slice(0, 7)}`); + } - return { ok: true, deploymentId: dep.id }; + return { ok: true, deploymentIds }; }); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index 8d03eab..d8c56b1 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -16,7 +16,7 @@ import { serversRoutes } from "./servers"; import { volumesRoutes } from "./volumes"; import { settingsRoutes } from "./settings"; -const BYPASS_PATHS = new Set(["/api/auth/login", "/api/auth/logout", "/api/auth/refresh", "/api/auth/me", "/api/health"]); +const BYPASS_PATHS = new Set(["/api/auth/login", "/api/auth/logout", "/api/auth/refresh", "/api/auth/me", "/api/health", "/api/github/callback", "/api/github/webhook"]); const authMiddleware = (app: Elysia) => app.onBeforeHandle(async ({ request, set, path }) => { diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index 66eb821..11e5bcf 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -34,6 +34,16 @@ export const projectsRoutes = new Elysia() set.status = 400; return { error: "name is required" }; } + if (body?.repoUrl) { + const projects = await listProjects(); + const normalize = (u: string) => u.replace(/\.git$/, "").replace(/\/+$/, "").toLowerCase(); + const incoming = normalize(body.repoUrl); + const duplicate = projects.find((p) => p.repoUrl && normalize(p.repoUrl) === incoming); + if (duplicate) { + set.status = 409; + return { error: `A project with this repository URL already exists: "${duplicate.name}"` }; + } + } const project = await createProject({ name: body.name, description: body.description, @@ -52,6 +62,16 @@ export const projectsRoutes = new Elysia() .patch( "/projects/:id", async ({ params: { id }, body, set }: any) => { + if (body?.repoUrl) { + const projects = await listProjects(); + const normalize = (u: string) => u.replace(/\.git$/, "").replace(/\/+$/, "").toLowerCase(); + const incoming = normalize(body.repoUrl); + const duplicate = projects.find((p) => p.id !== id && p.repoUrl && normalize(p.repoUrl) === incoming); + if (duplicate) { + set.status = 409; + return { error: `A project with this repository URL already exists: "${duplicate.name}"` }; + } + } const project = await updateProject(id, { name: body?.name, description: body?.description, diff --git a/apps/api/src/db/repo/github.ts b/apps/api/src/db/repo/github.ts index f460e35..853bb3c 100644 --- a/apps/api/src/db/repo/github.ts +++ b/apps/api/src/db/repo/github.ts @@ -32,7 +32,7 @@ export const setGithubIntegration = async (input: { clientId: string; clientSecr clientId: input.clientId, clientSecret: input.clientSecret, appName: input.appName ?? "Dequel", - webhookSecret: input.webhookSecret ?? null, + webhookSecret: input.webhookSecret ?? existing.webhookSecret, }).where(eq(githubIntegrations.id, existing.id)).run(); const updated = tx.select().from(githubIntegrations).where(eq(githubIntegrations.id, existing.id)).get()!; return mapGithubIntegration(updated); diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts index 0986179..6c684ce 100644 --- a/apps/api/src/orchestrator/__tests__/cleanup.test.ts +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -22,13 +22,13 @@ mock.module(fileUrl('../runtime'), () => ({ tryRun: mockTryRun, })); -mock.module(fileUrl('../utils/config'), () => ({ +mock.module(fileUrl('../../utils/config'), () => ({ config: { redisUrl: 'redis://localhost:6379', }, })); -mock.module(fileUrl('../utils/docker-bin'), () => ({ +mock.module(fileUrl('../../utils/docker-bin'), () => ({ dockerBin: '/usr/bin/docker', })); diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts index 6bf3adc..9521f93 100644 --- a/apps/api/src/orchestrator/cleanup.ts +++ b/apps/api/src/orchestrator/cleanup.ts @@ -17,8 +17,8 @@ const getRedis = () => { export const pruneDocker = async () => { await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]); - await tryRun(dockerBin, ['image', 'prune', '-f']); - await tryRun(dockerBin, ['buildx', 'prune', '-f']); + await tryRun(dockerBin, ['image', 'prune', '-f', '--filter', 'until=24h']); + await tryRun(dockerBin, ['buildx', 'prune', '-f', '--filter', 'until=24h']); }; export const pruneDlq = async () => { diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index c672e3d..2e1fcd1 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -159,7 +159,7 @@ export class PipelineOrchestrator { ]); } - if (deployment.imageTag) { + if (deployment.imageTag && deployment.sourceType !== "image") { await tryRun("docker", ["rmi", "-f", deployment.imageTag]); } @@ -541,6 +541,8 @@ export class PipelineOrchestrator { }, ); + deployed = true; + await updateDeploymentStatus( deploymentId, "running", @@ -553,8 +555,6 @@ export class PipelineOrchestrator { }, ); - deployed = true; - if (projectName) { const dashSlug = projectName .toLowerCase() @@ -641,7 +641,7 @@ export class PipelineOrchestrator { ); await cleanupFailedDeployment( deploymentId, - imageTag, + deployment.sourceType === "image" ? undefined : imageTag, projectName, deployment.projectId, ).catch(e => diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index 2b8ecb3..c509642 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -39,7 +39,7 @@ export class DeploymentQueue { async remove(deploymentId: string) { await this.redis.lrem(QUEUE_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); await this.redis.zrem(RETRY_KEY, encodeJob({ id: deploymentId, attempt: 0 })); - const allAttempts = Array.from({ length: 10 }, (_, i) => + const allAttempts = Array.from({ length: config.queueRetryMax + 2 }, (_, i) => encodeJob({ id: deploymentId, attempt: i }), ); await this.redis.zrem(RETRY_KEY, ...allAttempts); @@ -62,7 +62,7 @@ export class DeploymentQueue { const workerRedis = createRedis(); try { while (!this.shuttingDown) { - await this.requeueDueJobs(); + await this.requeueDueJobs(workerRedis); const item = await workerRedis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); if (!item) continue; @@ -71,10 +71,10 @@ export class DeploymentQueue { try { const ok = await handler(job.id); - if (!ok) await this.retryOrDlq(job); + if (!ok) await this.retryOrDlq(job, workerRedis); } catch (err) { console.error(`[Queue] Worker ${workerId} handler error:`, err); - await this.retryOrDlq(job); + await this.retryOrDlq(job, workerRedis); } } } finally { @@ -82,22 +82,24 @@ export class DeploymentQueue { } } - private async retryOrDlq(job: JobPayload) { + private async retryOrDlq(job: JobPayload, redis?: Redis) { + const r = redis ?? this.redis; const attempt = job.attempt + 1; if (attempt > config.queueRetryMax) { - await this.redis.rpush(DLQ_KEY, encodeJob({ ...job, attempt })); + await r.rpush(DLQ_KEY, encodeJob({ ...job, attempt })); return; } const delayMs = config.queueRetryBaseMs * Math.pow(2, attempt - 1); const runAt = Date.now() + delayMs; - await this.redis.zadd(RETRY_KEY, String(runAt), encodeJob({ ...job, attempt })); + await r.zadd(RETRY_KEY, String(runAt), encodeJob({ ...job, attempt })); } - private async requeueDueJobs() { + private async requeueDueJobs(redis?: Redis) { + const r = redis ?? this.redis; const now = Date.now(); - const jobs = await this.redis.zrangebyscore(RETRY_KEY, 0, now, 'LIMIT', 0, 50); + const jobs = await r.zrangebyscore(RETRY_KEY, 0, now, 'LIMIT', 0, 50); if (!jobs.length) return; - await this.redis.zrem(RETRY_KEY, ...jobs); - await this.redis.rpush(QUEUE_KEY, ...jobs); + await r.zrem(RETRY_KEY, ...jobs); + await r.rpush(QUEUE_KEY, ...jobs); } } diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index f1b82e1..3592523 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -110,11 +110,12 @@ export const cleanupFailedDeployment = async ( imageTag?: string | null, projectName?: string, projectId?: string, + sourceType?: string | null, ) => { const containerName = getContainerName(deploymentId, projectName, projectId); await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); await tryRun(dockerBin, ['rm', '-f', containerName]); - if (imageTag) { + if (imageTag && sourceType !== "image") { await tryRun(dockerBin, ['rmi', '-f', imageTag]); } }; diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts index 0145a9f..e4cf27a 100644 --- a/apps/api/src/scaling/__tests__/engine.test.ts +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -103,7 +103,17 @@ mock.module(fileUrl('../../db/repo'), () => ({ updateDomainValidation: mock(() => Promise.resolve()), })); -// Mock docker-bin +mock.module(fileUrl('../../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + caddyRoutesDir: '/tmp/dequel-test-routes', + caddyBaseDomain: 'localhost', + appInternalPort: 3000, + dockerNetwork: 'dequel_net', + }, +})); + + mock.module(fileUrl('../../utils/docker-bin'), () => ({ dockerBin: '/usr/bin/docker' })); const { scalingEngine } = await import('../engine'); diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 8d3e9a8..dfc3ca2 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -1,5 +1,8 @@ FROM oven/bun:1 AS build +ARG DEQUEL_VERSION +ENV DEQUEL_VERSION=$DEQUEL_VERSION + WORKDIR /app COPY package.json ./ RUN bun install diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index 29c4c82..b705b3b 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -119,7 +119,7 @@ export function Layout({ children }: { children: React.ReactNode }) { setSidebarOpen={setSidebarOpen} /> -
+
{ setLoading(true); @@ -68,7 +69,8 @@ export function RepoPicker({ try { const [owner, repo] = selected.fullName.split("/"); const hooks = await getRepoHooks(owner, repo); - setWebhookActive(hooks.length > 0); + const expectedUrl = `${window.location.origin}/api/github/webhook`; + setWebhookActive(hooks.some((h) => h.url === expectedUrl)); } catch { setWebhookActive(false); } finally { @@ -81,6 +83,7 @@ export function RepoPicker({ const toggleWebhook = async () => { if (!selected) return; setWebhookLoading(true); + setWebhookError(""); try { const [owner, repo] = selected.fullName.split("/"); if (webhookActive) { @@ -90,8 +93,9 @@ export function RepoPicker({ await registerRepoHook(owner, repo); setWebhookActive(true); } - } catch { - setWebhookActive(false); + } catch (err) { + const message = err instanceof Error ? err.message : "Failed to update webhook"; + setWebhookError(message.includes("Not authenticated") ? "GitHub session expired. Reconnect GitHub, then try again." : message); } finally { setWebhookLoading(false); } @@ -167,11 +171,14 @@ export function RepoPicker({ Enable Auto-Deploy )} - - )} -
- ); - } + + )} + {webhookError && ( +

{webhookError}

+ )} +
+ ); + } return (
diff --git a/apps/web/src/components/layout/Sidebar.tsx b/apps/web/src/components/layout/Sidebar.tsx index fb04482..6a8b82b 100644 --- a/apps/web/src/components/layout/Sidebar.tsx +++ b/apps/web/src/components/layout/Sidebar.tsx @@ -48,8 +48,8 @@ export function Sidebar({
- + )} + + ) : ( diff --git a/apps/web/src/routes/Login.tsx b/apps/web/src/routes/Login.tsx index e88a030..094b918 100644 --- a/apps/web/src/routes/Login.tsx +++ b/apps/web/src/routes/Login.tsx @@ -44,7 +44,7 @@ export function Login() { dequel - v0.1 + v{__DEQUEL_VERSION__} diff --git a/apps/web/src/routes/ProjectDetail.tsx b/apps/web/src/routes/ProjectDetail.tsx index ee878f3..79625f9 100644 --- a/apps/web/src/routes/ProjectDetail.tsx +++ b/apps/web/src/routes/ProjectDetail.tsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useEffect, useRef } from "react"; import { useProject } from "../hooks/useProjects"; import { useDeployments } from "../hooks/useDeployments"; import { useNavigate, useLocation } from "@tanstack/react-router"; @@ -32,6 +32,17 @@ export function ProjectDetail({ const navigate = useNavigate(); const location = useLocation(); const activeTab = new URLSearchParams(location.search).get("tab") || "deployments"; + const tabsListRef = useRef(null); + + useEffect(() => { + if (!tabsListRef.current) return; + const activeTrigger = tabsListRef.current.querySelector( + `[data-value="${activeTab}"]`, + ) as HTMLElement | null; + if (activeTrigger) { + activeTrigger.scrollIntoView({ behavior: "smooth", block: "nearest", inline: "center" }); + } + }, [activeTab]); if (isLoading) return ( @@ -106,7 +117,7 @@ export function ProjectDetail({ navigate({ search: { tab: val } as any })} className="space-y-4"> - + Deployments diff --git a/apps/web/vite-env.d.ts b/apps/web/vite-env.d.ts new file mode 100644 index 0000000..38b89f1 --- /dev/null +++ b/apps/web/vite-env.d.ts @@ -0,0 +1 @@ +declare const __DEQUEL_VERSION__: string; diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 95d423e..67b345d 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -1,9 +1,16 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; import path from 'path'; +import fs from 'fs'; + +const versionPath = [path.resolve(__dirname, 'VERSION'), path.resolve(__dirname, '../../VERSION')].find(fs.existsSync); +const version = process.env.DEQUEL_VERSION || (versionPath ? fs.readFileSync(versionPath, 'utf-8').trim() : '0.0.0'); export default defineConfig({ plugins: [react()], + define: { + __DEQUEL_VERSION__: JSON.stringify(version), + }, resolve: { alias: { '@': path.resolve(__dirname, './src'), diff --git a/docker-compose.yml b/docker-compose.yml index 0f22018..c48322c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,6 +109,8 @@ services: restart: unless-stopped build: context: ./apps/web + args: + DEQUEL_VERSION: ${DEQUEL_VERSION:-} healthcheck: test: [ diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index 0dcac11..4cd18df 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -13,7 +13,9 @@ import /etc/caddy/routes/*.caddy encode zstd gzip handle /api/* { - reverse_proxy api:3001 + reverse_proxy api:3001 { + trusted_proxies private_ranges + } } handle /metrics* { @@ -40,7 +42,9 @@ import /etc/caddy/routes/*.caddy encode zstd gzip handle /api/* { - reverse_proxy api:3001 + reverse_proxy api:3001 { + trusted_proxies private_ranges + } } handle /metrics* { diff --git a/scripts/dequel b/scripts/dequel index fc3942d..6a8742e 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -6,6 +6,7 @@ SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" DEQUEL_HOME="${DEQUEL_HOME:-$SCRIPT_DIR}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" INSTALLED_VERSION="$(cat "$DEQUEL_HOME/VERSION" 2>/dev/null || echo "0.1.0")" +export DEQUEL_VERSION="$INSTALLED_VERSION" BOLD='\033[1m' DIM='\033[2m' @@ -17,6 +18,7 @@ NC='\033[0m' header() { printf "\n${BOLD}${AMBER}==>${NC}${BOLD} %s${NC}\n" "$*"; } info() { printf " ${DIM}%s${NC}\n" "$*"; } success() { printf " ${GREEN}✓${NC} %s\n" "$*"; } +warn() { printf " ${AMBER}⚠${NC} %s\n" "$*"; } fail() { printf " ${RED}✗${NC} %s\n" "$*"; exit 1; } cmd() { @@ -92,24 +94,27 @@ cmd_update() { curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/$f" done + mkdir -p "$tmp_dir/datasources" for f in loki.yml prometheus.yml; do - curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/$f" + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/datasources/$f" done for f in dashboards.yml deployed-apps.json; do curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/$f" done + mkdir -p "$DEQUEL_HOME/infra/caddy" \ + "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" - mkdir -p "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ - "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" for f in prometheus.yml loki-config.yml promtail-config.yml; do mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/$f" done for f in loki.yml prometheus.yml; do - mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + mv "$tmp_dir/datasources/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" done for f in dashboards.yml deployed-apps.json; do mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" @@ -127,7 +132,7 @@ cmd_update() { header "Recreating services" cmd up -d - success "Dequel updated to $tag" + success "Dequel updated to ${tag:-main}" } cmd_uninstall() { diff --git a/scripts/install.sh b/scripts/install.sh index d40e504..a9047b3 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -47,6 +47,7 @@ check_prerequisites() { if sudo usermod -aG docker "$USER" 2>/dev/null; then success "Added $USER to docker group" warn "Log out and back in (or run 'newgrp docker') for the change to take effect" + fail "Please log out and back in, then re-run the install script" else warn "Could not add user to docker group automatically" warn "Run this manually: sudo usermod -aG docker $USER && newgrp docker" From b2dbd88545d24c0bd695681d931b632ee669b2f0 Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 11:45:17 +0100 Subject: [PATCH 31/43] chore(docker): switch to remote images for api and web --- docker-compose.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c48322c..73c136b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,11 +28,11 @@ services: start_period: 5s api: - # image: ghcr.io/lftobs/dequel/api:latest + image: ghcr.io/lftobs/dequel/api:latest restart: unless-stopped - build: - context: . - dockerfile: ./apps/api/Dockerfile + # build: + # context: . + # dockerfile: ./apps/api/Dockerfile environment: PORT: 3001 DATABASE_PATH: /app/data/dequel.db @@ -105,12 +105,12 @@ services: start_period: 10s web: - # image: ghcr.io/lftobs/dequel/web:latest + image: ghcr.io/lftobs/dequel/web:latest restart: unless-stopped - build: - context: ./apps/web - args: - DEQUEL_VERSION: ${DEQUEL_VERSION:-} + # build: + # context: ./apps/web + # args: + # DEQUEL_VERSION: ${DEQUEL_VERSION:-} healthcheck: test: [ From 299a8b9581e4c3ce75ebb047abe615240285188a Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 14:38:09 +0100 Subject: [PATCH 32/43] chore(release): v0.2.0 --- .gitignore | 1 + .tegami/publish-lock.yaml | 17 +++ VERSION | 2 +- apps/api/package.json | 2 +- apps/docs/package.json | 2 +- apps/docs/src/content/changelogs/v0.2.0.md | 29 ++++ apps/web/package.json | 2 +- package.json | 8 +- scripts/tegami.mts | 155 +++++++++++++++++++++ 9 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 .tegami/publish-lock.yaml create mode 100644 apps/docs/src/content/changelogs/v0.2.0.md create mode 100644 scripts/tegami.mts diff --git a/.gitignore b/.gitignore index ce8189e..4994d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ docker-compose.yml bump.sh scripts/workflow/bump.sh __pycache__ +.tegami/changes-* diff --git a/.tegami/publish-lock.yaml b/.tegami/publish-lock.yaml new file mode 100644 index 0000000..51002f7 --- /dev/null +++ b/.tegami/publish-lock.yaml @@ -0,0 +1,17 @@ +core:packages: + - id: npm:dequel + updated: false + - changelogIds: [] + id: npm:dequel-docs + updated: true + - changelogIds: [] + id: npm:dequel-api + updated: true + - changelogIds: [] + id: npm:dequel-web + updated: true +npm:packages: + - id: npm:dequel + - id: npm:dequel-docs + - id: npm:dequel-api + - id: npm:dequel-web diff --git a/VERSION b/VERSION index 17e51c3..0ea3a94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.1 +0.2.0 diff --git a/apps/api/package.json b/apps/api/package.json index 66d9c11..b48ac4b 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -1,6 +1,6 @@ { "name": "dequel-api", - "version": "0.1.1", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/apps/docs/package.json b/apps/docs/package.json index 136545a..9091a24 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -1,7 +1,7 @@ { "name": "dequel-docs", "type": "module", - "version": "0.1.1", + "version": "0.2.0", "scripts": { "dev": "astro dev", "start": "astro dev", diff --git a/apps/docs/src/content/changelogs/v0.2.0.md b/apps/docs/src/content/changelogs/v0.2.0.md new file mode 100644 index 0000000..469b0b9 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.2.0.md @@ -0,0 +1,29 @@ +--- +version: 0.2.0 +date: "2026-07-17" +--- + +## What's Changed +### Features +- PAM-based SSH authentication for project deployments ([a6c69bc](https://github.com/Lftobs/dequel/commit/a6c69bc)) +- Deploy to a specific commit SHA, not just branch HEAD ([295178d](https://github.com/Lftobs/dequel/commit/295178d)) +- Drizzle ORM migrations and a new deploy UI ([ef3fd36](https://github.com/Lftobs/dequel/commit/ef3fd36)) +- Toggle to clear build cache on next deploy ([ffb2557](https://github.com/Lftobs/dequel/commit/ffb2557)) +- `dequel update` command for self-updating the platform ([875eabd](https://github.com/Lftobs/dequel/commit/875eabd)) +- Mobile-responsive navigation and layouts ([b9d0a96](https://github.com/Lftobs/dequel/commit/b9d0a96)) +- Automatic cleanup of old Docker images and build artifacts ([052b3ab](https://github.com/Lftobs/dequel/commit/052b3ab)) + +### Improvements +- GitHub OAuth now persists sessions across restarts and auto-detects the public URL from proxy headers ([15a8b9b](https://github.com/Lftobs/dequel/commit/15a8b9b)) +- PAM authentication moved to a standalone HTTP service for reliability ([caf25f2](https://github.com/Lftobs/dequel/commit/caf25f2)) +- Auth timeout and libc compatibility fixes ([afc28aa](https://github.com/Lftobs/dequel/commit/afc28aa)) +- Grafana dashboards and logging overhauled ([894a135](https://github.com/Lftobs/dequel/commit/894a135)) +- Install script improved for broader shell compatibility ([a0aa7c0](https://github.com/Lftobs/dequel/commit/a0aa7c0)) +- GitHub repo picker simplified to owner-affiliated repos only ([a3037ad](https://github.com/Lftobs/dequel/commit/a3037ad)) +- Docker Compose now pulls pre-built images from GHCR by default ([b2dbd88](https://github.com/Lftobs/dequel/commit/b2dbd88)) +- Docs navigation reorganized and UI polished ([f87ee8b](https://github.com/Lftobs/dequel/commit/f87ee8b)) + +### Bug Fixes +- Migration errors and UI log display issues resolved ([5c49ed9](https://github.com/Lftobs/dequel/commit/5c49ed9)) +- Project deletion now properly cascades to related records ([0a56270](https://github.com/Lftobs/dequel/commit/0a56270)) + diff --git a/apps/web/package.json b/apps/web/package.json index cea64d6..21ac8e9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "dequel-web", - "version": "0.1.1", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/package.json b/package.json index abe5b72..82c4e8a 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,15 @@ { "name": "dequel", - "version": "0.1.0", "private": true, + "workspaces": [ + "apps/*" + ], "scripts": { + "tegami": "bun scripts/tegami.mts", "sync-versions": "node -e \"const v = require('fs').readFileSync('VERSION','utf8').trim(); ['apps/api/package.json','apps/web/package.json','apps/docs/package.json'].forEach(f => { const p = require('./'+f); p.version = v; require('fs').writeFileSync(f, JSON.stringify(p, null, 2) + '\\n'); }); console.log('Versions synced to', v);\"", "postinstall": "bun sync-versions" + }, + "devDependencies": { + "tegami": "^1.2.5" } } diff --git a/scripts/tegami.mts b/scripts/tegami.mts new file mode 100644 index 0000000..2e659ba --- /dev/null +++ b/scripts/tegami.mts @@ -0,0 +1,155 @@ +import { execSync } from "node:child_process"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { tegami } from "tegami"; +import { runCli } from "tegami/cli"; +import { github } from "tegami/plugins/github"; +import type { Draft, TegamiPlugin } from "tegami"; + +const REPO = "Lftobs/dequel"; + +function getCommitMap(fromTag: string): Map { + const map = new Map(); + try { + const output = execSync( + `git log --oneline --format="%H %s" ${fromTag}..HEAD --no-merges`, + { encoding: "utf-8" } + ); + for (const line of output.trim().split("\n")) { + if (!line) continue; + const sha = line.slice(0, 40); + const msg = line.slice(41); + map.set(msg.toLowerCase(), sha); + } + } catch {} + return map; +} + +function formatChangelogBody(raw: string, commitMap: Map): string { + const lines = raw.replace(/^---[\s\S]*?---\n*/g, "").split("\n"); + const sectionTitles = new Map([ + ["New Features", "Features"], + ["Improvements", "Improvements"], + ["Bug Fixes", "Bug Fixes"], + ]); + + const result: string[] = []; + let sectionOpen = false; + + for (const line of lines) { + const trimmed = line.trimEnd(); + + const sectionMatch = trimmed.match(/^###\s+(.+)$/); + if (sectionMatch) { + if (sectionOpen) result.push(""); + const title = sectionTitles.get(sectionMatch[1]) ?? sectionMatch[1]; + result.push(`### ${title}`); + sectionOpen = true; + continue; + } + + const bulletMatch = trimmed.match(/^-\s+(.+)$/); + if (bulletMatch) { + const rawMsg = bulletMatch[1]; + const msg = rawMsg.replace(/^(feat|fix|refactor|chore|docs|style|perf|test|build|ci|revert)(\([^)]+\))?:\s*/, ""); + const display = msg.charAt(0).toUpperCase() + msg.slice(1); + const sha = commitMap.get(rawMsg) || commitMap.get(rawMsg.toLowerCase()); + if (sha) { + const shortSha = sha.slice(0, 7); + result.push(`- ${display} ([${shortSha}](https://github.com/${REPO}/commit/${sha}))`); + } else { + result.push(`- ${display}`); + } + continue; + } + + if (trimmed) result.push(trimmed); + } + + return result.join("\n") + "\n"; +} + +function docsChangelogPlugin(): TegamiPlugin { + return { + name: "docs-changelog", + enforce: "pre", + async applyDraft(this: any, draft: Draft) { + const logsDir = join(this.cwd, "apps/docs/src/content/changelogs"); + await mkdir(logsDir, { recursive: true }); + + const seen = new Set(); + const commitMap = getCommitMap("v0.1.1"); + + for (const [pkgId, packageDraft] of draft.getPackageDrafts()) { + if (!packageDraft.changelogs?.length) continue; + if (pkgId === "npm:dequel") continue; + const pkg = this.graph.get(pkgId); + if (!pkg) continue; + + for (const entry of packageDraft.changelogs) { + if (seen.has(entry.id)) continue; + seen.add(entry.id); + + const pkgJson = JSON.parse(await readFile(join(pkg.path, "package.json"), "utf8")); + const currentVersion = pkgJson.version; + if (!currentVersion || !packageDraft.type) continue; + const { inc, parse } = await import("semver"); + const parsed = parse(currentVersion); + if (!parsed) continue; + const newVersion = inc(parsed, packageDraft.type); + if (!newVersion) continue; + + const raw = entry.getRawContent(); + const body = formatChangelogBody(raw, commitMap); + const today = new Date().toISOString().slice(0, 10); + const dest = join(logsDir, `v${newVersion}.md`); + + const content = `--- +version: ${newVersion} +date: "${today}" +--- + +${body} +`; + + await writeFile(dest, content); + } + } + + for (const [pkgId, packageDraft] of draft.getPackageDrafts()) { + if (pkgId === "npm:dequel") continue; + packageDraft.changelogs = []; + } + }, + }; +} + +const paper = tegami({ + npm: { + client: "bun", + }, + plugins: [ + docsChangelogPlugin(), + github({ + repo: REPO, + release: false, + versionPr: { + base: "main", + }, + }), + ], + groups: { + dequel: { + syncBump: true, + syncGitTag: true, + }, + }, + packages: { + "npm:dequel": { publish: false }, + "npm:dequel-api": { group: "dequel" }, + "npm:dequel-web": { group: "dequel" }, + "npm:dequel-docs": { group: "dequel" }, + }, +}); + +await runCli(paper); From 309df3c560d33cec9654f2955f96f47ef3381d67 Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 14:41:00 +0100 Subject: [PATCH 33/43] docs: update v0.2.0 changelog details --- apps/docs/src/content/changelogs/v0.2.0.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/docs/src/content/changelogs/v0.2.0.md b/apps/docs/src/content/changelogs/v0.2.0.md index 469b0b9..8d97fbc 100644 --- a/apps/docs/src/content/changelogs/v0.2.0.md +++ b/apps/docs/src/content/changelogs/v0.2.0.md @@ -20,10 +20,8 @@ date: "2026-07-17" - Grafana dashboards and logging overhauled ([894a135](https://github.com/Lftobs/dequel/commit/894a135)) - Install script improved for broader shell compatibility ([a0aa7c0](https://github.com/Lftobs/dequel/commit/a0aa7c0)) - GitHub repo picker simplified to owner-affiliated repos only ([a3037ad](https://github.com/Lftobs/dequel/commit/a3037ad)) -- Docker Compose now pulls pre-built images from GHCR by default ([b2dbd88](https://github.com/Lftobs/dequel/commit/b2dbd88)) - Docs navigation reorganized and UI polished ([f87ee8b](https://github.com/Lftobs/dequel/commit/f87ee8b)) ### Bug Fixes - Migration errors and UI log display issues resolved ([5c49ed9](https://github.com/Lftobs/dequel/commit/5c49ed9)) - Project deletion now properly cascades to related records ([0a56270](https://github.com/Lftobs/dequel/commit/0a56270)) - From 66bd9f5409d97750c7264ccea741638e69bb8422 Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 15:41:19 +0100 Subject: [PATCH 34/43] chore(scripts): dynamic previous tag resolution in changelog generator --- scripts/tegami.mts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/tegami.mts b/scripts/tegami.mts index 2e659ba..a111449 100644 --- a/scripts/tegami.mts +++ b/scripts/tegami.mts @@ -8,8 +8,22 @@ import type { Draft, TegamiPlugin } from "tegami"; const REPO = "Lftobs/dequel"; +function getPreviousTag(): string { + try { + const output = execSync( + "git tag -l 'v*' --sort=-v:refname", + { encoding: "utf-8" } + ); + const tags = output.trim().split("\n").filter(Boolean); + return tags[0] ?? ""; + } catch { + return ""; + } +} + function getCommitMap(fromTag: string): Map { const map = new Map(); + if (!fromTag) return map; try { const output = execSync( `git log --oneline --format="%H %s" ${fromTag}..HEAD --no-merges`, @@ -78,7 +92,7 @@ function docsChangelogPlugin(): TegamiPlugin { await mkdir(logsDir, { recursive: true }); const seen = new Set(); - const commitMap = getCommitMap("v0.1.1"); + const commitMap = getCommitMap(getPreviousTag()); for (const [pkgId, packageDraft] of draft.getPackageDrafts()) { if (!packageDraft.changelogs?.length) continue; From 4cf6f284d2fc018feb5bb49dce7a1a9451bc85bb Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 15:41:38 +0100 Subject: [PATCH 35/43] chore(release): add tag release plugin to automation script --- .github/workflows/release.yml | 2 ++ .tegami/publish-lock.yaml | 17 ----------------- apps/api/Dockerfile | 5 ++--- apps/web/Dockerfile | 3 +++ scripts/tegami.mts | 26 +++++++++++++++++++++++--- 5 files changed, 30 insertions(+), 23 deletions(-) delete mode 100644 .tegami/publish-lock.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47be652..f1f3a6b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,6 +57,8 @@ jobs: with: context: apps/web push: true + build-args: | + DEQUEL_VERSION=${{ steps.version.outputs.VERSION }} tags: | ${{ env.WEB_IMAGE }}:${{ steps.version.outputs.VERSION }} ${{ env.WEB_IMAGE }}:latest diff --git a/.tegami/publish-lock.yaml b/.tegami/publish-lock.yaml deleted file mode 100644 index 51002f7..0000000 --- a/.tegami/publish-lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -core:packages: - - id: npm:dequel - updated: false - - changelogIds: [] - id: npm:dequel-docs - updated: true - - changelogIds: [] - id: npm:dequel-api - updated: true - - changelogIds: [] - id: npm:dequel-web - updated: true -npm:packages: - - id: npm:dequel - - id: npm:dequel-docs - - id: npm:dequel-api - - id: npm:dequel-web diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 867b354..a91de2e 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -32,11 +32,10 @@ RUN curl -sSL https://mise.jdx.dev/install.sh | sh \ WORKDIR /app -COPY apps/api/package.json ./ +COPY package.json ./ RUN bun install -COPY apps/api/src ./src -COPY scripts ./scripts +COPY src ./src RUN mkdir -p /app/data /app/workspace /caddy/routes diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 8d3e9a8..dfc3ca2 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -1,5 +1,8 @@ FROM oven/bun:1 AS build +ARG DEQUEL_VERSION +ENV DEQUEL_VERSION=$DEQUEL_VERSION + WORKDIR /app COPY package.json ./ RUN bun install diff --git a/scripts/tegami.mts b/scripts/tegami.mts index a111449..4d0e58e 100644 --- a/scripts/tegami.mts +++ b/scripts/tegami.mts @@ -138,12 +138,32 @@ ${body} }; } +function tagReleasePlugin(): TegamiPlugin { + return { + name: "tag-release", + async afterPublish(this: any) { + const pkg = this.graph.get("npm:dequel-api"); + if (!pkg) return; + const pkgJson = JSON.parse(await readFile(join(pkg.path, "package.json"), "utf8")); + const tag = `v${pkgJson.version}`; + try { + execSync(`git tag ${tag}`, { cwd: this.cwd }); + execSync(`git push origin ${tag}`, { cwd: this.cwd }); + console.log(`[Tegami] Pushed tag ${tag} — release workflow will trigger`); + } catch (e) { + console.warn(`[Tegami] Failed to push tag ${tag}:`, e); + } + }, + }; +} + const paper = tegami({ npm: { client: "bun", }, plugins: [ docsChangelogPlugin(), + tagReleasePlugin(), github({ repo: REPO, release: false, @@ -160,9 +180,9 @@ const paper = tegami({ }, packages: { "npm:dequel": { publish: false }, - "npm:dequel-api": { group: "dequel" }, - "npm:dequel-web": { group: "dequel" }, - "npm:dequel-docs": { group: "dequel" }, + "npm:dequel-api": { group: "dequel", publish: false }, + "npm:dequel-web": { group: "dequel", publish: false }, + "npm:dequel-docs": { group: "dequel", publish: false }, }, }); From 05de3faa52205842e2e4862225a974d32be94caa Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 15:41:56 +0100 Subject: [PATCH 36/43] chore(release): add release script and remove tag-release plugin Replace the automated Tegami release plugin with a manual npm script that tags and pushes the current version defined in the VERSION file. --- package.json | 1 + scripts/tegami.mts | 20 -------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/package.json b/package.json index 82c4e8a..3d67e7e 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ ], "scripts": { "tegami": "bun scripts/tegami.mts", + "release": "git tag v$(cat VERSION) && git push origin v$(cat VERSION)", "sync-versions": "node -e \"const v = require('fs').readFileSync('VERSION','utf8').trim(); ['apps/api/package.json','apps/web/package.json','apps/docs/package.json'].forEach(f => { const p = require('./'+f); p.version = v; require('fs').writeFileSync(f, JSON.stringify(p, null, 2) + '\\n'); }); console.log('Versions synced to', v);\"", "postinstall": "bun sync-versions" }, diff --git a/scripts/tegami.mts b/scripts/tegami.mts index 4d0e58e..8029ce1 100644 --- a/scripts/tegami.mts +++ b/scripts/tegami.mts @@ -138,32 +138,12 @@ ${body} }; } -function tagReleasePlugin(): TegamiPlugin { - return { - name: "tag-release", - async afterPublish(this: any) { - const pkg = this.graph.get("npm:dequel-api"); - if (!pkg) return; - const pkgJson = JSON.parse(await readFile(join(pkg.path, "package.json"), "utf8")); - const tag = `v${pkgJson.version}`; - try { - execSync(`git tag ${tag}`, { cwd: this.cwd }); - execSync(`git push origin ${tag}`, { cwd: this.cwd }); - console.log(`[Tegami] Pushed tag ${tag} — release workflow will trigger`); - } catch (e) { - console.warn(`[Tegami] Failed to push tag ${tag}:`, e); - } - }, - }; -} - const paper = tegami({ npm: { client: "bun", }, plugins: [ docsChangelogPlugin(), - tagReleasePlugin(), github({ repo: REPO, release: false, From 9c4203a62c18956636dac93174c4a4f942742a7b Mon Sep 17 00:00:00 2001 From: lftobs Date: Fri, 17 Jul 2026 17:08:04 +0100 Subject: [PATCH 37/43] fix(release): resolve dequel update collisions, auth script downloads, and use pre-built registry images --- docker-compose.yml | 15 ++++++++------- scripts/dequel | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0f22018..3e8ca22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,11 +28,12 @@ services: start_period: 5s api: - # image: ghcr.io/lftobs/dequel/api:latest + image: ghcr.io/lftobs/dequel/api:latest restart: unless-stopped - build: - context: . - dockerfile: ./apps/api/Dockerfile + # build: + # context: . + # + # dockerfile: ./apps/api/Dockerfile environment: PORT: 3001 DATABASE_PATH: /app/data/dequel.db @@ -105,10 +106,10 @@ services: start_period: 10s web: - # image: ghcr.io/lftobs/dequel/web:latest + image: ghcr.io/lftobs/dequel/web:latest restart: unless-stopped - build: - context: ./apps/web + # build: + # context: ./apps/web healthcheck: test: [ diff --git a/scripts/dequel b/scripts/dequel index fc3942d..2112d2a 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -88,31 +88,55 @@ cmd_update() { curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + mkdir -p "$tmp_dir/infra/monitoring/grafana/datasources" \ + "$tmp_dir/infra/monitoring/grafana/dashboards" \ + "$tmp_dir/scripts/auth" + for f in prometheus.yml loki-config.yml promtail-config.yml; do - curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/$f" + curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/infra/monitoring/$f" done for f in loki.yml prometheus.yml; do - curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/$f" + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/infra/monitoring/grafana/datasources/$f" done for f in dashboards.yml deployed-apps.json; do - curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/$f" + curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/infra/monitoring/grafana/dashboards/$f" + done + + for f in pam-server.py pam-verify.py; do + curl -fsSL "$base_url/scripts/auth/$f" -o "$tmp_dir/scripts/auth/$f" done + python3 -c " +import re +with open('$tmp_dir/docker-compose.yml', 'r') as f: + content = f.read() +content = re.sub(r'#\s*(image:\s*ghcr\.io/lftobs/dequel/api:latest)', r'\1', content) +content = re.sub(r'#\s*(image:\s*ghcr\.io/lftobs/dequel/web:latest)', r'\1', content) +content = re.sub(r'(build:\s*\n\s*context:\s*\.\s*\n\s*dockerfile:\s*\./apps/api/Dockerfile)', lambda m: '\n'.join(' # ' + line.strip() for line in m.group(1).split('\n')), content) +content = re.sub(r'(build:\s*\n\s*context:\s*\./apps/web)', lambda m: '\n'.join(' # ' + line.strip() for line in m.group(1).split('\n')), content) +with open('$tmp_dir/docker-compose.yml', 'w') as f: + f.write(content) +" 2>/dev/null || true + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" mkdir -p "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ - "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" \ + "$DEQUEL_HOME/scripts/auth" for f in prometheus.yml loki-config.yml promtail-config.yml; do - mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/$f" + mv "$tmp_dir/infra/monitoring/$f" "$DEQUEL_HOME/infra/monitoring/$f" done for f in loki.yml prometheus.yml; do - mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + mv "$tmp_dir/infra/monitoring/grafana/datasources/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" done for f in dashboards.yml deployed-apps.json; do - mv "$tmp_dir/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + mv "$tmp_dir/infra/monitoring/grafana/dashboards/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + done + for f in pam-server.py pam-verify.py; do + mv "$tmp_dir/scripts/auth/$f" "$DEQUEL_HOME/scripts/auth/$f" done mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" From 13c73d050ad4113ff23d48ef79d59a06b79b5012 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 18 Jul 2026 00:15:53 +0100 Subject: [PATCH 38/43] fix(release): configure GRAFANA_URL with subpath for api service --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 73c136b..e52127b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,7 @@ services: DOCKER_BIN: /usr/bin/docker RAILPACK_VERBOSE: "1" RAILPACK_BUILD_TIMEOUT_MS: "1200000" + GRAFANA_URL: http://grafana:3000/grafana volumes: - ./data:/app/data - ./workspace:/app/workspace From 453e627a9ad3e84620010f8b28a1437ff9465293 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 18 Jul 2026 00:29:24 +0100 Subject: [PATCH 39/43] fix(web): resolve Ko-fi popup viewport clipping and overflow in sidebar --- apps/web/src/index.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/web/src/index.css b/apps/web/src/index.css index a1cff50..70d0249 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -157,6 +157,10 @@ .floating-chat-kofi-popup-iframe, .floating-chat-kofi-popup-iframe-mobi { position: fixed !important; + left: 16px !important; + bottom: 80px !important; + top: auto !important; + right: auto !important; z-index: 999999 !important; } From bbaaf9c3bc4f56a70b64bd55d2b047c0caa7d881 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 18 Jul 2026 01:07:56 +0100 Subject: [PATCH 40/43] docs: add v0.1.1 and v0.2.0 to changelog --- apps/docs/src/content/docs/changelog.md | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/apps/docs/src/content/docs/changelog.md b/apps/docs/src/content/docs/changelog.md index da489ff..c615fbc 100644 --- a/apps/docs/src/content/docs/changelog.md +++ b/apps/docs/src/content/docs/changelog.md @@ -5,6 +5,54 @@ description: All notable changes to Dequel, tracked per release. slug: changelog --- +## 0.2.0 — 2026-07-17 + +### Features + +- PAM-based SSH authentication for project deployments +- Deploy to a specific commit SHA, not just branch HEAD +- Drizzle ORM migrations and a new deploy UI +- Toggle to clear build cache on next deploy +- `dequel update` command for self-updating the platform +- Mobile-responsive navigation and layouts +- Automatic cleanup of old Docker images and build artifacts + +### Improvements + +- GitHub OAuth now persists sessions across restarts and auto-detects the public URL from proxy headers +- PAM authentication moved to a standalone HTTP service for reliability +- Auth timeout and libc compatibility fixes +- Grafana dashboards and logging overhauled +- Install script improved for broader shell compatibility +- GitHub repo picker simplified to owner-affiliated repos only +- Docs navigation reorganized and UI polished + +### Bug Fixes + +- Migration errors and UI log display issues resolved +- Project deletion now properly cascades to related records + +## 0.1.1 — 2026-06-19 + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers + ## 0.1.0 — 2026-06-08 ### Added From 37f7f395dd5a782c6cf3621032340e590225a4a4 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 18 Jul 2026 01:33:51 +0100 Subject: [PATCH 41/43] docs: dynamic changelog from content collection --- apps/docs/src/content.config.ts | 10 ++++++- apps/docs/src/content/changelogs/v0.1.0.md | 34 ++++++++++++++++++++++ apps/docs/src/content/changelogs/v0.1.1.md | 23 +++++++++++++++ apps/docs/src/pages/docs/changelog.astro | 30 +++++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 apps/docs/src/content/changelogs/v0.1.0.md create mode 100644 apps/docs/src/content/changelogs/v0.1.1.md create mode 100644 apps/docs/src/pages/docs/changelog.astro diff --git a/apps/docs/src/content.config.ts b/apps/docs/src/content.config.ts index 98b1621..90dc20e 100644 --- a/apps/docs/src/content.config.ts +++ b/apps/docs/src/content.config.ts @@ -12,4 +12,12 @@ const docs = defineCollection({ }), }); -export const collections = { docs }; +const changelogs = defineCollection({ + loader: glob({ pattern: "*.md", base: "./src/content/changelogs" }), + schema: z.object({ + version: z.string(), + date: z.string(), + }), +}); + +export const collections = { docs, changelogs }; diff --git a/apps/docs/src/content/changelogs/v0.1.0.md b/apps/docs/src/content/changelogs/v0.1.0.md new file mode 100644 index 0000000..9a23869 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.1.0.md @@ -0,0 +1,34 @@ +--- +version: 0.1.0 +date: "2026-06-08" +--- + +### Added + +- Initial deployment platform with Git, ZIP, and Docker Compose source deploy +- Automatic build detection via Railpack +- Managed PostgreSQL and MySQL database provisioning +- Custom domain attachment with automatic SSL via Caddy / Let's Encrypt +- CPU-threshold based horizontal auto-scaling with configurable cooldown +- Per-project environment variable management with redeploy hooks +- Persistent Docker volume attachments +- Full observability stack: Prometheus, Loki, Grafana, cAdvisor +- CPU / memory threshold alerts via email or webhook +- API key management for programmatic access +- Job queue via Redis for async operations +- Deployment rollback support +- Boot-time reconciliation of container state +- Unified project versioning via root `VERSION` file and sync script +- One-command install script (`install.sh`) for quick setup +- Automated release pipeline via GitHub Actions +- Changelog page in documentation site +- Vercel deployment configuration for documentation site + +### Changed + +- `docker-compose.yml` now references images from `ghcr.io/dequel/*` with local build as fallback +- README updated with new install flow + +### Fixed + +- Railpack build timeout handling and log scrolling diff --git a/apps/docs/src/content/changelogs/v0.1.1.md b/apps/docs/src/content/changelogs/v0.1.1.md new file mode 100644 index 0000000..55bb037 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.1.1.md @@ -0,0 +1,23 @@ +--- +version: 0.1.1 +date: "2026-06-19" +--- + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers diff --git a/apps/docs/src/pages/docs/changelog.astro b/apps/docs/src/pages/docs/changelog.astro new file mode 100644 index 0000000..52cc31c --- /dev/null +++ b/apps/docs/src/pages/docs/changelog.astro @@ -0,0 +1,30 @@ +--- +import { getCollection, render } from 'astro:content'; +import Layout from '../../layouts/Layout.astro'; + +const changelogs = (await getCollection('changelogs')).sort((a, b) => + b.data.date.localeCompare(a.data.date) +); +--- + + +

Changelog

+

All notable changes to Dequel, tracked per release.

+ + {changelogs.map(async (entry) => { + const { Content } = await render(entry); + return ( +
+

{entry.data.version} — {entry.data.date}

+ +
+ ); + })} + + {!changelogs.length &&

No changelogs yet.

} + From 98fe1fc1874454705dd261d351675664e900c3d2 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 18 Jul 2026 01:33:51 +0100 Subject: [PATCH 42/43] docs: dynamic changelog from content collection --- apps/docs/src/content.config.ts | 10 ++++++- apps/docs/src/content/changelogs/v0.1.0.md | 34 ++++++++++++++++++++++ apps/docs/src/content/changelogs/v0.1.1.md | 23 +++++++++++++++ apps/docs/src/pages/docs/changelog.astro | 30 +++++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 apps/docs/src/content/changelogs/v0.1.0.md create mode 100644 apps/docs/src/content/changelogs/v0.1.1.md create mode 100644 apps/docs/src/pages/docs/changelog.astro diff --git a/apps/docs/src/content.config.ts b/apps/docs/src/content.config.ts index 98b1621..90dc20e 100644 --- a/apps/docs/src/content.config.ts +++ b/apps/docs/src/content.config.ts @@ -12,4 +12,12 @@ const docs = defineCollection({ }), }); -export const collections = { docs }; +const changelogs = defineCollection({ + loader: glob({ pattern: "*.md", base: "./src/content/changelogs" }), + schema: z.object({ + version: z.string(), + date: z.string(), + }), +}); + +export const collections = { docs, changelogs }; diff --git a/apps/docs/src/content/changelogs/v0.1.0.md b/apps/docs/src/content/changelogs/v0.1.0.md new file mode 100644 index 0000000..9a23869 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.1.0.md @@ -0,0 +1,34 @@ +--- +version: 0.1.0 +date: "2026-06-08" +--- + +### Added + +- Initial deployment platform with Git, ZIP, and Docker Compose source deploy +- Automatic build detection via Railpack +- Managed PostgreSQL and MySQL database provisioning +- Custom domain attachment with automatic SSL via Caddy / Let's Encrypt +- CPU-threshold based horizontal auto-scaling with configurable cooldown +- Per-project environment variable management with redeploy hooks +- Persistent Docker volume attachments +- Full observability stack: Prometheus, Loki, Grafana, cAdvisor +- CPU / memory threshold alerts via email or webhook +- API key management for programmatic access +- Job queue via Redis for async operations +- Deployment rollback support +- Boot-time reconciliation of container state +- Unified project versioning via root `VERSION` file and sync script +- One-command install script (`install.sh`) for quick setup +- Automated release pipeline via GitHub Actions +- Changelog page in documentation site +- Vercel deployment configuration for documentation site + +### Changed + +- `docker-compose.yml` now references images from `ghcr.io/dequel/*` with local build as fallback +- README updated with new install flow + +### Fixed + +- Railpack build timeout handling and log scrolling diff --git a/apps/docs/src/content/changelogs/v0.1.1.md b/apps/docs/src/content/changelogs/v0.1.1.md new file mode 100644 index 0000000..55bb037 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.1.1.md @@ -0,0 +1,23 @@ +--- +version: 0.1.1 +date: "2026-06-19" +--- + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers diff --git a/apps/docs/src/pages/docs/changelog.astro b/apps/docs/src/pages/docs/changelog.astro new file mode 100644 index 0000000..52cc31c --- /dev/null +++ b/apps/docs/src/pages/docs/changelog.astro @@ -0,0 +1,30 @@ +--- +import { getCollection, render } from 'astro:content'; +import Layout from '../../layouts/Layout.astro'; + +const changelogs = (await getCollection('changelogs')).sort((a, b) => + b.data.date.localeCompare(a.data.date) +); +--- + + +

Changelog

+

All notable changes to Dequel, tracked per release.

+ + {changelogs.map(async (entry) => { + const { Content } = await render(entry); + return ( +
+

{entry.data.version} — {entry.data.date}

+ +
+ ); + })} + + {!changelogs.length &&

No changelogs yet.

} + From 6c3acc7e77382bc0842f4ed9ba68d2415cbca9d5 Mon Sep 17 00:00:00 2001 From: lftobs Date: Sat, 18 Jul 2026 01:43:18 +0100 Subject: [PATCH 43/43] refactor(docs): migrate changelog to content collection Split single changelog file into individual versioned files within a new `changelogs` content collection. --- apps/docs/.astro/astro/content.d.ts | 31 +++++++++++++++++++++++++---- apps/docs/.astro/settings.json | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/apps/docs/.astro/astro/content.d.ts b/apps/docs/.astro/astro/content.d.ts index a3ac838..57c8503 100644 --- a/apps/docs/.astro/astro/content.d.ts +++ b/apps/docs/.astro/astro/content.d.ts @@ -140,10 +140,33 @@ declare module 'astro:content' { >; type ContentEntryMap = { - "docs": { -"changelog.md": { - id: "changelog.md"; - slug: "changelog"; + "changelogs": { +"v0.1.0.md": { + id: "v0.1.0.md"; + slug: "v010"; + body: string; + collection: "changelogs"; + data: any +} & { render(): Render[".md"] }; +"v0.1.1.md": { + id: "v0.1.1.md"; + slug: "v011"; + body: string; + collection: "changelogs"; + data: any +} & { render(): Render[".md"] }; +"v0.2.0.md": { + id: "v0.2.0.md"; + slug: "v020"; + body: string; + collection: "changelogs"; + data: any +} & { render(): Render[".md"] }; +}; +"docs": { +"auth.md": { + id: "auth.md"; + slug: "auth"; body: string; collection: "docs"; data: any diff --git a/apps/docs/.astro/settings.json b/apps/docs/.astro/settings.json index 3a7f780..be2c74f 100644 --- a/apps/docs/.astro/settings.json +++ b/apps/docs/.astro/settings.json @@ -1,5 +1,5 @@ { "_variables": { - "lastUpdateCheck": 1780853643721 + "lastUpdateCheck": 1784334507362 } } \ No newline at end of file