diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7ce11e4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.gitignore +.gitattributes +AGENTS.md +CHANGELOG.md +CONTRIBUTING.md +VERSION +*.md +node_modules +apps/web +apps/docs +data +workspace +infra +dist +.env +__pycache__ +*.log +docker-compose.yml +CLI.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..cacf5e1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,53 @@ +--- +name: Bug Report +about: Report a bug to help us improve Dequel +title: "[Bug]: " +labels: bug, triage +assignees: "" +--- + +## Description + +A clear and concise description of what the bug is. + +## Steps to Reproduce + +1. Go to '...' +2. Click on '...' +3. Scroll down to '...' +4. See error + +## Expected Behavior + +What did you expect to happen? + +## Actual Behavior + +What actually happened? + +## Screenshots / Logs + +If applicable, add screenshots or relevant logs. + +## Environment + +- **Dequel Version**: (check `VERSION` or `scripts/dequel status`) +- **OS**: +- **Docker version**: +- **Bun version** (if relevant): +- **Browser** (if relevant): + +## Affected Component + +- [ ] API (Backend) +- [ ] Web Dashboard (Frontend) +- [ ] Docs +- [ ] CLI / Install Script +- [ ] Docker / Deployment +- [ ] Build System (Railpack / BuildKit) +- [ ] Caddy / Ingress +- [ ] Monitoring (Prometheus / Grafana / Loki) + +## Additional Context + +Add any other context, workarounds, or related issues. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..cd40805 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,41 @@ +--- +name: Feature Request +about: Suggest an idea for Dequel +title: "[Feature]: " +labels: enhancement +assignees: "" +--- + +## Problem Statement + +Is your feature request related to a problem? Please describe what you're trying to solve. + +## Proposed Solution + +A clear and concise description of what you want to happen. + +## Alternative Solutions + +Any alternative solutions or features you've considered. + +## Affected Component + +- [ ] API (Backend) +- [ ] Web Dashboard (Frontend) +- [ ] Docs +- [ ] CLI / Install Script +- [ ] Build System (Railpack / BuildKit) +- [ ] Caddy / Ingress +- [ ] Monitoring (Prometheus / Grafana / Loki) + +## Mockups / Examples + +If applicable, add mockups, diagrams, or examples from other projects. + +## Additional Context + +Add any other context or screenshots. + +## Would you like to implement this? + +- [ ] Yes, I'd be happy to submit a PR diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..c5bbfa3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,43 @@ +## Description + +Please provide a summary of the changes and the motivation behind them. What problem does this PR solve? + +Fixes #(issue) + +## Type of Change + +- [ ] Bug fix (non-breaking change that fixes an issue) +- [ ] New feature (non-breaking change that adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Refactor (no functional changes) +- [ ] CI / Build / Tooling +- [ ] Other (please describe): + +## How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. + +- [ ] Existing tests pass (`bun test` in `apps/api/`) +- [ ] New tests added (if applicable) +- [ ] Manual testing performed (describe steps) + +## Checklist + +- [ ] My code follows the project's code style (no comments, named exports, functional components, etc.) +- [ ] I have read the [contributing guidelines](../CONTRIBUTING.md) +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have updated the documentation (if applicable) +- [ ] My changes generate no new warnings or lint errors +- [ ] I have run `bun test` in `apps/api/` and all tests pass +- [ ] I have synced the VERSION file if needed (`bun run sync-versions`) + +## Screenshots (if applicable) + +| Before | After | +|--------|-------| +| (insert here) | (insert here) | + +## Additional Context + +Add any other context about the PR here (e.g., migration notes, deployment considerations, rollback strategy). diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19c64a9..f1f3a6b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,6 +57,8 @@ jobs: with: context: apps/web push: true + build-args: | + DEQUEL_VERSION=${{ steps.version.outputs.VERSION }} tags: | ${{ env.WEB_IMAGE }}:${{ steps.version.outputs.VERSION }} ${{ env.WEB_IMAGE }}:latest @@ -71,18 +73,22 @@ jobs: cp docker-compose.yml "$TAR_DIR/" cp scripts/dequel "$TAR_DIR/dequel" cp infra/caddy/Caddyfile "$TAR_DIR/infra/caddy/" - cp infra/monitoring/prometheus.yml "$TAR_DIR/infra/monitoring/" - cp infra/monitoring/loki-config.yml "$TAR_DIR/infra/monitoring/" - cp infra/monitoring/promtail-config.yml "$TAR_DIR/infra/monitoring/" - cp infra/monitoring/grafana/datasources/loki.yml "$TAR_DIR/infra/monitoring/grafana/datasources/" - cp infra/monitoring/grafana/datasources/prometheus.yml "$TAR_DIR/infra/monitoring/grafana/datasources/" + cp -r infra/monitoring "$TAR_DIR/infra/" cd "$TAR_DIR" && tar -czf "../dequel-config-${VERSION}.tar.gz" . + - name: Extract changelog entry for this version + run: | + VERSION="${{ steps.version.outputs.VERSION }}" + awk '/^## \['"$VERSION"'\] -/{flag=1; next} /^## \[/{if(flag) exit} flag' CHANGELOG.md > release-notes.md + if [ ! -s release-notes.md ]; then + echo "No changelog entry for v$VERSION, will use auto-generated notes" + fi + - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: name: v${{ steps.version.outputs.VERSION }} - body_path: CHANGELOG.md + body_path: release-notes.md generate_release_notes: true files: | scripts/install.sh diff --git a/.github/workflows/vuln-scan.yml b/.github/workflows/vuln-scan.yml new file mode 100644 index 0000000..c83645d --- /dev/null +++ b/.github/workflows/vuln-scan.yml @@ -0,0 +1,50 @@ +name: Security Scans + +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: security-scans-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: read + +jobs: + forbidden-pattern-scan: + name: Forbidden Pattern Scan + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + + - name: Run forbidden pattern scan + run: bash scripts/workflow/forbidden-pattern-scan.sh "${{ github.workspace }}" + + lazarus-scanner: + name: Lazarus Scanner + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Download lazarus_scanner.py + run: | + curl -fsSL -o lazarus_scanner.py \ + https://raw.githubusercontent.com/hngprojects/lazarus-scanner/b1367e3a7a1463fbef90919867555a73f6acdf66/lazarus_scanner.py + echo "ddbb2181479f9a07d5859ceeac9160fd45084d28d95311ce49fc63fc1959e831 lazarus_scanner.py" | sha256sum --check || { echo "FAIL: SHA256 mismatch"; exit 1; } + + - name: Run Lazarus scanner + run: python3 lazarus_scanner.py diff --git a/.gitignore b/.gitignore index 3b0d81f..4994d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ infra/caddy/routes/ bugs apps/api/index docker-compose.yml +bump.sh +scripts/workflow/bump.sh +__pycache__ +.tegami/changes-* diff --git a/CHANGELOG.md b/CHANGELOG.md index f781212..c2c89aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.1] - 2026-06-19 + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup; Promtail scoped to `dequel_net` network; Grafana datasources use stable UIDs for reliable dashboard provisioning +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers, preventing Docker network ID changes from breaking deployments + +### Documentation + +- Installation guide, quickstart, and system configuration docs updated for `CADDY_BASE_DOMAIN` + ## [0.1.0] - 2026-06-08 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b4d8d89 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,107 @@ +# Contributing to Dequel + +Thank you for your interest in contributing! Here's how to get started. + +## Getting Started + +1. Fork and clone the repo +2. Install dependencies: `bun install` (from repo root — needed for tests, version syncing, and local tooling) +3. Read [`AGENTS.md`](./AGENTS.md) for the full architecture and conventions + +## Reporting Bugs + +Open a [Bug Report](https://github.com/Lftobs/dequel/issues/new?template=bug_report.md). Include: + +- Steps to reproduce +- Expected vs actual behavior +- Dequel version (`VERSION` file or `scripts/dequel status`) +- Environment details (OS, Docker version, browser if relevant) + +## Suggesting Features + +Open a [Feature Request](https://github.com/Lftobs/dequel/issues/new?template=feature_request.md). Describe: + +- The problem you're solving +- Your proposed solution +- Any alternatives considered +- Whether you'd like to implement it yourself + +## Development + +### Running Locally + +Run the full stack with Docker Compose: + +```bash +docker compose up -d --build +``` + +This starts all services (API, Web, Caddy, BuildKit, Redis, Prometheus, Loki, Grafana). The dashboard is at `https://localhost`, API at `https://localhost/api`, Grafana at `https://localhost/grafana` (admin/admin). + +### Code Conventions + +- **No comments** in source code unless absolutely necessary +- **Named exports** over default exports +- **Functional components + hooks** in React +- **Tailwind CSS** for styling (web and docs) +- Max ~500 lines per file — split into feature-grouped directories +- `set -euo pipefail` in all bash scripts + +### Database Migrations + +Run from `apps/api/`: + +```bash +cd apps/api +bunx drizzle-kit generate +bunx drizzle-kit push +``` + +### Testing + +Run from `apps/api/`: + +```bash +cd apps/api && bun test +``` + +Always run tests before committing API changes. + +### Versioning + +```bash +./bump.sh v0.2.0 +``` + +This updates `VERSION`, all `package.json` files, and optionally adds a changelog entry. + +## Pull Requests + +1. Create a PR from your fork using the [PR template](./.github/PULL_REQUEST_TEMPLATE.md) +2. Ensure all tests pass (`cd apps/api && bun test`) +3. Keep changes focused — one feature/fix per PR +4. Update documentation if your change affects user-facing behavior +5. If changing API behavior, update the docs site content + +### PR Checklist + +- [ ] Tests pass (`cd apps/api && bun test`) +- [ ] No new warnings or lint errors +- [ ] Documentation updated (if applicable) +- [ ] Version synced (`bun run sync-versions`) if `VERSION` changed +- [ ] Follows code conventions (no comments, named exports, etc.) + +## Release Process + +Maintainers cut releases by tagging: + +```bash +git tag vX.Y.Z +git push origin vX.Y.Z +``` + +CI builds Docker images to `ghcr.io/lftobs/dequel/{api,web}:X.Y.Z`, deploys docs to Vercel, and creates a GitHub Release. + +## Questions? + +Open a [Discussion](https://github.com/Lftobs/dequel/discussions) for questions and community support. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..925f81e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Lftobs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/VERSION b/VERSION index 6e8bf73..0ea3a94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0 +0.2.0 diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 4392ef5..a91de2e 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unzip \ tar \ gnupg \ + python3 \ && rm -rf /var/lib/apt/lists/* # Install Docker CLI + buildx plugin from Docker's official repo diff --git a/apps/api/drizzle.config.ts b/apps/api/drizzle.config.ts new file mode 100644 index 0000000..7a85d99 --- /dev/null +++ b/apps/api/drizzle.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "drizzle-kit"; + +export default defineConfig({ + dialect: "sqlite", + schema: "./src/db/schema.ts", + out: "./src/db/migrations", + dbCredentials: { + url: "./data/dequel.db", + }, +}); diff --git a/apps/api/package.json b/apps/api/package.json index 210da08..b48ac4b 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -1,6 +1,6 @@ { "name": "dequel-api", - "version": "0.1.0", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/apps/api/src/api/auth/index.ts b/apps/api/src/api/auth/index.ts new file mode 100644 index 0000000..b362544 --- /dev/null +++ b/apps/api/src/api/auth/index.ts @@ -0,0 +1,95 @@ +import { Elysia } from "elysia"; +import { signAccessToken, verifyAccessToken, generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } from "../../utils/auth"; +import { config } from "../../utils/config"; + +const PAM_AUTH_URL = "http://pam-auth:4567"; + +const callPam = async (username: string, password: string): Promise<{ ok: boolean; username?: string; error?: string }> => { + try { + const res = await fetch(`${PAM_AUTH_URL}/auth`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username, password }), + signal: AbortSignal.timeout(5000), + }); + const data = await res.json(); + return data; + } catch (err) { + return { ok: false, error: "Auth service unavailable" }; + } +}; + +const SESSION_COOKIE_OPTS = { + path: "/", + httpOnly: true, + sameSite: "lax" as const, + secure: config.caddyBaseDomain !== "localhost", + maxAge: 900, +}; + +const REFRESH_COOKIE_OPTS = { + path: "/", + httpOnly: true, + sameSite: "strict" as const, + secure: config.caddyBaseDomain !== "localhost", + maxAge: 7 * 24 * 60 * 60, +}; + +export const authRoutes = new Elysia() + .post("/auth/login", async ({ body, cookie: { dequel_session, dequel_refresh }, set }) => { + const { username, password } = body as { username?: string; password?: string }; + if (!username || !password) { + set.status = 400; + return { error: "Username and password required" }; + } + const result = await callPam(username, password); + if (!result.ok) { + set.status = 401; + return { error: result.error || "Authentication failed" }; + } + const accessToken = await signAccessToken(username); + const refreshToken = generateRefreshToken(); + await storeRefreshToken(username, refreshToken); + dequel_session.value = accessToken; + dequel_session.set(SESSION_COOKIE_OPTS); + dequel_refresh.value = refreshToken; + dequel_refresh.set(REFRESH_COOKIE_OPTS); + return { ok: true, username }; + }) + .post("/auth/logout", async ({ cookie: { dequel_session, dequel_refresh } }) => { + const rt = dequel_refresh.value; + if (rt) { + try { await blacklistRefreshToken(rt); } catch {} + } + dequel_session.remove(); + dequel_refresh.remove(); + return { ok: true }; + }) + .post("/auth/refresh", async ({ cookie: { dequel_session, dequel_refresh }, set }) => { + const rt = dequel_refresh.value; + if (!rt) { + set.status = 401; + return { error: "No refresh token" }; + } + const username = await validateRefreshToken(rt); + if (!username) { + set.status = 401; + return { error: "Invalid or expired refresh token" }; + } + await blacklistRefreshToken(rt); + const accessToken = await signAccessToken(username); + const newRefreshToken = generateRefreshToken(); + await storeRefreshToken(username, newRefreshToken); + dequel_session.value = accessToken; + dequel_session.set(SESSION_COOKIE_OPTS); + dequel_refresh.value = newRefreshToken; + dequel_refresh.set(REFRESH_COOKIE_OPTS); + return { ok: true, username }; + }) + .get("/auth/me", async ({ cookie: { dequel_session } }) => { + const token = dequel_session.value; + if (!token) return { authenticated: false }; + const payload = await verifyAccessToken(token); + if (!payload) return { authenticated: false }; + return { authenticated: true, username: payload.sub }; + }); diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index a8c4147..99b1cf0 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -62,6 +62,10 @@ export const deploymentsRoutes = new Elysia() const environment = String(form.get("environment") ?? "").trim() || undefined; + const commitSha = + String(form.get("commitSha") ?? "").trim() || + undefined; + const clearCache = form.get("clearCache") === "true"; if ( sourceType !== "git" && sourceType !== "upload" && @@ -86,6 +90,8 @@ export const deploymentsRoutes = new Elysia() sourceRef: gitUrl, branch, environment, + commitSha, + clearCache, }); orchestrator.enqueue(deployment.id); return deployment; @@ -118,6 +124,7 @@ export const deploymentsRoutes = new Elysia() sourceRef: uploadPath, branch, environment, + clearCache, }); orchestrator.enqueue(deployment.id); return deployment; diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b2b6e83..b0cfef5 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -1,18 +1,53 @@ import { Elysia } from "elysia"; +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; import { getGithubIntegration, setGithubIntegration, createDeployment, listProjects } from "../../db/repo"; import { orchestrator } from "../../orchestrator"; import { config } from "../../utils/config"; -const SESSIONS = new Map(); -const SESSION_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours +const SESSIONS_FILE = join(process.env.DATA_DIR ?? "./data", ".github-sessions.json"); -const getSession = (cookie: string | null): string | null => { +let SESSIONS = new Map(); + +const loadSessions = () => { + try { + const raw = readFileSync(SESSIONS_FILE, "utf-8"); + const entries: [string, { token: string }][] = JSON.parse(raw); + SESSIONS = new Map(entries); + } catch {} +}; + +const saveSessions = () => { + try { + const dir = SESSIONS_FILE.substring(0, SESSIONS_FILE.lastIndexOf("/")); + mkdirSync(dir, { recursive: true }); + writeFileSync(SESSIONS_FILE, JSON.stringify([...SESSIONS]), "utf-8"); + } catch {} +}; + +loadSessions(); + +const validateToken = async (token: string): Promise => { + try { + const res = await fetch("https://api.github.com/user", { + headers: { Authorization: `Bearer ${token}`, "User-Agent": "dequel" }, + }); + return res.ok; + } catch { + return true; + } +}; + +const getSession = async (cookie: string | null): Promise => { if (!cookie) return null; const match = cookie.match(/github_session=([^;]+)/); if (!match) return null; const session = SESSIONS.get(match[1]); - if (!session || session.expiresAt < Date.now()) { - if (session) SESSIONS.delete(match[1]); + if (!session) return null; + const valid = await validateToken(session.token); + if (!valid) { + SESSIONS.delete(match[1]); + saveSessions(); return null; } return session.token; @@ -20,10 +55,31 @@ const getSession = (cookie: string | null): string | null => { const createSession = (token: string): string => { const id = crypto.randomUUID(); - SESSIONS.set(id, { token, expiresAt: Date.now() + SESSION_TTL_MS }); + SESSIONS.set(id, { token }); + saveSessions(); return id; }; +const publicUrl = (request: Request): URL => { + const proto = request.headers.get("x-forwarded-proto") || "http"; + const host = request.headers.get("x-forwarded-host") || request.headers.get("host") || "localhost"; + return new URL(`${proto}://${host}${new URL(request.url).pathname}`); +}; + +const describeGithubError = (err: unknown): { status: number; message: string } => { + const raw = err instanceof Error ? err.message : String(err); + const match = raw.match(/^GitHub API error (\d+): (.*)$/s); + if (!match) return { status: 502, message: raw }; + const status = Number(match[1]); + if (status === 403 && /not accessible by integration/i.test(match[2])) { + return { + status: 502, + message: "GitHub App is missing the 'Webhooks' repository permission. In your GitHub App settings, go to Permissions & events and set Webhooks to Read and write, approve the new permission, then try again.", + }; + } + return { status: 502, message: `GitHub API error ${status}: ${match[2]}` }; +}; + const fetchGitHub = async (path: string, token: string) => { const res = await fetch(`https://api.github.com${path}`, { headers: { @@ -84,7 +140,7 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 400; return { error: "GitHub integration not configured" }; } - const url = new URL(request.url); + const url = publicUrl(request); const redirectUri = `${url.protocol}//${url.host}/api/github/callback`; const state = crypto.randomUUID(); const params = new URLSearchParams({ @@ -107,7 +163,8 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) set.status = 400; return { error: "GitHub integration not configured" }; } - const origin = new URL(request.url).origin; + const url = publicUrl(request); + const origin = url.origin; const redirectUri = `${origin}/api/github/callback`; const res = await fetch("https://github.com/login/oauth/access_token", { @@ -132,12 +189,12 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) } const sessionId = createSession(data.access_token); set.status = 302; - set.headers["Set-Cookie"] = `github_session=${sessionId}; Path=/; HttpOnly; SameSite=Lax; Max-Age=${SESSION_TTL_MS / 1000}`; + set.headers["Set-Cookie"] = `github_session=${sessionId}; Path=/; HttpOnly; SameSite=Lax; Max-Age=315360000`; set.headers["Location"] = `${origin}/?github=connected`; }) .get("/user", async ({ request, set }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; @@ -146,29 +203,20 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) }) .get("/repos", async ({ request, set }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const repos = await fetchGitHub("/user/repos?per_page=100&sort=updated&type=all", token); - const user = await fetchGitHub("/user", token) as { login: string }; - const orgs = await fetchGitHub("/user/orgs", token) as { login: string }[]; - const allRepos = Array.isArray(repos) ? repos : []; - const orgRepos: any[] = []; - for (const org of Array.isArray(orgs) ? orgs : []) { - try { - const orgR = await fetchGitHub(`/orgs/${org.login}/repos?per_page=100&sort=updated&type=all`, token); - if (Array.isArray(orgR)) orgRepos.push(...orgR); - } catch {} + const allRepos: any[] = []; + let page = 1; + while (page <= 10) { + const repos = await fetchGitHub(`/user/repos?per_page=100&page=${page}&sort=updated&affiliation=owner`, token); + if (!Array.isArray(repos) || repos.length === 0) break; + allRepos.push(...repos); + page++; } - const seen = new Set(); - const merged = [...allRepos, ...orgRepos].filter((r) => { - if (seen.has(r.id)) return false; - seen.add(r.id); - return true; - }); - return merged.map((r: any) => ({ + return allRepos.map((r: any) => ({ id: r.id, name: r.name, fullName: r.full_name, @@ -183,82 +231,119 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) }) .get("/repos/:owner/:repo/hooks", async ({ request, set, params }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); - return Array.isArray(hooks) ? hooks.map((h: any) => ({ id: h.id, url: h.config.url, active: h.active, events: h.events })) : []; + try { + const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + return Array.isArray(hooks) ? hooks.map((h: any) => ({ id: h.id, url: h.config.url, active: h.active, events: h.events })) : []; + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; + } }) .post("/repos/:owner/:repo/hook", async ({ request, set, params }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const baseUrl = config.caddyBaseDomain === 'localhost' ? 'http://localhost' : `https://${config.caddyBaseDomain}`; - const webhookUrl = `${baseUrl}/api/github/webhook`; + const webhookUrl = `${publicUrl(request).origin}/api/github/webhook`; const integration = await getGithubIntegration(); const secret = integration?.webhookSecret || config.githubWebhookSecret; - const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); - const existing = Array.isArray(hooks) ? hooks.find((h: any) => h.config?.url === webhookUrl) : null; - - if (existing) { - return { id: existing.id, created: false, url: webhookUrl }; + try { + const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + const existing = Array.isArray(hooks) ? hooks.find((h: any) => h.config?.url === webhookUrl) : null; + + if (existing) { + return { id: existing.id, created: false, url: webhookUrl }; + } + + // Remove stale Dequel webhooks left over from a previous tunnel/domain + const stale = Array.isArray(hooks) + ? hooks.filter((h: any) => typeof h.config?.url === "string" && h.config.url.endsWith("/api/github/webhook") && h.config.url !== webhookUrl) + : []; + for (const h of stale) { + await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks/${h.id}`, token, "DELETE").catch(() => {}); + } + + const hook = await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks`, token, "POST", { + name: "web", + active: true, + events: ["push"], + config: { + url: webhookUrl, + content_type: "json", + secret, + insecure_ssl: "0", + }, + }); + return { id: hook.id, created: true, url: webhookUrl }; + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; } - - const hook = await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks`, token, "POST", { - name: "web", - active: true, - events: ["push"], - config: { - url: webhookUrl, - content_type: "json", - secret, - insecure_ssl: "0", - }, - }); - return { id: hook.id, created: true, url: webhookUrl }; }) .delete("/repos/:owner/:repo/hook", async ({ request, set, params }) => { - const token = getSession(request.headers.get("cookie")); + const token = await getSession(request.headers.get("cookie")); if (!token) { set.status = 401; return { error: "Not authenticated" }; } - const baseUrl = config.caddyBaseDomain === 'localhost' ? 'http://localhost' : `https://${config.caddyBaseDomain}`; - const webhookUrl = `${baseUrl}/api/github/webhook`; - const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + const webhookUrl = `${publicUrl(request).origin}/api/github/webhook`; + let hooks: any; + try { + hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; + } const existing = Array.isArray(hooks) ? hooks.find((h: any) => h.config?.url === webhookUrl) : null; if (!existing) { return { ok: true, removed: false }; } - await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks/${existing.id}`, token, "DELETE"); - return { ok: true, removed: true }; + try { + await fetchGitHubWithBody(`/repos/${params.owner}/${params.repo}/hooks/${existing.id}`, token, "DELETE"); + return { ok: true, removed: true }; + } catch (err) { + const { status, message } = describeGithubError(err); + set.status = status; + return { error: message }; + } }) .post("/disconnect", async ({ set, request }) => { const cookie = request.headers.get("cookie"); const match = cookie?.match(/github_session=([^;]+)/); - if (match) SESSIONS.delete(match[1]); + if (match) { + SESSIONS.delete(match[1]); + saveSessions(); + } set.headers["Set-Cookie"] = "github_session=; Path=/; Max-Age=0"; return { ok: true }; }) .post("/webhook", async ({ request, set }) => { const integration = await getGithubIntegration(); + const event = request.headers.get("x-github-event") || ""; + const delivery = request.headers.get("x-github-delivery") || ""; + console.log(`[GitWebhook] received event=${event} delivery=${delivery}`); + if (!integration?.webhookSecret) { + console.log("[GitWebhook] rejected: no webhook secret configured on this Dequel instance"); set.status = 400; return { error: "GitHub webhook not configured" }; } const signature = request.headers.get("x-hub-signature-256") || ""; - const event = request.headers.get("x-github-event") || ""; - const delivery = request.headers.get("x-github-delivery") || ""; const rawBody = await request.text(); const encoder = new TextEncoder(); @@ -273,15 +358,18 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) const expectedSig = "sha256=" + Array.from(new Uint8Array(expectedSigRaw)).map(b => b.toString(16).padStart(2, "0")).join(""); if (signature.length !== expectedSig.length) { + console.log(`[GitWebhook] rejected delivery=${delivery}: signature length mismatch (check webhook secret matches Dequel settings)`); set.status = 401; return { error: "Invalid signature" }; } if (!crypto.timingSafeEqual(Buffer.from(expectedSig), Buffer.from(signature))) { + console.log(`[GitWebhook] rejected delivery=${delivery}: signature mismatch (check webhook secret matches Dequel settings)`); set.status = 401; return { error: "Invalid signature" }; } if (event !== "push") { + console.log(`[GitWebhook] ignored delivery=${delivery}: unsupported event ${event}`); return { ok: true, ignored: `unsupported event: ${event}` }; } @@ -302,37 +390,40 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) const branch = payload?.ref?.replace("refs/heads/", "") || "main"; const commitSha = payload?.after || ""; + if (!commitSha || commitSha === "0000000000000000000000000000000000000000") { + console.log(`[GitWebhook] ignored delivery=${delivery}: deletion event, no commit to deploy`); + return { ok: true, ignored: "deletion event, no commit to deploy" }; + } + const projects = await listProjects(); - const project = projects.find(p => - p.repoUrl && ( - p.repoUrl === repoUrl || - p.repoUrl.replace(/\.git$/, "") === repoUrl.replace(/\.git$/, "") - ) - ); + const normalize = (u: string) => u.replace(/\.git$/, "").replace(/\/+$/, "").toLowerCase(); + const matching = projects.filter(p => p.repoUrl && normalize(p.repoUrl) === normalize(repoUrl)); - if (!project) { + if (matching.length === 0) { + console.log(`[GitWebhook] ignored delivery=${delivery}: no project found for repo ${repoUrl}. Known project repos: ${projects.map(p => p.repoUrl).filter(Boolean).join(", ") || "(none)"}`); return { ok: true, ignored: `no project found for repo: ${repoUrl}` }; } - if (project.repoBranch && project.repoBranch !== branch) { - return { ok: true, ignored: `branch "${branch}" does not match project branch "${project.repoBranch}"` }; - } + const targets = matching.filter(p => !p.repoBranch || p.repoBranch === branch); - if (!commitSha || commitSha === "0000000000000000000000000000000000000000") { - return { ok: true, ignored: "deletion event, no commit to deploy" }; + if (targets.length === 0) { + console.log(`[GitWebhook] ignored delivery=${delivery}: branch "${branch}" does not match any project watching ${repoUrl} (branches: ${matching.map(p => p.repoBranch ?? "any").join(", ")})`); + return { ok: true, ignored: `branch "${branch}" does not match any watching project's branch` }; } - const dep = await createDeployment({ - projectId: project.id, - sourceType: "git", - sourceRef: repoUrl, - branch, - commitSha, - }); - - orchestrator.enqueue(dep.id); - - console.log(`[GitWebhook] Auto-deploy triggered for ${project.name} (${branch}) — commit ${commitSha.slice(0, 7)}`); + const deploymentIds: string[] = []; + for (const project of targets) { + const dep = await createDeployment({ + projectId: project.id, + sourceType: "git", + sourceRef: repoUrl, + branch, + commitSha, + }); + orchestrator.enqueue(dep.id); + deploymentIds.push(dep.id); + console.log(`[GitWebhook] Auto-deploy triggered for ${project.name} (${branch}) — commit ${commitSha.slice(0, 7)}`); + } - return { ok: true, deploymentId: dep.id }; + return { ok: true, deploymentIds }; }); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index a47c08a..d8c56b1 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,6 +1,7 @@ import { Elysia } from "elysia"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; +import { authRoutes } from "./auth"; import { databasesRoutes } from "./databases"; import { deploymentsRoutes } from "./deployments"; import { domainsRoutes } from "./domains"; @@ -15,27 +16,42 @@ import { serversRoutes } from "./servers"; import { volumesRoutes } from "./volumes"; import { settingsRoutes } from "./settings"; +const BYPASS_PATHS = new Set(["/api/auth/login", "/api/auth/logout", "/api/auth/refresh", "/api/auth/me", "/api/health", "/api/github/callback", "/api/github/webhook"]); + const authMiddleware = (app: Elysia) => - app.onBeforeHandle(async ({ request, set }) => { - const authHeader = request.headers.get( - "authorization", - ); - if (!authHeader?.startsWith("Bearer ")) - return; - const token = authHeader.slice(7); - if (!token) return; - const { validateApiKey } = - await import("../db/repo"); - const key = await validateApiKey(token); - if (!key) { + app.onBeforeHandle(async ({ request, set, path }) => { + if (BYPASS_PATHS.has(path)) return; + + const cookie = request.headers.get("cookie") || ""; + const match = cookie.match(/(?:^|;\s*)dequel_session=([^;]+)/); + if (match) { + const { verifyAccessToken } = await import("../utils/auth"); + const payload = await verifyAccessToken(match[1]); + if (payload) return; set.status = 401; - return { error: "Invalid API key" }; + return { error: "Invalid session" }; } + + const authHeader = request.headers.get("authorization"); + if (authHeader?.startsWith("Bearer ")) { + const token = authHeader.slice(7); + if (token) { + const { validateApiKey } = await import("../db/repo"); + const key = await validateApiKey(token); + if (key) return; + set.status = 401; + return { error: "Invalid API key" }; + } + } + + set.status = 401; + return { error: "Authentication required" }; }); export const apiRoutes = new Elysia({ prefix: "/api", }) + .use(authRoutes) .use(authMiddleware) .use(healthRoutes) .use(projectsRoutes) diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index 11473a7..11e5bcf 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -6,7 +6,8 @@ import { listProjects, getProjectById, updateProject, - deleteProject, + deleteProjectCascade, + listDomains, } from "../../db/repo"; import { tryRun, reloadCaddy } from "../../orchestrator/runtime"; import { removeFromCaddyRoute } from "../../utils/domain-verifier"; @@ -33,6 +34,16 @@ export const projectsRoutes = new Elysia() set.status = 400; return { error: "name is required" }; } + if (body?.repoUrl) { + const projects = await listProjects(); + const normalize = (u: string) => u.replace(/\.git$/, "").replace(/\/+$/, "").toLowerCase(); + const incoming = normalize(body.repoUrl); + const duplicate = projects.find((p) => p.repoUrl && normalize(p.repoUrl) === incoming); + if (duplicate) { + set.status = 409; + return { error: `A project with this repository URL already exists: "${duplicate.name}"` }; + } + } const project = await createProject({ name: body.name, description: body.description, @@ -51,6 +62,16 @@ export const projectsRoutes = new Elysia() .patch( "/projects/:id", async ({ params: { id }, body, set }: any) => { + if (body?.repoUrl) { + const projects = await listProjects(); + const normalize = (u: string) => u.replace(/\.git$/, "").replace(/\/+$/, "").toLowerCase(); + const incoming = normalize(body.repoUrl); + const duplicate = projects.find((p) => p.id !== id && p.repoUrl && normalize(p.repoUrl) === incoming); + if (duplicate) { + set.status = 409; + return { error: `A project with this repository URL already exists: "${duplicate.name}"` }; + } + } const project = await updateProject(id, { name: body?.name, description: body?.description, @@ -78,7 +99,6 @@ export const projectsRoutes = new Elysia() return { error: "Project not found" }; } - // Docker containers cleanup for (const name of info.deploymentContainerNames) { await tryRun(dockerBin, ['stop', '-t', '5', name]); await tryRun(dockerBin, ['rm', '-f', name]); @@ -88,22 +108,18 @@ export const projectsRoutes = new Elysia() await tryRun(dockerBin, ['rm', '-f', name]); } - // Docker volumes cleanup for (const name of [...info.databaseVolumeNames, ...info.volumeDockerNames]) { await tryRun(dockerBin, ['volume', 'rm', '-f', name]); } - // Docker images cleanup for (const tag of info.deploymentImageTags) { if (tag) await tryRun(dockerBin, ['rmi', '-f', tag]); } - // Remove domains from Caddy route file for (const { domain, projectName } of info.domains) { await removeFromCaddyRoute(domain, id, projectName); } - // Delete the Caddy route file for this project's slug const caddyFilePath = join(config.caddyRoutesDir, `${info.slug}.caddy`); await unlink(caddyFilePath).catch(() => {}); await reloadCaddy().catch(() => {}); @@ -205,14 +221,14 @@ export const projectsRoutes = new Elysia() } const regexEscaped = domains.map(d => d.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\\\$&')).join('|'); - + // Loki metric query for request rate const query = `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [5m]))`; - + const end = Math.floor(Date.now() / 1000); const start = end - (6 * 60 * 60); // 6 hours ago const step = "60s"; // 1 min resolution - + try { const response = await fetch( `http://loki:3100/loki/api/v1/query_range?query=${encodeURIComponent(query)}&start=${start}&end=${end}&step=${step}` @@ -303,8 +319,8 @@ export const projectsRoutes = new Elysia() ws.onmessage = (event) => { if (closed) return; try { - const dataStr = typeof event.data === "string" - ? event.data + const dataStr = typeof event.data === "string" + ? event.data : new TextDecoder().decode(event.data as any); const data = JSON.parse(dataStr) as any; if (data.streams) { diff --git a/apps/api/src/api/server-info/index.ts b/apps/api/src/api/server-info/index.ts index 9a163a4..c27604c 100644 --- a/apps/api/src/api/server-info/index.ts +++ b/apps/api/src/api/server-info/index.ts @@ -3,7 +3,7 @@ import { Elysia } from "elysia"; export const serverInfoRoutes = new Elysia().get( "/server/ip", async () => { - const { resolveServerIp } = await import("../../utils/dns"); - return { ip: await resolveServerIp() }; + const { checkBaseDomainStatus } = await import("../../utils/dns"); + return await checkBaseDomainStatus(); }, ); diff --git a/apps/api/src/databases/manager.ts b/apps/api/src/databases/manager.ts index 6821bda..8c02b04 100644 --- a/apps/api/src/databases/manager.ts +++ b/apps/api/src/databases/manager.ts @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import type { Database } from '../types'; import { updateDatabaseStatus } from '../db/repo'; @@ -52,6 +53,7 @@ export const provisionDatabase = async (dbRecord: Database): Promise => { '--name', containerName, '--network', config.dockerNetwork, '--network-alias', containerName, + '-l', DEQUEL_MANAGED_LABEL, ...(dbRecord.cpuLimit ? ['--cpus', String(dbRecord.cpuLimit)] : []), ...(dbRecord.memoryLimitMb ? ['--memory', `${Math.round(dbRecord.memoryLimitMb)}m`] : []), '-v', `${volumeName}:/var/lib/${dbRecord.type === 'mysql' ? 'mysql' : 'postgresql/data'}`, diff --git a/apps/api/src/db/__tests__/auth.test.ts b/apps/api/src/db/__tests__/auth.test.ts new file mode 100644 index 0000000..22dc2ef --- /dev/null +++ b/apps/api/src/db/__tests__/auth.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, mock, beforeAll, beforeEach, afterAll } from 'bun:test'; +import { Database } from 'bun:sqlite'; + +const TEST_SECRET = 'test-jwt-secret-for-testing-purposes-only'; + +let db: Database; + +const fileUrl = (path: string) => new URL(path, import.meta.url).toString(); +mock.module(fileUrl('../client.ts'), () => ({ + getDb: () => db, +})); + +beforeAll(async () => { + db = new Database(':memory:'); + db.run(` + CREATE TABLE refresh_tokens ( + id text PRIMARY KEY NOT NULL, + username text NOT NULL, + token_hash text NOT NULL UNIQUE, + expires_at text NOT NULL, + created_at text NOT NULL, + blacklisted_at text + ) + `); + const { initAuth } = await import('../../utils/auth'); + initAuth(TEST_SECRET); +}); + +beforeEach(() => { + db.run('DELETE FROM refresh_tokens'); +}); + +afterAll(() => { + db.close(); +}); + +describe('storeRefreshToken / validateRefreshToken', () => { + it('stores and validates a refresh token', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + const username = await validateRefreshToken(token); + expect(username).toBe('testuser'); + }); + + it('returns null for unknown token', async () => { + const { validateRefreshToken } = await import('../../utils/auth'); + const result = await validateRefreshToken('dqr_nonexistent'); + expect(result).toBeNull(); + }); +}); + +describe('blacklistRefreshToken', () => { + it('blacklists a refresh token', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + expect(await validateRefreshToken(token)).toBe('testuser'); + await blacklistRefreshToken(token); + expect(await validateRefreshToken(token)).toBeNull(); + }); + + it('does not affect other tokens when blacklisting one', async () => { + const { generateRefreshToken, storeRefreshToken, validateRefreshToken, blacklistRefreshToken } = await import('../../utils/auth'); + const tokenA = generateRefreshToken(); + const tokenB = generateRefreshToken(); + await storeRefreshToken('user1', tokenA); + await storeRefreshToken('user2', tokenB); + await blacklistRefreshToken(tokenA); + expect(await validateRefreshToken(tokenA)).toBeNull(); + expect(await validateRefreshToken(tokenB)).toBe('user2'); + }); +}); + +describe('cleanupExpiredTokens', () => { + it('removes expired tokens', async () => { + const { generateRefreshToken, storeRefreshToken, cleanupExpiredTokens } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + const row = db.query('SELECT token_hash FROM refresh_tokens ORDER BY created_at DESC LIMIT 1').get() as { token_hash: string }; + db.run(`UPDATE refresh_tokens SET expires_at = '2000-01-01T00:00:00.000Z' WHERE token_hash = ?`, [row.token_hash]); + await cleanupExpiredTokens(); + const remaining = db.query('SELECT COUNT(*) as c FROM refresh_tokens').get() as { c: number }; + expect(remaining.c).toBe(0); + }); + + it('keeps non-expired tokens', async () => { + const { generateRefreshToken, storeRefreshToken, cleanupExpiredTokens } = await import('../../utils/auth'); + const token = generateRefreshToken(); + await storeRefreshToken('testuser', token); + await cleanupExpiredTokens(); + const remaining = db.query('SELECT COUNT(*) as c FROM refresh_tokens').get() as { c: number }; + expect(remaining.c).toBe(1); + }); +}); diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 2c3682f..70797f7 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -1,8 +1,7 @@ -import { existsSync } from "node:fs"; -import { readdirSync } from "node:fs"; +import { existsSync, readdirSync } from "node:fs"; import { migrate as drizzleMigrate } from "drizzle-orm/bun-sqlite/migrator"; import { getDrizzle } from "./drizzle"; -import { getDb } from "./client"; +import { sql } from "drizzle-orm"; import { config } from "../utils/config"; import { getSmtpSettings, upsertSmtpSettings } from "./repo/settings"; import { getGithubIntegration, setGithubIntegration } from "./repo/github"; @@ -30,26 +29,22 @@ export const migrate = async () => { drizzleMigrate(db, { migrationsFolder }); - // Apply schema additions that Drizzle Kit migrations may miss - const sqlite = await getDb(); - const tableInfo = sqlite.query("PRAGMA table_info('projects')").all() as { name: string }[]; - const columns = tableInfo.map(r => r.name); - if (!columns.includes('port')) { - sqlite.exec("ALTER TABLE projects ADD COLUMN port integer"); - console.log("[Migrate] Added projects.port column"); - } - if (!columns.includes('source_dir')) { - sqlite.exec("ALTER TABLE projects ADD COLUMN source_dir text"); - console.log("[Migrate] Added projects.source_dir column"); - } - if (!columns.includes('source_type')) { - sqlite.exec("ALTER TABLE projects ADD COLUMN source_type text NOT NULL DEFAULT 'git'"); - console.log("[Migrate] Added projects.source_type column"); - } - + await addClearCacheColumn(db); await seedFromConfig(); }; +const addClearCacheColumn = async (db: ReturnType) => { + try { + db.run(sql`ALTER TABLE deployments ADD COLUMN clear_cache integer NOT NULL DEFAULT 0`); + console.log("[Migrate] Added clear_cache column to deployments table"); + } catch (err) { + const cause = err instanceof Error && "cause" in err ? err.cause : err; + if (cause instanceof Error && cause.message.includes("duplicate column name")) return; + console.error("[Migrate] Failed to add clear_cache column:", err); + throw err; + } +}; + const seedFromConfig = async () => { if (config.githubClientId && config.githubClientSecret) { const existing = await getGithubIntegration(); diff --git a/apps/api/src/db/migrations/0001_refresh_tokens.sql b/apps/api/src/db/migrations/0001_refresh_tokens.sql new file mode 100644 index 0000000..0c284c3 --- /dev/null +++ b/apps/api/src/db/migrations/0001_refresh_tokens.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS refresh_tokens ( + id text PRIMARY KEY NOT NULL, + username text NOT NULL, + token_hash text NOT NULL UNIQUE, + expires_at text NOT NULL, + created_at text NOT NULL, + blacklisted_at text +); diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index df2ce5b..d7da1e3 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -2,12 +2,19 @@ "version": "6", "dialect": "sqlite", "entries": [ -{ - "idx": 0, + { + "idx": 0, "version": "6", "when": 1718000000000, "tag": "0000_baseline", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1782100000000, + "tag": "0001_refresh_tokens", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/apps/api/src/db/repo/deployments.ts b/apps/api/src/db/repo/deployments.ts index b4a983d..bdaf080 100644 --- a/apps/api/src/db/repo/deployments.ts +++ b/apps/api/src/db/repo/deployments.ts @@ -20,6 +20,7 @@ const mapDeployment = (row: typeof deployments.$inferSelect): Deployment => ({ replicas: row.replicas, environment: row.environment, failureReason: row.failureReason, + clearCache: Boolean(row.clearCache), createdAt: row.createdAt, updatedAt: row.updatedAt, }); @@ -38,6 +39,7 @@ export const createDeployment = async (input: CreateDeploymentInput): Promise | null = null; - -export const startGitWatcher = () => { - if (interval) return; - interval = setInterval(poll, POLL_INTERVAL_MS); - console.log('[GitWatcher] Started (poll every 60s)'); -}; - -export const stopGitWatcher = () => { - if (interval) { clearInterval(interval); interval = null; } -}; - -async function poll() { - try { - const db = await getDb(); - const projects = db.query( - "SELECT * FROM projects WHERE repo_url IS NOT NULL AND repo_url != ''", - ).all() as any[]; - - for (const project of projects) { - const branch = project.repo_branch || 'main'; - - const latest = db.query( - 'SELECT * FROM deployments WHERE project_id = ? AND source_type = ? ORDER BY created_at DESC LIMIT 1', - ).get(project.id, 'git') as any; - - if (!latest?.commit_sha) continue; - - const remoteSha = await getRemoteSha(project.repo_url, branch); - if (!remoteSha) continue; - - if (remoteSha === latest.commit_sha) continue; - - console.log(`[GitWatcher] New commit detected for ${project.name} (${branch}): ${remoteSha.slice(0, 7)}`); - - const dep = await createDeployment({ - projectId: project.id, - sourceType: 'git', - sourceRef: project.repo_url, - branch, - commitSha: remoteSha, - }); - - orchestrator.enqueue(dep.id); - } - } catch (err) { - console.error('[GitWatcher] Poll error:', err); - } -} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 3906e06..e63a333 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,22 +9,28 @@ import { config } from './utils/config'; import { getDb } from './db/client'; import { scalingEngine } from './scaling/engine'; import { serverManager } from './servers/manager'; -import { startGitWatcher } from './git/watcher'; import { startDomainPolling } from './utils/domain-verifier'; import { alertEvaluator } from './monitoring/evaluator'; +import { loadOrCreateJwtSecret } from './utils/secrets'; +import { initAuth, cleanupExpiredTokens } from './utils/auth'; +import { startBuildCleanup } from './orchestrator/cleanup'; const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); await mkdir(config.caddyRoutesDir, { recursive: true }); + const jwtSecret = await loadOrCreateJwtSecret(dirname(config.databasePath)); + initAuth(jwtSecret); + await migrate(); await orchestrator.reconcileState(); orchestrator.startWorker(); scalingEngine.start(); serverManager.start(); - startGitWatcher(); startDomainPolling(); alertEvaluator.start(); + startBuildCleanup(); + setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { requestsTotal: 0, diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts new file mode 100644 index 0000000..6c684ce --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const tryRunCalls: { cmd: string; args: string[] }[] = []; +const mockTryRun = mock(async (cmd: string, args: string[]) => { + tryRunCalls.push({ cmd, args }); +}); +const mockRedisDel = mock(async () => {}); +const mockRedisLlLen = mock(async () => 0); +const mockRedisQuit = mock(async () => {}); + +beforeEach(() => { + tryRunCalls.length = 0; + mockTryRun.mockClear(); + mockRedisDel.mockClear(); + mockRedisLlLen.mockClear(); + mockRedisQuit.mockClear(); +}); + +mock.module(fileUrl('../runtime'), () => ({ + tryRun: mockTryRun, +})); + +mock.module(fileUrl('../../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + }, +})); + +mock.module(fileUrl('../../utils/docker-bin'), () => ({ + dockerBin: '/usr/bin/docker', +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + llen = mockRedisLlLen; + del = mockRedisDel; + quit = mockRedisQuit; + }, +})); + +const { pruneDocker, pruneDlq } = await import('../cleanup'); + +describe('pruneDocker', () => { + beforeEach(() => { + tryRunCalls.length = 0; + }); + + it('prunes containers with dequel label filter only', async () => { + await pruneDocker(); + + const containerPrune = tryRunCalls.find( + c => c.args[0] === 'container' && c.args[1] === 'prune', + ); + expect(containerPrune).toBeDefined(); + expect(containerPrune!.args).toContain('-f'); + expect(containerPrune!.args).toContain('--filter'); + expect(containerPrune!.args.some(a => a.includes('com.dequel.managed'))).toBe(true); + }); + + it('prunes dangling images without -a flag (preserves cache)', async () => { + await pruneDocker(); + + const imagePrune = tryRunCalls.find( + c => c.args[0] === 'image' && c.args[1] === 'prune', + ); + expect(imagePrune).toBeDefined(); + expect(imagePrune!.args).not.toContain('-a'); + + const buildxPrune = tryRunCalls.find( + c => c.args[0] === 'buildx' && c.args[1] === 'prune', + ); + expect(buildxPrune).toBeDefined(); + expect(buildxPrune!.args).not.toContain('-a'); + }); +}); + +describe('pruneDlq', () => { + it('does not need a Redis connection per tick (shared connection)', async () => { + await pruneDlq(); + + expect(mockRedisLlLen).toHaveBeenCalled(); + expect(mockRedisQuit).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts new file mode 100644 index 0000000..0e8a0b9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect, mock, beforeEach } from 'bun:test'; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +let cleanupFailedDeploymentCalled = false; +let buildShouldFail = false; + +beforeEach(() => { + cleanupFailedDeploymentCalled = false; + buildShouldFail = false; +}); + +const mockDb = { + getDeploymentById: mock(() => Promise.resolve({ + id: 'dep-1', + projectId: 'proj-1', + sourceType: 'git', + sourceRef: 'https://github.com/test/repo.git', + branch: 'main', + status: 'pending', + commitSha: null, + clearCache: false, + environment: null, + imageTag: null, + containerName: null, + })), + getProjectById: mock(() => Promise.resolve({ + id: 'proj-1', + name: 'Test Project', + sourceDir: null, + port: 3000, + cpuLimit: null, + memoryLimitMb: null, + repoUrl: 'https://github.com/test/repo.git', + })), + updateDeploymentStatus: mock(() => Promise.resolve()), + updateDeploymentCommitSha: mock(() => Promise.resolve()), + appendLog: mock(async (_depId: string, stage: string, message: string) => { + if (stage === 'system' && message === 'Deployment is running') { + throw new Error('Simulated DB write failure'); + } + return { sequence: 1 }; + }), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), + listVolumes: mock(() => Promise.resolve([])), + listDeployments: mock(() => Promise.resolve([])), + listAllDatabases: mock(() => Promise.resolve([])), + deleteDeploymentAndLogs: mock(() => Promise.resolve()), + getScalingPolicy: mock(() => Promise.resolve(null)), + updateDomainValidation: mock(() => Promise.resolve()), + listDomains: mock(() => Promise.resolve([])), +}; + +mock.module(fileUrl('../../db/repo'), () => mockDb); + +mock.module(fileUrl('../runtime'), () => ({ + deployContainer: mock(() => Promise.resolve({ + containerName: 'test-project-abc12345', + liveUrl: 'http://test-project.localhost', + })), + cleanupFailedDeployment: mock(async () => { + cleanupFailedDeploymentCalled = true; + }), + ensureContainerRunning: mock(() => Promise.resolve()), + reloadCaddy: mock(() => Promise.resolve()), + tryRun: mock(() => Promise.resolve('')), +})); + +mock.module(fileUrl('../railpack'), () => ({ + buildWithRailpack: mock(async ( + _workspace: string, + _imageTag: string, + _onLog: (line: string) => Promise, + _opts?: unknown, + ) => { + if (buildShouldFail) throw new Error('Build failed'); + }), + CancelledError: class CancelledError extends Error { + constructor() { super('Cancelled'); } + }, +})); + +mock.module(fileUrl('../source'), () => ({ + prepareSourceWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + prepareUploadWorkspace: mock(() => Promise.resolve('/tmp/workspace/dep-1')), + cleanupWorkspace: mock(() => Promise.resolve()), + getHeadSha: mock(() => Promise.resolve('abc123def456')), +})); + +mock.module(fileUrl('../../utils/grafana'), () => ({ + ensureProjectDashboard: mock(() => Promise.resolve()), +})); + +mock.module('ioredis', () => ({ + default: class FakeRedis { + queue: string[] = []; + rpush = mock(async (_key: string, ...values: string[]) => { this.queue.push(...values); }); + blpop = mock(async () => { const v = this.queue.shift(); return v ? ['queue', v] : null; }); + lrem = mock(async () => {}); + zadd = mock(async () => {}); + zrem = mock(async () => {}); + zrangebyscore = mock(async () => []); + quit = mock(async () => {}); + }, +})); + +const { PipelineOrchestrator } = await import('../pipeline'); + +describe('runDeployment cleanup behavior', () => { + it('does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds', async () => { + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(false); + }); + + it('calls cleanupFailedDeployment when failure occurs during build (before deployContainer)', async () => { + buildShouldFail = true; + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment('dep-1'); + + expect(cleanupFailedDeploymentCalled).toBe(true); + }); +}); diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts new file mode 100644 index 0000000..9521f93 --- /dev/null +++ b/apps/api/src/orchestrator/cleanup.ts @@ -0,0 +1,49 @@ +import Redis from 'ioredis'; +import { config } from '../utils/config'; +import { dockerBin } from '../utils/docker-bin'; +import { tryRun } from './runtime'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; + +const DLQ_KEY = 'dequel:deploy:dlq'; +const GC_INTERVAL_MS = 1_800_000; + +let interval: ReturnType | null = null; +let redis: Redis | null = null; + +const getRedis = () => { + if (!redis) redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + return redis; +}; + +export const pruneDocker = async () => { + await tryRun(dockerBin, ['container', 'prune', '-f', '--filter', `label=${DEQUEL_MANAGED_LABEL}`]); + await tryRun(dockerBin, ['image', 'prune', '-f', '--filter', 'until=24h']); + await tryRun(dockerBin, ['buildx', 'prune', '-f', '--filter', 'until=24h']); +}; + +export const pruneDlq = async () => { + try { + const r = getRedis(); + const size = await r.llen(DLQ_KEY); + if (size > 0) { + await r.del(DLQ_KEY); + console.log(`[Cleanup] Purged ${size} items from dead letter queue`); + } + } catch (e) { + console.warn('[Cleanup] DLQ prune failed:', e); + } +}; + +export const startBuildCleanup = () => { + if (interval) return; + console.log('[Cleanup] Docker garbage collector started (every 30min)'); + interval = setInterval(async () => { + await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e)); + await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e)); + }, GC_INTERVAL_MS); +}; + +export const stopBuildCleanup = () => { + if (interval) { clearInterval(interval); interval = null; } + if (redis) { redis.quit(); redis = null; } +}; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 34367ad..2e1fcd1 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -21,11 +21,13 @@ import { getHeadSha, } from "./source"; import { + cleanupFailedDeployment, deployContainer, ensureContainerRunning, reloadCaddy, tryRun, } from "./runtime"; +import { ensureProjectDashboard } from "../utils/grafana"; const now = () => new Date() @@ -157,6 +159,10 @@ export class PipelineOrchestrator { ]); } + if (deployment.imageTag && deployment.sourceType !== "image") { + await tryRun("docker", ["rmi", "-f", deployment.imageTag]); + } + await deleteDeploymentAndLogs( deploymentId, ); @@ -289,6 +295,9 @@ export class PipelineOrchestrator { let workspacePath = ""; let uploadedArchivePath: string | null = null; + let imageTag: string | undefined; + let projectName: string | undefined; + let deployed = false; try { await emitLog( @@ -297,7 +306,7 @@ export class PipelineOrchestrator { "Deployment enqueued", ); - const imageTag = + imageTag = await this.resolveImageTag( deployment, ); @@ -315,14 +324,15 @@ export class PipelineOrchestrator { deployment.sourceType === "git" ) { - const branchLabel = - deployment.branch + const commitLabel = deployment.commitSha + ? ` (commit ${deployment.commitSha.slice(0, 7)})` + : deployment.branch ? ` (branch ${deployment.branch})` : ""; await emitLog( deploymentId, "build", - `Cloning git repository: ${deployment.sourceRef}${branchLabel}`, + `Cloning git repository: ${deployment.sourceRef}${commitLabel}`, ); workspacePath = await prepareSourceWorkspace( @@ -330,6 +340,8 @@ export class PipelineOrchestrator { deployment.sourceRef, deployment.branch ?? undefined, + deployment.commitSha ?? + undefined, ); const sha = await getHeadSha( workspacePath, @@ -381,7 +393,7 @@ export class PipelineOrchestrator { line, ); }, - { cacheKey, sourceDir: project?.sourceDir, signal: controller.signal }, + { cacheKey, sourceDir: project?.sourceDir, signal: controller.signal, clearCache: deployment.clearCache }, ); } else { await emitLog( @@ -480,7 +492,6 @@ export class PipelineOrchestrator { await this.checkCancelled(deploymentId); - let projectName: string | undefined; let cpuLimit: | number | null @@ -530,6 +541,8 @@ export class PipelineOrchestrator { }, ); + deployed = true; + await updateDeploymentStatus( deploymentId, "running", @@ -542,6 +555,25 @@ export class PipelineOrchestrator { }, ); + if (projectName) { + const dashSlug = projectName + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63); + const containerRegex = `${dashSlug}-.*`; + ensureProjectDashboard( + deployment.projectId!, + projectName, + containerRegex, + ).catch((e) => + console.warn( + "[Pipeline] Grafana dashboard creation failed:", + e, + ), + ); + } + if ( oldContainerName && deployment.projectId @@ -600,6 +632,23 @@ export class PipelineOrchestrator { "failed", { failureReason: message }, ); + + if (!deployed) { + await emitLog( + deploymentId, + "system", + "Cleaning up Docker resources from failed deployment", + ); + await cleanupFailedDeployment( + deploymentId, + deployment.sourceType === "image" ? undefined : imageTag, + projectName, + deployment.projectId, + ).catch(e => + console.warn(`[Cleanup] Failed to clean deployment ${deploymentId}:`, e), + ); + } + if (deployment.projectId) { try { const dbs = @@ -626,14 +675,22 @@ export class PipelineOrchestrator { return false; } finally { this.abortControllers.delete(deploymentId); - if (workspacePath) - await cleanupWorkspace( - workspacePath, - ); - if (uploadedArchivePath) - await rm(uploadedArchivePath, { - force: true, - }); + try { + if (workspacePath) + await cleanupWorkspace( + workspacePath, + ); + } catch { + // cleanup failures must never mask the deployment result + } + try { + if (uploadedArchivePath) + await rm(uploadedArchivePath, { + force: true, + }); + } catch { + // cleanup failures must never mask the deployment result + } } } diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index 3fdfe51..c509642 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -22,12 +22,14 @@ const decodeJob = (raw: string): JobPayload | null => { } }; +const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + export class DeploymentQueue { private redis: Redis; private shuttingDown = false; constructor() { - this.redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null }); + this.redis = createRedis(); } async enqueue(deploymentId: string) { @@ -37,11 +39,13 @@ export class DeploymentQueue { async remove(deploymentId: string) { await this.redis.lrem(QUEUE_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); await this.redis.zrem(RETRY_KEY, encodeJob({ id: deploymentId, attempt: 0 })); - const allAttempts = Array.from({ length: 10 }, (_, i) => + const allAttempts = Array.from({ length: config.queueRetryMax + 2 }, (_, i) => encodeJob({ id: deploymentId, attempt: i }), ); await this.redis.zrem(RETRY_KEY, ...allAttempts); - await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: 0 })); + for (let i = 0; i <= config.queueRetryMax + 1; i++) { + await this.redis.lrem(DLQ_KEY, 0, encodeJob({ id: deploymentId, attempt: i })); + } } async start(handler: (deploymentId: string) => Promise) { @@ -55,40 +59,47 @@ export class DeploymentQueue { } private async runWorker(workerId: number, handler: (deploymentId: string) => Promise) { - while (!this.shuttingDown) { - await this.requeueDueJobs(); + const workerRedis = createRedis(); + try { + while (!this.shuttingDown) { + await this.requeueDueJobs(workerRedis); - const item = await this.redis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); - if (!item) continue; - const job = decodeJob(item[1]); - if (!job) continue; + const item = await workerRedis.blpop(QUEUE_KEY, BLOCK_TIMEOUT_SEC); + if (!item) continue; + const job = decodeJob(item[1]); + if (!job) continue; - try { - const ok = await handler(job.id); - if (!ok) await this.retryOrDlq(job); - } catch (err) { - console.error(`[Queue] Worker ${workerId} handler error:`, err); - await this.retryOrDlq(job); + try { + const ok = await handler(job.id); + if (!ok) await this.retryOrDlq(job, workerRedis); + } catch (err) { + console.error(`[Queue] Worker ${workerId} handler error:`, err); + await this.retryOrDlq(job, workerRedis); + } } + } finally { + await workerRedis.quit(); } } - private async retryOrDlq(job: JobPayload) { + private async retryOrDlq(job: JobPayload, redis?: Redis) { + const r = redis ?? this.redis; const attempt = job.attempt + 1; if (attempt > config.queueRetryMax) { - await this.redis.rpush(DLQ_KEY, encodeJob({ ...job, attempt })); + await r.rpush(DLQ_KEY, encodeJob({ ...job, attempt })); return; } const delayMs = config.queueRetryBaseMs * Math.pow(2, attempt - 1); const runAt = Date.now() + delayMs; - await this.redis.zadd(RETRY_KEY, String(runAt), encodeJob({ ...job, attempt })); + await r.zadd(RETRY_KEY, String(runAt), encodeJob({ ...job, attempt })); } - private async requeueDueJobs() { + private async requeueDueJobs(redis?: Redis) { + const r = redis ?? this.redis; const now = Date.now(); - const jobs = await this.redis.zrangebyscore(RETRY_KEY, 0, now, 'LIMIT', 0, 50); + const jobs = await r.zrangebyscore(RETRY_KEY, 0, now, 'LIMIT', 0, 50); if (!jobs.length) return; - await this.redis.zrem(RETRY_KEY, ...jobs); - await this.redis.rpush(QUEUE_KEY, ...jobs); + await r.zrem(RETRY_KEY, ...jobs); + await r.rpush(QUEUE_KEY, ...jobs); } } diff --git a/apps/api/src/orchestrator/railpack.ts b/apps/api/src/orchestrator/railpack.ts index 2f15b40..541d64c 100644 --- a/apps/api/src/orchestrator/railpack.ts +++ b/apps/api/src/orchestrator/railpack.ts @@ -722,6 +722,7 @@ export const buildWithRailpack = async ( cacheKey?: string; sourceDir?: string | null; signal?: AbortSignal; + clearCache?: boolean; }, ): Promise => { await onLog( @@ -731,13 +732,18 @@ export const buildWithRailpack = async ( // Ensure builder exists (persists across builds) await ensureBuilder(); - const cacheKey = + let cacheKey = opts?.cacheKey ?? imageTag .split(":")[0] .replace(/-[0-9a-f]{8}$/i, "") // Strip unique deployment short ID suffix .replace(/[^a-zA-Z0-9_-]/g, "-"); + if (opts?.clearCache) { + cacheKey = `${cacheKey}-clear-${Date.now()}`; + await onLog(`Bypassing cache for clean build (cacheKey: ${cacheKey})`); + } + const cleanSourceDir = opts?.sourceDir ? opts.sourceDir.replace(/^\//, "") : null; diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index 22e7422..3592523 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; const slugify = (s: string) => s.toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63); @@ -75,6 +76,7 @@ const waitForRunningContainer = async ( await new Promise(r => setTimeout(r, 500)); } await onLog(`Container ${containerName} did not reach running state — attempting docker start`); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); await tryRun(dockerBin, ['start', containerName]); await tryRun(dockerBin, ['network', 'connect', config.dockerNetwork, containerName]); }; @@ -82,7 +84,10 @@ const waitForRunningContainer = async ( export const ensureContainerRunning = async (containerName: string) => { try { const status = (await run(dockerBin, ['inspect', '-f', '{{.State.Status}}', containerName])).trim(); - if (status !== 'running') await run(dockerBin, ['start', containerName]); + if (status !== 'running') { + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await run(dockerBin, ['start', containerName]); + } await tryRun(dockerBin, ['network', 'connect', config.dockerNetwork, containerName]); } catch (error) { console.error(`Failed to reconcile container ${containerName}:`, error); @@ -90,10 +95,31 @@ export const ensureContainerRunning = async (containerName: string) => { }; export const reloadCaddy = async () => { + if (process.env.NODE_ENV === 'test') return; const caddyContainer = await getCaddyContainer(); await run(dockerBin, ['exec', caddyContainer, 'caddy', 'reload', '--config', '/etc/caddy/Caddyfile']); }; +export const getContainerName = (deploymentId: string, projectName?: string, projectId?: string) => { + const slug = slugify(projectName || projectId || deploymentId); + return `${slug}-${deploymentId.slice(0, 8)}`; +}; + +export const cleanupFailedDeployment = async ( + deploymentId: string, + imageTag?: string | null, + projectName?: string, + projectId?: string, + sourceType?: string | null, +) => { + const containerName = getContainerName(deploymentId, projectName, projectId); + await tryRun(dockerBin, ['network', 'disconnect', '-f', config.dockerNetwork, containerName]); + await tryRun(dockerBin, ['rm', '-f', containerName]); + if (imageTag && sourceType !== "image") { + await tryRun(dockerBin, ['rmi', '-f', imageTag]); + } +}; + export const deployContainer = async ( deploymentId: string, imageTag: string, @@ -112,6 +138,7 @@ export const deployContainer = async ( 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, '-e', `PORT=${opts.appPort ?? config.appInternalPort}`, ]; diff --git a/apps/api/src/orchestrator/source.ts b/apps/api/src/orchestrator/source.ts index 2f05918..f2f813f 100644 --- a/apps/api/src/orchestrator/source.ts +++ b/apps/api/src/orchestrator/source.ts @@ -34,14 +34,26 @@ const run = (cmd: string, args: string[], cwd?: string) => }); }); -export const prepareSourceWorkspace = async (deploymentId: string, gitUrl: string, branch?: string) => { +const isValidSha = (s: string) => /^[0-9a-f]{7,40}$/i.test(s); + +export const prepareSourceWorkspace = async (deploymentId: string, gitUrl: string, branch?: string, commitSha?: string) => { const root = join(config.workspaceRoot, deploymentId); await rm(root, { recursive: true, force: true }); await mkdir(root, { recursive: true }); - const args = ['clone', '--depth', '1']; - if (branch) args.push('--branch', branch); - args.push(gitUrl, root); - await run('git', args); + if (commitSha) { + if (!isValidSha(commitSha)) { + throw new Error(`Invalid commit SHA: ${commitSha}`); + } + await run('git', ['init'], root); + await run('git', ['remote', 'add', 'origin', gitUrl], root); + await run('git', ['fetch', '--depth', '1', 'origin', commitSha], root); + await run('git', ['checkout', 'FETCH_HEAD'], root); + } else { + const args = ['clone', '--depth', '1']; + if (branch) args.push('--branch', branch); + args.push(gitUrl, root); + await run('git', args); + } return root; }; diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts new file mode 100644 index 0000000..e4cf27a --- /dev/null +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -0,0 +1,284 @@ +import { describe, it, expect, mock, beforeEach, afterAll } from 'bun:test'; + +const TEST_DEPLOYMENT = { id: 'dep-1', projectId: 'proj-1', containerName: 'deploy-dep-1' }; + +const TEST_POLICY = { + id: 'pol-1', projectId: 'proj-1', enabled: true, + minReplicas: 1, maxReplicas: 5, + cpuThresholdPercent: 70, memoryThresholdPercent: 85, + cooldownSeconds: 120, createdAt: '', updatedAt: '', +}; + +// Mock state variables +let mockRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; +let mockTryRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; +let mockReadFileContent: string | null = null; +let mockWriteFilePath: string | null = null; +let mockWriteContent: string | null = null; +let mockRedisData: Record = {}; +let mockGetScalingPolicyResult: ((id: string) => any) | null = null; +let mockListEnvVarsResult: any[] = []; + +beforeEach(() => { + mockRunImpl = null; + mockTryRunImpl = null; + mockReadFileContent = null; + mockWriteFilePath = null; + mockWriteContent = null; + mockRedisData = {}; + mockGetScalingPolicyResult = null; + mockListEnvVarsResult = []; +}); + +afterAll(() => { + mockRunImpl = null; + mockTryRunImpl = null; + mockReadFileContent = null; + mockWriteFilePath = null; + mockWriteContent = null; + mockRedisData = {}; + mockGetScalingPolicyResult = null; + mockListEnvVarsResult = []; +}); + +// Mock docker-utils +const fileUrl = (path: string) => new URL(path, import.meta.url).toString(); +mock.module(fileUrl('../docker-utils'), () => ({ + run: mock((cmd: string, args: string[]) => { + if (mockRunImpl) return mockRunImpl(cmd, args); + return Promise.resolve(''); + }), + tryRun: mock((cmd: string, args: string[]) => { + if (mockTryRunImpl) return mockTryRunImpl(cmd, args); + return Promise.resolve(''); + }), +})); + +// Mock fs +mock.module('node:fs/promises', () => { + const fs = require('node:fs'); + const pathMod = require('node:path'); + return { + readFile: mock(async (path: string, options?: any) => { + if (mockReadFileContent !== null) { + return mockReadFileContent; + } + if (fs.existsSync(path)) { + return fs.readFileSync(path, options || 'utf8'); + } + throw new Error(`ENOENT: no such file or directory, open '${path}'`); + }), + writeFile: mock(async (path: string, content: string, options?: any) => { + mockWriteFilePath = path; + mockWriteContent = content; + const dir = pathMod.dirname(path); + if (fs.existsSync(dir)) { + fs.writeFileSync(path, content, options || 'utf8'); + } + }), + }; +}); + +// Mock redis +mock.module('ioredis', () => ({ + default: class FakeRedis { + hgetall = mock(async () => mockRedisData); + hset = mock(async (_key: string, data: Record) => { + mockRedisData = { ...mockRedisData, ...data }; + }); + quit = mock(async () => {}); + }, +})); + +// Mock db/repo +mock.module(fileUrl('../../db/repo'), () => ({ + getProjectById: mock(() => Promise.resolve(null)), + listDeployments: mock(() => Promise.resolve([])), + getScalingPolicy: mock((id: string) => + mockGetScalingPolicyResult ? Promise.resolve(mockGetScalingPolicyResult(id)) : Promise.resolve(null) + ), + updateDeploymentStatus: mock(() => Promise.resolve()), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve(mockListEnvVarsResult)), + listDomains: mock(() => Promise.resolve([])), + updateDomainValidation: mock(() => Promise.resolve()), +})); + +mock.module(fileUrl('../../utils/config'), () => ({ + config: { + redisUrl: 'redis://localhost:6379', + caddyRoutesDir: '/tmp/dequel-test-routes', + caddyBaseDomain: 'localhost', + appInternalPort: 3000, + dockerNetwork: 'dequel_net', + }, +})); + + +mock.module(fileUrl('../../utils/docker-bin'), () => ({ dockerBin: '/usr/bin/docker' })); + +const { scalingEngine } = await import('../engine'); + +// --- Tests --- + +describe('parseMemToMb', () => { + it('converts GiB to MB', () => { + expect((scalingEngine as any).parseMemToMb('2GiB')).toBe(2048); + }); + + it('converts MiB directly', () => { + expect((scalingEngine as any).parseMemToMb('512MiB')).toBe(512); + }); + + it('converts KiB to MB', () => { + expect((scalingEngine as any).parseMemToMb('1024KiB')).toBe(1); + }); + + it('returns 0 for unrecognized format', () => { + expect((scalingEngine as any).parseMemToMb('')).toBe(0); + expect((scalingEngine as any).parseMemToMb('abc')).toBe(0); + }); + + it('handles MB format', () => { + expect((scalingEngine as any).parseMemToMb('256MB')).toBe(256); + }); +}); + +describe('getCurrentReplicas', () => { + it('returns 1 when no route file exists', async () => { + mockReadFileContent = null; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(1); + }); + + it('returns 1 when no reverse_proxy in file', async () => { + mockReadFileContent = 'proj-1.localhost:80 {\n log\n}\n'; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(1); + }); + + it('counts unique containers in reverse_proxy targets', async () => { + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(2); + }); + + it('ignores port suffixes when counting containers', async () => { + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy a:80 b:80 c:80\n}\n'; + const result = await (scalingEngine as any).getCurrentReplicas('proj-1'); + expect(result).toBe(3); + }); +}); + +describe('evaluate', () => { + beforeEach(() => { + mockGetScalingPolicyResult = () => TEST_POLICY; + mockListEnvVarsResult = []; + mockReadFileContent = null; + mockRunImpl = null; + mockTryRunImpl = null; + }); + + it('returns early when deployment has no projectId', async () => { + await (scalingEngine as any).evaluate({ id: 'dep-1', projectId: null, containerName: null }); + }); + + it('returns early when no scaling policy', async () => { + mockGetScalingPolicyResult = null; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + }); + + it('returns early when policy is disabled', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, enabled: false }); + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + }); + + it('returns early when container stats fail', async () => { + mockRunImpl = async () => 'not-json'; + const result = await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(result).toBeUndefined(); + }); + + it('sets highCpuSince on first high CPU reading', async () => { + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.highCpuSince).toBeTruthy(); + }); + + it('does not scale up if highCpuSince is not old enough', async () => { + mockRedisData = { highCpuSince: String(Date.now() - 5000), lastScaleUp: '0' }; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it('scales up when CPU exceeds threshold past cooldown', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n'; + + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.some(a => a.includes('stats'))) return '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + if (args.some(a => a.includes('Image'))) return 'my-app:latest'; + if (args.some(a => a.includes('Env'))) return '["PORT=3000"]'; + if (args.some(a => a.includes('Mounts'))) return '[]'; + if (args[0] === 'run') return 'new-container-id'; + if (args[0] === 'ps') return 'caddy-abc123\n'; + if (args.some(a => a.includes('reload'))) return ''; + return ''; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain('deploy-dep-1-replica-2'); + }); + + it('does not scale up when already at max replicas', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it('does not scale down when CPU is moderate', async () => { + mockRunImpl = async () => '{"CPUPerc":"50%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it('scales down when CPU stays low for 5 min', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 1, cooldownSeconds: 0 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + mockTryRunImpl = async () => ''; + + let callCount = 0; + mockRunImpl = async (_cmd: string, args: string[]) => { + callCount++; + if (args.includes('stats')) return '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === 'stop') return ''; + if (args[0] === 'rm') return ''; + if (args[0] === 'ps') return 'caddy-abc123\n'; + if (args.includes('reload')) return ''; + return ''; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + + expect(mockWriteContent).toContain('deploy-dep-1'); + expect(mockWriteContent).not.toContain('replica-2'); + }); + + it('does not scale down below minReplicas', async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 2, cooldownSeconds: 0 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: '0' }; + mockReadFileContent = 'proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n'; + mockRunImpl = async () => '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); +}); diff --git a/apps/api/src/scaling/docker-utils.ts b/apps/api/src/scaling/docker-utils.ts new file mode 100644 index 0000000..44553cd --- /dev/null +++ b/apps/api/src/scaling/docker-utils.ts @@ -0,0 +1,17 @@ +import { spawn } from 'node:child_process'; + +export const run = (cmd: string, args: string[]) => + new Promise((resolve, reject) => { + const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += String(chunk); }); + child.stderr.on('data', (chunk) => { stderr += String(chunk); }); + child.on('close', (code) => { + if (code === 0) resolve((stdout + '\n' + stderr).trim()); + else reject(new Error(`${cmd} ${args.join(' ')} failed (${code}): ${stderr}`)); + }); + }); + +export const tryRun = (cmd: string, args: string[]) => + run(cmd, args).catch(() => ''); diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 510d5f3..afdc173 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -1,27 +1,12 @@ -import { spawn } from 'node:child_process'; import { readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import Redis from 'ioredis'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; +import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; +import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy } from '../db/repo'; -const run = (cmd: string, args: string[]) => - new Promise((resolve, reject) => { - const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => { stdout += String(chunk); }); - child.stderr.on('data', (chunk) => { stderr += String(chunk); }); - child.on('close', (code) => { - if (code === 0) resolve((stdout + '\n' + stderr).trim()); - else reject(new Error(`${cmd} ${args.join(' ')} failed (${code}): ${stderr}`)); - }); - }); - -const tryRun = (cmd: string, args: string[]) => - run(cmd, args).catch(() => ''); - interface ContainerStats { containerName: string; cpuPercent: number; @@ -217,6 +202,7 @@ class ScalingEngine { await run(dockerBin, [ 'run', '-d', '--name', containerName, '--network', config.dockerNetwork, + '-l', DEQUEL_MANAGED_LABEL, ...volumes, ...envVars, imageTag, ]); diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index d315c31..e7bb643 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -221,6 +221,7 @@ export interface Deployment { replicas: number; environment: string | null; failureReason: string | null; + clearCache: boolean; createdAt: string; updatedAt: string; } @@ -239,6 +240,7 @@ export interface CreateDeploymentInput { branch?: string; commitSha?: string; environment?: string; + clearCache?: boolean; } export interface DeploymentLog { diff --git a/apps/api/src/utils/__tests__/auth.test.ts b/apps/api/src/utils/__tests__/auth.test.ts new file mode 100644 index 0000000..62cb317 --- /dev/null +++ b/apps/api/src/utils/__tests__/auth.test.ts @@ -0,0 +1,155 @@ +import { + describe, + it, + expect, + mock, + beforeAll, + beforeEach, + afterAll, +} from "bun:test"; +import { Database } from "bun:sqlite"; + +const TEST_SECRET = + "test-jwt-secret-for-testing-purposes-only"; + +describe("JWT operations", () => { + beforeAll(async () => { + const { initAuth } = + await import("../auth"); + initAuth(TEST_SECRET); + }); + + it("signs and verifies an access token", async () => { + const { + signAccessToken, + verifyAccessToken, + } = await import("../auth"); + const token = + await signAccessToken("testuser"); + expect(typeof token).toBe("string"); + expect(token.split(".").length).toBe(3); + const payload = + await verifyAccessToken(token); + expect(payload).not.toBeNull(); + expect(payload!.sub).toBe("testuser"); + expect(payload!.iat).toBeGreaterThan(0); + expect(payload!.exp).toBe( + payload!.iat + 900, + ); + }); + + it("rejects a tampered token", async () => { + const { + signAccessToken, + verifyAccessToken, + } = await import("../auth"); + const token = + await signAccessToken("testuser"); + const parts = token.split("."); + const tampered = [ + "bad", + parts[1], + parts[2], + ].join("."); + expect( + await verifyAccessToken(tampered), + ).toBeNull(); + }); + + it("rejects token with wrong secret", async () => { + const { + signAccessToken, + verifyAccessToken, + initAuth, + } = await import("../auth"); + const token = + await signAccessToken("testuser"); + initAuth("different-secret"); + const result = + await verifyAccessToken(token); + expect(result).toBeNull(); + initAuth(TEST_SECRET); + }); + + it("rejects malformed token", async () => { + const { verifyAccessToken } = + await import("../auth"); + expect( + await verifyAccessToken("not-a-jwt"), + ).toBeNull(); + expect( + await verifyAccessToken( + "only.two.parts.here", + ), + ).toBeNull(); + expect( + await verifyAccessToken(""), + ).toBeNull(); + }); + + it("rejects expired token", async () => { + const { signAccessToken, verifyAccessToken } = + await import("../auth"); + const token = await signAccessToken("testuser"); + const [, payloadPart] = token.split("."); + const padded = payloadPart + .replace(/-/g, "+") + .replace(/_/g, "/") + .padEnd( + Math.ceil(payloadPart.length / 4) * 4, + "=", + ); + const payload = JSON.parse( + atob(padded), + ) as { exp: number }; + + const originalNow = Date.now; + Date.now = () => (payload.exp + 1) * 1000; + try { + expect( + await verifyAccessToken(token), + ).toBeNull(); + } finally { + Date.now = originalNow; + } + }); +}); + +describe("hashToken", () => { + it("produces consistent hashes", async () => { + const { hashToken } = + await import("../auth"); + const h1 = hashToken("test-token"); + const h2 = hashToken("test-token"); + expect(h1).toBe(h2); + expect(h1.length).toBe(64); + }); + + it("produces different hashes for different tokens", async () => { + const { hashToken } = + await import("../auth"); + const h1 = hashToken("token-a"); + const h2 = hashToken("token-b"); + expect(h1).not.toBe(h2); + }); +}); + +describe("generateRefreshToken", () => { + it("generates token with correct prefix", async () => { + const { generateRefreshToken } = + await import("../auth"); + const token = generateRefreshToken(); + expect(token.startsWith("dqr_")).toBe( + true, + ); + expect(token.length).toBe(4 + 64); + }); + + it("generates unique tokens", async () => { + const { generateRefreshToken } = + await import("../auth"); + const t1 = generateRefreshToken(); + const t2 = generateRefreshToken(); + expect(t1).not.toBe(t2); + }); +}); diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.ts new file mode 100644 index 0000000..31fc480 --- /dev/null +++ b/apps/api/src/utils/auth.ts @@ -0,0 +1,117 @@ +import { randomBytes } from 'node:crypto'; +import { getDrizzle } from '../db/drizzle'; +import { getDb } from '../db/client'; +import { eq, and, sql } from 'drizzle-orm'; +import { refreshTokens } from '../db/schema'; + +let jwtSecret = ''; + +export const initAuth = (secret: string) => { + jwtSecret = secret; +}; + +const b64url = (buf: ArrayBuffer): string => + btoa(String.fromCharCode(...new Uint8Array(buf))) + .replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_'); + +const b64urlDecode = (str: string): ArrayBuffer => { + str = str.replace(/-/g, '+').replace(/_/g, '/'); + while (str.length % 4) str += '='; + return Uint8Array.from(atob(str), c => c.charCodeAt(0)).buffer; +}; + +const encoder = new TextEncoder(); + +const hmacSign = async (data: string): Promise => { + const key = await crypto.subtle.importKey( + 'raw', encoder.encode(jwtSecret), + { name: 'HMAC', hash: 'SHA-256' }, + false, ['sign'], + ); + const sig = await crypto.subtle.sign('HMAC', key, encoder.encode(data)); + return b64url(sig); +}; + +export interface JwtPayload { + sub: string; + iat: number; + exp: number; +} + +export const signAccessToken = async (username: string): Promise => { + const header = b64url(encoder.encode(JSON.stringify({ alg: 'HS256', typ: 'JWT' }))); + const now = Math.floor(Date.now() / 1000); + const payload = b64url(encoder.encode(JSON.stringify({ + sub: username, + iat: now, + exp: now + 900, + }))); + const sig = await hmacSign(`${header}.${payload}`); + return `${header}.${payload}.${sig}`; +}; + +export const verifyAccessToken = async (token: string): Promise => { + const parts = token.split('.'); + if (parts.length !== 3) return null; + const [header, payload, sig] = parts; + const expected = await hmacSign(`${header}.${payload}`); + if (sig !== expected) return null; + try { + const data = JSON.parse(new TextDecoder().decode(b64urlDecode(payload))); + if (data.exp && data.exp < Math.floor(Date.now() / 1000)) return null; + return data; + } catch { + return null; + } +}; + +export const hashToken = (token: string): string => { + const hash = Bun.CryptoHasher.hash('sha256', token); + return Buffer.from(hash).toString('hex'); +}; + +export const generateRefreshToken = (): string => + `dqr_${randomBytes(32).toString('hex')}`; + +export const storeRefreshToken = async (username: string, token: string): Promise => { + const drizzle = await getDrizzle(); + const tokenHash = hashToken(token); + const now = new Date().toISOString(); + const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString(); + const id = randomBytes(16).toString('hex'); + drizzle.insert(refreshTokens).values({ + id, + username, + tokenHash, + expiresAt, + createdAt: now, + }).run(); +}; + +export const validateRefreshToken = async (token: string): Promise => { + const drizzle = await getDrizzle(); + const tokenHash = hashToken(token); + const row = drizzle.select().from(refreshTokens) + .where(and( + eq(refreshTokens.tokenHash, tokenHash), + sql`${refreshTokens.blacklistedAt} IS NULL`, + sql`datetime(${refreshTokens.expiresAt}) > datetime('now')`, + )) + .get(); + return row?.username ?? null; +}; + +export const blacklistRefreshToken = async (token: string): Promise => { + const drizzle = await getDrizzle(); + const tokenHash = hashToken(token); + const now = new Date().toISOString(); + drizzle.update(refreshTokens) + .set({ blacklistedAt: now }) + .where(eq(refreshTokens.tokenHash, tokenHash)) + .run(); +}; + +export const cleanupExpiredTokens = async (): Promise => { + const db = await getDb(); + db.run(`DELETE FROM refresh_tokens WHERE expires_at < datetime('now')`); +}; diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index 8c6772e..d311de9 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -1,43 +1,124 @@ -import { loadFileConfig, type FileConfig } from "./config-loader"; +import { loadFileConfig } from "./config-loader"; const fileConfig = loadFileConfig(); const withFile = ( - key: string, - envDefault: string, - transform?: (v: string) => T, + key: string, + envDefault: string, + transform?: (v: string) => T, ): T => { - const envVal = process.env[key]; - if (envVal !== undefined) { - return transform ? transform(envVal) : (envVal as unknown as T); - } - const fileVal = (fileConfig as Record)[key]; - if (fileVal !== undefined) return fileVal as T; - return transform ? transform(envDefault) : (envDefault as unknown as T); + const envVal = process.env[key]; + if (envVal !== undefined) { + return transform + ? transform(envVal) + : (envVal as unknown as T); + } + const fileVal = ( + fileConfig as Record + )[key]; + if (fileVal !== undefined) + return fileVal as T; + return transform + ? transform(envDefault) + : (envDefault as unknown as T); }; +const SYSTEM = { + dockerNetwork: "dequel_net", + buildkitHost: "tcp://buildkit:1234", + redisUrl: "redis://redis:6379", +} as const; + export const config = { - port: withFile("PORT", "3001", Number), - databasePath: withFile("DATABASE_PATH", "/app/data/dequel.db"), - workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), - caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), - caddyBaseDomain: withFile("CADDY_BASE_DOMAIN", "localhost"), - dockerNetwork: withFile("DOCKER_NETWORK", "dequel_net"), - appInternalPort: withFile("APP_INTERNAL_PORT", "3000", Number), - buildkitHost: withFile("BUILDKIT_HOST", "tcp://buildkit:1234"), - envEncryptionKey: withFile("ENV_ENCRYPTION_KEY", "dev-env-key-change-me"), - redisUrl: withFile("REDIS_URL", "redis://redis:6379"), - queueConcurrency: withFile("QUEUE_CONCURRENCY", "1", Number), - queueRetryMax: withFile("QUEUE_RETRY_MAX", "5", Number), - queueRetryBaseMs: withFile("QUEUE_RETRY_BASE_MS", "5000", Number), - smtpHost: withFile("SMTP_HOST", ""), - smtpPort: withFile("SMTP_PORT", "587", Number), - smtpUser: withFile("SMTP_USER", ""), - smtpPass: withFile("SMTP_PASS", ""), - smtpFrom: withFile("SMTP_FROM", "dequel@localhost"), - alertEvalIntervalMs: withFile("ALERT_EVAL_INTERVAL_MS", "60000", Number), - githubClientId: withFile("GITHUB_CLIENT_ID", ""), - githubClientSecret: withFile("GITHUB_CLIENT_SECRET", ""), - githubAppName: withFile("GITHUB_APP_NAME", "Dequel"), - githubWebhookSecret: withFile("GITHUB_WEBHOOK_SECRET", ""), + ...SYSTEM, + databasePath: withFile("DATABASE_PATH", "/app/data/dequel.db"), + workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), + caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), + port: withFile( + "PORT", + "17474", + Number, + ), + caddyBaseDomain: withFile( + "CADDY_BASE_DOMAIN", + "localhost", + ), + appInternalPort: withFile( + "APP_INTERNAL_PORT", + "17476", + Number, + ), + envEncryptionKey: withFile( + "ENV_ENCRYPTION_KEY", + "dev-env-key-change-me", + ), + queueConcurrency: withFile( + "QUEUE_CONCURRENCY", + "3", + Number, + ), + queueRetryMax: withFile( + "QUEUE_RETRY_MAX", + "5", + Number, + ), + queueRetryBaseMs: withFile( + "QUEUE_RETRY_BASE_MS", + "5000", + Number, + ), + smtpHost: withFile( + "SMTP_HOST", + "", + ), + smtpPort: withFile( + "SMTP_PORT", + "587", + Number, + ), + smtpUser: withFile( + "SMTP_USER", + "", + ), + smtpPass: withFile( + "SMTP_PASS", + "", + ), + smtpFrom: withFile( + "SMTP_FROM", + "dequel@localhost", + ), + alertEvalIntervalMs: withFile( + "ALERT_EVAL_INTERVAL_MS", + "60000", + Number, + ), + githubClientId: withFile( + "GITHUB_CLIENT_ID", + "", + ), + githubClientSecret: withFile( + "GITHUB_CLIENT_SECRET", + "", + ), + githubAppName: withFile( + "GITHUB_APP_NAME", + "Dequel", + ), + githubWebhookSecret: withFile( + "GITHUB_WEBHOOK_SECRET", + "", + ), + grafanaUrl: withFile( + "GRAFANA_URL", + "http://grafana:3000", + ), + grafanaUser: withFile( + "GRAFANA_USER", + "admin", + ), + grafanaPass: withFile( + "GRAFANA_PASS", + "admin", + ), }; diff --git a/apps/api/src/utils/dequel-labels.ts b/apps/api/src/utils/dequel-labels.ts new file mode 100644 index 0000000..cac7a70 --- /dev/null +++ b/apps/api/src/utils/dequel-labels.ts @@ -0,0 +1 @@ +export const DEQUEL_MANAGED_LABEL = 'com.dequel.managed=true'; diff --git a/apps/api/src/utils/dns.ts b/apps/api/src/utils/dns.ts index 831d10d..0097c91 100644 --- a/apps/api/src/utils/dns.ts +++ b/apps/api/src/utils/dns.ts @@ -1,4 +1,5 @@ import { promises as dns } from 'node:dns'; +import { config } from './config'; export const resolveServerIp = async (): Promise => { try { @@ -9,6 +10,35 @@ export const resolveServerIp = async (): Promise => { } }; +export type BaseDomainStatus = { + ip: string; + baseDomain: string; + resolves: boolean; + url: string; +}; + +export const checkBaseDomainStatus = async (): Promise => { + const ip = await resolveServerIp(); + const baseDomain = config.caddyBaseDomain; + const isLocalhost = baseDomain === 'localhost'; + + let resolves = false; + if (!isLocalhost) { + try { + const addresses = await dns.resolve4(baseDomain); + resolves = addresses.includes(ip); + } catch {} + } else { + resolves = true; + } + + const url = isLocalhost || resolves + ? `${isLocalhost ? 'http' : 'https'}://${baseDomain}${isLocalhost ? ':80' : ''}` + : `http://${ip}`; + + return { ip, baseDomain, resolves, url }; +}; + export const validateDomain = async ( domain: string, serverIp: string, diff --git a/apps/api/src/utils/grafana.ts b/apps/api/src/utils/grafana.ts new file mode 100644 index 0000000..7559654 --- /dev/null +++ b/apps/api/src/utils/grafana.ts @@ -0,0 +1,616 @@ +import { config } from "./config"; +import { listDomains } from "../db/repo"; + +interface GrafanaDashboard { + dashboard: { + title: string; + uid: string; + tags: string[]; + schemaVersion: number; + version: number; + timezone: string; + refresh: string; + panels: unknown[]; + }; + overwrite: boolean; +} + +let sessionCookie: string | null = null; +let sessionExpires = 0; + +async function grafanaLogin(): Promise { + if (sessionCookie && Date.now() < sessionExpires) { + return sessionCookie; + } + try { + const res = await fetch(`${config.grafanaUrl}/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + user: config.grafanaUser, + password: config.grafanaPass, + }), + }); + const setCookie = res.headers.get("set-cookie"); + if (!setCookie) return null; + sessionCookie = setCookie.split(";")[0]; + sessionExpires = Date.now() + 60 * 60 * 1000; + return sessionCookie; + } catch { + return null; + } +} + +async function grafanaPost( + path: string, + body: unknown, +): Promise { + const cookie = await grafanaLogin(); + if (!cookie) return null; + try { + const res = await fetch(`${config.grafanaUrl}/api${path}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Cookie: cookie, + }, + body: JSON.stringify(body), + }); + return await res.json(); + } catch { + return null; + } +} + +export async function ensureProjectDashboard( + projectId: string, + projectName: string, + containerRegex: string, +): Promise { + const slug = projectName + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63); + + const domains = [`${slug}.${config.caddyBaseDomain}`]; + try { + const projectDomains = await listDomains(projectId); + const verified = projectDomains.filter(d => d.validationStatus === 'verified'); + for (const d of verified) { + domains.push(d.domain); + } + } catch (e) { + console.warn("[Grafana] Failed to list domains for dashboard query:", e); + } + + const regexEscaped = domains.map(d => d.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\\\$&')).join('|'); + + const dashboard: GrafanaDashboard = { + dashboard: { + title: `Dequel \u2014 ${projectName}`, + uid: `dequel-project-${slug}`, + tags: ["dequel", "project", slug], + schemaVersion: 39, + version: 1, + timezone: "browser", + refresh: "10s", + panels: [ + { + type: "row", + title: "Overview", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 0 }, + }, + { + id: 5, + type: "stat", + title: "Requests (period)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 0, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "blue" + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__range]))`, + refId: "A" + } + ] + }, + { + id: 6, + type: "stat", + title: "% Success", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 3, y: 1 }, + fieldConfig: { + defaults: { + unit: "percent", + color: { mode: "thresholds" }, + thresholds: { + mode: "absolute", + steps: [ + { color: "red", value: null }, + { color: "yellow", value: 90 }, + { color: "green", value: 95 } + ] + } + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `(sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 400 [$__range])) / sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__range])) * 100) or 0`, + refId: "A" + } + ] + }, + { + id: 7, + type: "stat", + title: "Avg Latency", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 6, y: 1 }, + fieldConfig: { + defaults: { + unit: "s", + color: { mode: "thresholds" }, + thresholds: { + mode: "absolute", + steps: [ + { color: "green", value: null }, + { color: "yellow", value: 0.5 }, + { color: "red", value: 2.0 } + ] + } + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `avg_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__range])`, + refId: "A" + } + ] + }, + { + id: 8, + type: "stat", + title: "Reqs (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 9, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "blue" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [2m]))`, + refId: "A" + } + ] + }, + { + id: 9, + type: "stat", + title: "% Success (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 3, x: 12, y: 1 }, + fieldConfig: { + defaults: { + unit: "percent", + color: { mode: "thresholds" }, + thresholds: { + mode: "absolute", + steps: [ + { color: "red", value: null }, + { color: "yellow", value: 90 }, + { color: "green", value: 95 } + ] + } + } + }, + options: { + graphMode: "area", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `(sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 400 [2m])) / sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [2m])) * 100) or 0`, + refId: "A" + } + ] + }, + { + id: 10, + type: "stat", + title: "HTTP 1/2xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2, x: 15, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "green" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 300 [2m]))`, + refId: "A" + } + ] + }, + { + id: 11, + type: "stat", + title: "HTTP 3xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2, x: 17, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "orange" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 300 and status < 400 [2m]))`, + refId: "A" + } + ] + }, + { + id: 12, + type: "stat", + title: "HTTP 4xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2.5, x: 19, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "yellow" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 400 and status < 500 [2m]))`, + refId: "A" + } + ] + }, + { + id: 13, + type: "stat", + title: "HTTP 5xx (2m)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 3, w: 2.5, x: 21.5, y: 1 }, + fieldConfig: { + defaults: { + unit: "none", + color: { mode: "fixed" }, + fixedColor: "red" + } + }, + options: { + graphMode: "line", + reduceOptions: { calcs: ["lastNotNull"] } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 500 [2m]))`, + refId: "A" + } + ] + }, + { + type: "row", + title: "HTTP Ingress Performance", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 4 }, + }, + { + id: 14, + type: "timeseries", + title: "HTTP Requests / Ingress", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 8, x: 0, y: 5 }, + fieldConfig: { + defaults: { + unit: "reqps", + custom: { fillOpacity: 10, lineWidth: 1.5 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(rate({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval])) by (request_host)`, + legendFormat: "{{request_host}}", + refId: "A" + } + ] + }, + { + id: 15, + type: "timeseries", + title: "HTTP Status Codes", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 8, x: 8, y: 5 }, + fieldConfig: { + defaults: { + unit: "reqps", + custom: { fillOpacity: 10, lineWidth: 1.5 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(rate({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval])) by (status)`, + legendFormat: "HTTP {{status}}", + refId: "A" + } + ] + }, + { + id: 16, + type: "timeseries", + title: "Total HTTP Requests", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 8, x: 16, y: 5 }, + fieldConfig: { + defaults: { + unit: "none", + custom: { fillOpacity: 25, lineWidth: 1 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval]))`, + legendFormat: "Requests", + refId: "A" + } + ] + }, + { + type: "row", + title: "Latency", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 13 }, + }, + { + id: 17, + type: "timeseries", + title: "Latency (Average Percentiles)", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 12, x: 0, y: 14 }, + fieldConfig: { + defaults: { + unit: "s", + custom: { fillOpacity: 10, lineWidth: 1.5 } + } + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `quantile_over_time(0.99, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "p99", + refId: "A" + }, + { + datasource: { type: "loki", uid: "loki" }, + expr: `quantile_over_time(0.95, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "p95", + refId: "B" + }, + { + datasource: { type: "loki", uid: "loki" }, + expr: `quantile_over_time(0.50, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "p50 (median)", + refId: "C" + }, + { + datasource: { type: "loki", uid: "loki" }, + expr: `avg_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + legendFormat: "Average", + refId: "D" + } + ] + }, + { + id: 18, + type: "heatmap", + title: "Latency Heatmap", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 8, w: 12, x: 12, y: 14 }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `log_histogram({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + refId: "A" + } + ] + }, + { + type: "row", + title: "Resource Usage", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 22 }, + }, + { + id: 1, + type: "timeseries", + title: "CPU Usage", + datasource: { type: "prometheus", uid: "prometheus" }, + gridPos: { h: 8, w: 12, x: 0, y: 23 }, + fieldConfig: { + defaults: { + unit: "short", + custom: { + stacking: { mode: "normal" }, + fillOpacity: 30, + lineWidth: 1, + }, + }, + overrides: [], + }, + options: { + legend: { + displayMode: "table", + placement: "right", + showLegend: true, + }, + tooltip: { mode: "multi" }, + }, + targets: [ + { + datasource: { type: "prometheus", uid: "prometheus" }, + expr: `rate(container_cpu_usage_seconds_total{name=~"${containerRegex}"}[$__rate_interval])`, + legendFormat: "{{name}}", + refId: "A", + }, + ], + }, + { + id: 2, + type: "timeseries", + title: "Memory Usage", + datasource: { type: "prometheus", uid: "prometheus" }, + gridPos: { h: 8, w: 12, x: 12, y: 23 }, + fieldConfig: { + defaults: { + unit: "bytes", + custom: { + stacking: { mode: "normal" }, + fillOpacity: 30, + lineWidth: 1, + }, + }, + overrides: [], + }, + options: { + legend: { + displayMode: "table", + placement: "right", + showLegend: true, + }, + tooltip: { mode: "multi" }, + }, + targets: [ + { + datasource: { type: "prometheus", uid: "prometheus" }, + expr: `container_memory_working_set_bytes{name=~"${containerRegex}"}`, + legendFormat: "{{name}}", + refId: "A", + }, + ], + }, + { + type: "row", + title: "Logs & Troubleshooting", + collapsed: false, + gridPos: { h: 1, w: 24, x: 0, y: 31 }, + }, + { + id: 19, + type: "logs", + title: "HTTP Request Error Logs", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 10, w: 12, x: 0, y: 32 }, + options: { + showLabels: true, + showTime: true, + wrapLogMessage: true, + enableLogDetails: true, + dedupStrategy: "none", + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 400`, + refId: "A", + }, + ], + }, + { + id: 3, + type: "logs", + title: "Application Container Logs", + datasource: { type: "loki", uid: "loki" }, + gridPos: { h: 10, w: 12, x: 12, y: 32 }, + options: { + showLabels: true, + showTime: true, + wrapLogMessage: true, + enableLogDetails: true, + dedupStrategy: "none", + }, + targets: [ + { + datasource: { type: "loki", uid: "loki" }, + expr: `{container=~"${containerRegex}"}`, + refId: "A", + }, + ], + }, + ], + }, + overwrite: true, + }; + + const result = await grafanaPost("/dashboards/db", dashboard); + if (result) { + console.log( + `[Grafana] Dashboard created/updated for ${projectName}`, + ); + } else { + console.warn( + `[Grafana] Failed to create dashboard for ${projectName}`, + ); + } +} diff --git a/apps/api/src/utils/secrets.ts b/apps/api/src/utils/secrets.ts new file mode 100644 index 0000000..e37e112 --- /dev/null +++ b/apps/api/src/utils/secrets.ts @@ -0,0 +1,16 @@ +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { randomBytes } from 'node:crypto'; + +export const loadOrCreateJwtSecret = async (dataDir: string): Promise => { + const path = join(dataDir, '.jwt_secret'); + try { + const existing = await readFile(path, 'utf8'); + return existing.trim(); + } catch { + await mkdir(dirname(path), { recursive: true }); + const secret = randomBytes(32).toString('hex'); + await writeFile(path, secret + '\n', { mode: 0o600 }); + return secret; + } +}; diff --git a/apps/docs/.astro/astro/content.d.ts b/apps/docs/.astro/astro/content.d.ts index a3ac838..57c8503 100644 --- a/apps/docs/.astro/astro/content.d.ts +++ b/apps/docs/.astro/astro/content.d.ts @@ -140,10 +140,33 @@ declare module 'astro:content' { >; type ContentEntryMap = { - "docs": { -"changelog.md": { - id: "changelog.md"; - slug: "changelog"; + "changelogs": { +"v0.1.0.md": { + id: "v0.1.0.md"; + slug: "v010"; + body: string; + collection: "changelogs"; + data: any +} & { render(): Render[".md"] }; +"v0.1.1.md": { + id: "v0.1.1.md"; + slug: "v011"; + body: string; + collection: "changelogs"; + data: any +} & { render(): Render[".md"] }; +"v0.2.0.md": { + id: "v0.2.0.md"; + slug: "v020"; + body: string; + collection: "changelogs"; + data: any +} & { render(): Render[".md"] }; +}; +"docs": { +"auth.md": { + id: "auth.md"; + slug: "auth"; body: string; collection: "docs"; data: any diff --git a/apps/docs/.astro/settings.json b/apps/docs/.astro/settings.json index 3a7f780..be2c74f 100644 --- a/apps/docs/.astro/settings.json +++ b/apps/docs/.astro/settings.json @@ -1,5 +1,5 @@ { "_variables": { - "lastUpdateCheck": 1780853643721 + "lastUpdateCheck": 1784334507362 } } \ No newline at end of file diff --git a/apps/docs/package.json b/apps/docs/package.json index cc2b340..9091a24 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -1,7 +1,7 @@ { "name": "dequel-docs", "type": "module", - "version": "0.1.0", + "version": "0.2.0", "scripts": { "dev": "astro dev", "start": "astro dev", diff --git a/apps/docs/src/components/Hero.astro b/apps/docs/src/components/Hero.astro index 0a7c8d8..359b666 100644 --- a/apps/docs/src/components/Hero.astro +++ b/apps/docs/src/components/Hero.astro @@ -10,7 +10,7 @@ const { tag = "Private Beta", titleNormal = "Deploy anything.", titleItalic = "Scale everything.", - subText = "The self-hosted cloud platform for developers who want Railway-grade deployment workflows on their own infrastructure. Container orchestration, observability, and scaling — all from one dashboard." + subText = "The self-hosted cloud platform for developers who want Railway-grade deployment workflows on their own infrastructure. Container orchestration, observability, and scaling — all from one dashboard.", } = Astro.props; --- @@ -21,7 +21,7 @@ const {

- {titleNormal}
+ {titleNormal}
{titleItalic}

@@ -33,15 +33,28 @@ const { Read the docs → - -
Client Request
my-app.com
- -
→
+ +
+ + +
Git Push
Repo Webhook
-
→
+
+ +
@@ -79,18 +84,23 @@ Auto SSL / Proxy PORT 80/443 -
→
+
+ + +
Dequel API Docker Orchestrator -
- ↓ +
+
-
→
+
+ +
diff --git a/apps/docs/src/content.config.ts b/apps/docs/src/content.config.ts index 98b1621..90dc20e 100644 --- a/apps/docs/src/content.config.ts +++ b/apps/docs/src/content.config.ts @@ -12,4 +12,12 @@ const docs = defineCollection({ }), }); -export const collections = { docs }; +const changelogs = defineCollection({ + loader: glob({ pattern: "*.md", base: "./src/content/changelogs" }), + schema: z.object({ + version: z.string(), + date: z.string(), + }), +}); + +export const collections = { docs, changelogs }; diff --git a/apps/docs/src/content/changelogs/v0.1.0.md b/apps/docs/src/content/changelogs/v0.1.0.md new file mode 100644 index 0000000..9a23869 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.1.0.md @@ -0,0 +1,34 @@ +--- +version: 0.1.0 +date: "2026-06-08" +--- + +### Added + +- Initial deployment platform with Git, ZIP, and Docker Compose source deploy +- Automatic build detection via Railpack +- Managed PostgreSQL and MySQL database provisioning +- Custom domain attachment with automatic SSL via Caddy / Let's Encrypt +- CPU-threshold based horizontal auto-scaling with configurable cooldown +- Per-project environment variable management with redeploy hooks +- Persistent Docker volume attachments +- Full observability stack: Prometheus, Loki, Grafana, cAdvisor +- CPU / memory threshold alerts via email or webhook +- API key management for programmatic access +- Job queue via Redis for async operations +- Deployment rollback support +- Boot-time reconciliation of container state +- Unified project versioning via root `VERSION` file and sync script +- One-command install script (`install.sh`) for quick setup +- Automated release pipeline via GitHub Actions +- Changelog page in documentation site +- Vercel deployment configuration for documentation site + +### Changed + +- `docker-compose.yml` now references images from `ghcr.io/dequel/*` with local build as fallback +- README updated with new install flow + +### Fixed + +- Railpack build timeout handling and log scrolling diff --git a/apps/docs/src/content/changelogs/v0.1.1.md b/apps/docs/src/content/changelogs/v0.1.1.md new file mode 100644 index 0000000..55bb037 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.1.1.md @@ -0,0 +1,23 @@ +--- +version: 0.1.1 +date: "2026-06-19" +--- + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers diff --git a/apps/docs/src/content/changelogs/v0.2.0.md b/apps/docs/src/content/changelogs/v0.2.0.md new file mode 100644 index 0000000..8d97fbc --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.2.0.md @@ -0,0 +1,27 @@ +--- +version: 0.2.0 +date: "2026-07-17" +--- + +## What's Changed +### Features +- PAM-based SSH authentication for project deployments ([a6c69bc](https://github.com/Lftobs/dequel/commit/a6c69bc)) +- Deploy to a specific commit SHA, not just branch HEAD ([295178d](https://github.com/Lftobs/dequel/commit/295178d)) +- Drizzle ORM migrations and a new deploy UI ([ef3fd36](https://github.com/Lftobs/dequel/commit/ef3fd36)) +- Toggle to clear build cache on next deploy ([ffb2557](https://github.com/Lftobs/dequel/commit/ffb2557)) +- `dequel update` command for self-updating the platform ([875eabd](https://github.com/Lftobs/dequel/commit/875eabd)) +- Mobile-responsive navigation and layouts ([b9d0a96](https://github.com/Lftobs/dequel/commit/b9d0a96)) +- Automatic cleanup of old Docker images and build artifacts ([052b3ab](https://github.com/Lftobs/dequel/commit/052b3ab)) + +### Improvements +- GitHub OAuth now persists sessions across restarts and auto-detects the public URL from proxy headers ([15a8b9b](https://github.com/Lftobs/dequel/commit/15a8b9b)) +- PAM authentication moved to a standalone HTTP service for reliability ([caf25f2](https://github.com/Lftobs/dequel/commit/caf25f2)) +- Auth timeout and libc compatibility fixes ([afc28aa](https://github.com/Lftobs/dequel/commit/afc28aa)) +- Grafana dashboards and logging overhauled ([894a135](https://github.com/Lftobs/dequel/commit/894a135)) +- Install script improved for broader shell compatibility ([a0aa7c0](https://github.com/Lftobs/dequel/commit/a0aa7c0)) +- GitHub repo picker simplified to owner-affiliated repos only ([a3037ad](https://github.com/Lftobs/dequel/commit/a3037ad)) +- Docs navigation reorganized and UI polished ([f87ee8b](https://github.com/Lftobs/dequel/commit/f87ee8b)) + +### Bug Fixes +- Migration errors and UI log display issues resolved ([5c49ed9](https://github.com/Lftobs/dequel/commit/5c49ed9)) +- Project deletion now properly cascades to related records ([0a56270](https://github.com/Lftobs/dequel/commit/0a56270)) diff --git a/apps/docs/src/content/docs/auth.md b/apps/docs/src/content/docs/auth.md new file mode 100644 index 0000000..704e7fa --- /dev/null +++ b/apps/docs/src/content/docs/auth.md @@ -0,0 +1,54 @@ +--- +title: Authentication & Access Control +category: Networking & Security +description: Secure your Dequel dashboard with PAM-based authentication, session management, and API keys. +slug: auth +--- + +Dequel authenticates users against the **Linux system user database** via PAM (Pluggable Authentication Modules). Only users who are members of the `dequel` group can sign in. + +## User Setup + +Create a Linux user and add them to the `dequel` group: + +```bash +sudo useradd -m -s /bin/bash +sudo passwd +sudo usermod -aG dequel +``` + +The user signs into the Dequel dashboard with their Linux username and password. + +## Session Management + +- **Access tokens**: Short-lived (15 min) HMAC-SHA256 JWTs stored in an `httpOnly` cookie. +- **Refresh tokens**: Long-lived (7 days) random tokens stored in the database. Automatically rotated on refresh. +- **Token blacklisting**: Logging out or refreshing invalidates the old refresh token immediately. +- **JWT secret**: Auto-generated on first API startup, stored at `data/.jwt_secret` (0600 permissions). Deleting this file invalidates all sessions. + +## API Keys + +For programmatic access (CI/CD, CLI tools), Dequel uses pre-shared API keys: + +1. Go to **Settings → API Keys** in the dashboard. +2. Create a new key — the full token is shown once. +3. Use it as a Bearer token in the `Authorization` header: + +```bash +curl -H "Authorization: Bearer dqk_" https://dequel.example.com/api/projects +``` + +API keys are scoped to the entire platform with the same permissions as the user who created them. + +## Endpoints + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `POST` | `/api/auth/login` | None | Sign in with username/password | +| `POST` | `/api/auth/logout` | Session | Sign out, blacklist refresh token | +| `POST` | `/api/auth/refresh` | Session | Rotate refresh token | +| `GET` | `/api/auth/me` | None | Check current session status | +| `GET` | `/api/health` | None | Health check | + +The web dashboard redirects to `/login` when no valid session is detected. + diff --git a/apps/docs/src/content/docs/changelog.md b/apps/docs/src/content/docs/changelog.md index fdc5455..c615fbc 100644 --- a/apps/docs/src/content/docs/changelog.md +++ b/apps/docs/src/content/docs/changelog.md @@ -5,6 +5,54 @@ description: All notable changes to Dequel, tracked per release. slug: changelog --- +## 0.2.0 — 2026-07-17 + +### Features + +- PAM-based SSH authentication for project deployments +- Deploy to a specific commit SHA, not just branch HEAD +- Drizzle ORM migrations and a new deploy UI +- Toggle to clear build cache on next deploy +- `dequel update` command for self-updating the platform +- Mobile-responsive navigation and layouts +- Automatic cleanup of old Docker images and build artifacts + +### Improvements + +- GitHub OAuth now persists sessions across restarts and auto-detects the public URL from proxy headers +- PAM authentication moved to a standalone HTTP service for reliability +- Auth timeout and libc compatibility fixes +- Grafana dashboards and logging overhauled +- Install script improved for broader shell compatibility +- GitHub repo picker simplified to owner-affiliated repos only +- Docs navigation reorganized and UI polished + +### Bug Fixes + +- Migration errors and UI log display issues resolved +- Project deletion now properly cascades to related records + +## 0.1.1 — 2026-06-19 + +### Added + +- Per-project Grafana dashboards automatically created on successful deployment +- Configurable `CADDY_BASE_DOMAIN` for public ingress with automatic Let's Encrypt SSL +- Dynamic `railpack.json` generation with deployment abort support +- GitHub webhook management and project management API endpoints +- Project source and port configuration options +- SMTP configuration and system settings API + +### Changed + +- Monitoring stack hardened: Prometheus now validates TSDB blocks and quarantines corrupted ones on startup +- `PUBLIC_URL` is now derived from `CADDY_BASE_DOMAIN` instead of requiring separate configuration +- Refactored infrastructure monitoring configs into dedicated files for maintainability + +### Fixed + +- Container network reconciliation now force-disconnects stale network references before starting containers + ## 0.1.0 — 2026-06-08 ### Added @@ -37,3 +85,4 @@ slug: changelog ### Fixed - Railpack build timeout handling and log scrolling + diff --git a/apps/docs/src/content/docs/configuration.md b/apps/docs/src/content/docs/configuration.md index ac38575..26fbc0a 100644 --- a/apps/docs/src/content/docs/configuration.md +++ b/apps/docs/src/content/docs/configuration.md @@ -44,9 +44,3 @@ app.listen(PORT, '0.0.0.0', () => { }); -
- - Next: Deployments - - -
diff --git a/apps/docs/src/content/docs/databases.md b/apps/docs/src/content/docs/databases.md index 5c0cd14..c907adb 100644 --- a/apps/docs/src/content/docs/databases.md +++ b/apps/docs/src/content/docs/databases.md @@ -35,9 +35,3 @@ To inject this secret into your container runtime, navigate to **Environment Var DATABASE_URL=postgresql://postgres:random-password@db-my-web-app.dequel.local:5432/postgres -
- - Next: Persistent Volumes - - -
diff --git a/apps/docs/src/content/docs/deployments.md b/apps/docs/src/content/docs/deployments.md index 7d5a4c0..811d51f 100644 --- a/apps/docs/src/content/docs/deployments.md +++ b/apps/docs/src/content/docs/deployments.md @@ -29,9 +29,3 @@ Each deployment creates an isolated, immutable build artifact. - **Build Logs:** View the live compiler steps as Docker pulls layers, executes commands, and builds cache images. - **Zero-Downtime Rollbacks:** Instantly switch traffic back to a previous build artifact by clicking the **Promote** button on any previously successful deployment. -
- - Next: Environment Variables - - -
diff --git a/apps/docs/src/content/docs/domains.md b/apps/docs/src/content/docs/domains.md index 1605aee..dd3e3e7 100644 --- a/apps/docs/src/content/docs/domains.md +++ b/apps/docs/src/content/docs/domains.md @@ -29,9 +29,3 @@ In your DNS registrar's control panel (Cloudflare, GoDaddy, Namecheap, etc.), ad - **Verified:** DNS records resolve correctly. The proxy router is configured. - **Failed:** Resolving record targets did not match the required target targets. Check your DNS values. -
- - Next: SSL Certificates - - -
diff --git a/apps/docs/src/content/docs/env-vars.md b/apps/docs/src/content/docs/env-vars.md index 28422b0..fbb0ce3 100644 --- a/apps/docs/src/content/docs/env-vars.md +++ b/apps/docs/src/content/docs/env-vars.md @@ -26,9 +26,3 @@ Because environment variables are injected during container startup, modifying o Dequel displays a **Redeployment Warning Banner** at the top of the tab whenever variables are mutated. Click the **Redeploy Now** button on the banner to execute a zero-downtime rolling update with the new configurations. -
- - Next: Scaling Policies - - -
diff --git a/apps/docs/src/content/docs/index.md b/apps/docs/src/content/docs/index.md index c283f8a..47318df 100644 --- a/apps/docs/src/content/docs/index.md +++ b/apps/docs/src/content/docs/index.md @@ -27,9 +27,4 @@ When you connect a repository or upload a folder: 3. The container image is pushed to the local cluster registry and distributed across running nodes. 4. Caddy routing endpoints are automatically updated to direct traffic to the newly created instance ports. -
- - Next: Quickstart Guide - - -
+ diff --git a/apps/docs/src/content/docs/installation.md b/apps/docs/src/content/docs/installation.md index 5fda364..41c2a61 100644 --- a/apps/docs/src/content/docs/installation.md +++ b/apps/docs/src/content/docs/installation.md @@ -34,6 +34,18 @@ dequel start Open `http://localhost` to access the dashboard (or the configured `CADDY_BASE_DOMAIN` in production). +## Post-Installation: User Setup + +Dequel authenticates against Linux system users in the `dequel` group. After install, create a user: + +```bash +sudo useradd -m -s /bin/bash +sudo passwd +sudo usermod -aG dequel +``` + +See [Authentication & Access Control](/docs/auth) for details on session management and API keys. + ## Manual Setup (no install script) If the installer fails, set up the platform manually with just Docker Compose and the config files: @@ -150,3 +162,4 @@ dequel update ``` This pulls the latest images from GitHub Container Registry and recreates the services. + diff --git a/apps/docs/src/content/docs/quickstart.md b/apps/docs/src/content/docs/quickstart.md index 00ea75d..201720b 100644 --- a/apps/docs/src/content/docs/quickstart.md +++ b/apps/docs/src/content/docs/quickstart.md @@ -51,9 +51,3 @@ On the deployments page, you can choose between connecting your GitHub repositor Once the deployment changes status from `BUILDING` to `READY`, click the external link button near your custom domain or the auto-generated base domain to see your running web container. -
- - Next: Configuration - - -
diff --git a/apps/docs/src/content/docs/scaling.md b/apps/docs/src/content/docs/scaling.md index c9b241d..cafb7d8 100644 --- a/apps/docs/src/content/docs/scaling.md +++ b/apps/docs/src/content/docs/scaling.md @@ -28,9 +28,3 @@ Auto-scaling lets you scale compute bounds in response to real-time resource dem To disable auto-scaling policies or switch back to static scaling, click **Delete Policy** in the scaling panel. Confirm the change in the custom Radix confirmation dialog to update the scaling configuration safely. -
- - Next: Managed Databases - - -
diff --git a/apps/docs/src/content/docs/ssl.md b/apps/docs/src/content/docs/ssl.md index d79d3ff..07724db 100644 --- a/apps/docs/src/content/docs/ssl.md +++ b/apps/docs/src/content/docs/ssl.md @@ -32,8 +32,3 @@ If your domain status is stuck on `PENDING_SSL`: - Verify that your DNS CNAME/A records have fully propagated. - Ensure that no CDN service (e.g. Cloudflare proxy) is blocking HTTP ACME challenge endpoints (`/.well-known/acme-challenge/`). -
- - Back to Intro - -
diff --git a/apps/docs/src/content/docs/system-config.md b/apps/docs/src/content/docs/system-config.md index 989c4af..5851c4f 100644 --- a/apps/docs/src/content/docs/system-config.md +++ b/apps/docs/src/content/docs/system-config.md @@ -106,3 +106,4 @@ Config file equivalent: "envEncryptionKey": "your-secure-key-here" } ``` + diff --git a/apps/docs/src/content/docs/volumes.md b/apps/docs/src/content/docs/volumes.md index 304c424..7583e1b 100644 --- a/apps/docs/src/content/docs/volumes.md +++ b/apps/docs/src/content/docs/volumes.md @@ -26,9 +26,3 @@ Under the **Volumes** tab, click **Add Volume**: Because data is tied directly to cluster block volumes, data persists even if you update environment variables, scale container replicas, or deploy code rollbacks. -
- - Next: Custom Domains - - -
diff --git a/apps/docs/src/layouts/Layout.astro b/apps/docs/src/layouts/Layout.astro index 41f0636..4156d78 100644 --- a/apps/docs/src/layouts/Layout.astro +++ b/apps/docs/src/layouts/Layout.astro @@ -34,6 +34,22 @@ const description = ""; const allDocs = await getCollection("docs"); +const orderedSlugs = [ + "docs", // Introduction + "installation", // Installation + "quickstart", // Quickstart Guide + "configuration", // Configuration + "deployments", // Deployments + "env-vars", // Environment Variables + "scaling", // Scaling Policies + "system-config", // System Configuration + "databases", // Managed Databases + "volumes", // Persistent Volumes + "auth", // Authentication & Access Control + "domains", // Custom Domains + "ssl", // SSL Certificates + "changelog", // Changelog +]; const categoryOrder = [ "Getting Started", "Core Architecture", @@ -51,6 +67,15 @@ for (const entry of allDocs) { slug: entry.data.slug, }); } + +for (const cat in grouped) { + grouped[cat].sort((a, b) => { + const indexA = orderedSlugs.indexOf(a.slug); + const indexB = orderedSlugs.indexOf(b.slug); + return indexA - indexB; + }); +} + const docMenu = categoryOrder .filter((cat) => grouped[cat]) .map((cat) => ({ title: cat, items: grouped[cat] })); @@ -477,6 +502,87 @@ const docMenu = categoryOrder } + + { + (() => { + const currentIndex = orderedSlugs.indexOf(currentSlug); + if (currentIndex === -1) return null; + + const prevSlug = currentIndex > 0 ? orderedSlugs[currentIndex - 1] : null; + const nextSlug = currentIndex < orderedSlugs.length - 1 ? orderedSlugs[currentIndex + 1] : null; + + const prevDoc = prevSlug ? allDocs.find((d) => d.data.slug === prevSlug) : null; + const nextDoc = nextSlug ? allDocs.find((d) => d.data.slug === nextSlug) : null; + + if (!prevDoc && !nextDoc) return null; + + return ( +
+ {prevDoc ? ( + + + + +
+ + Previous + + + {prevDoc.data.title} + +
+ + ) : ( + diff --git a/apps/docs/src/pages/docs/changelog.astro b/apps/docs/src/pages/docs/changelog.astro new file mode 100644 index 0000000..52cc31c --- /dev/null +++ b/apps/docs/src/pages/docs/changelog.astro @@ -0,0 +1,30 @@ +--- +import { getCollection, render } from 'astro:content'; +import Layout from '../../layouts/Layout.astro'; + +const changelogs = (await getCollection('changelogs')).sort((a, b) => + b.data.date.localeCompare(a.data.date) +); +--- + + +

Changelog

+

All notable changes to Dequel, tracked per release.

+ + {changelogs.map(async (entry) => { + const { Content } = await render(entry); + return ( +
+

{entry.data.version} — {entry.data.date}

+ +
+ ); + })} + + {!changelogs.length &&

No changelogs yet.

} + diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 8d3e9a8..dfc3ca2 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -1,5 +1,8 @@ FROM oven/bun:1 AS build +ARG DEQUEL_VERSION +ENV DEQUEL_VERSION=$DEQUEL_VERSION + WORKDIR /app COPY package.json ./ RUN bun install diff --git a/apps/web/package.json b/apps/web/package.json index bc71984..21ac8e9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "dequel-web", - "version": "0.1.0", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/api/client.ts b/apps/web/src/api/client.ts index 05e1c67..8438ae2 100644 --- a/apps/web/src/api/client.ts +++ b/apps/web/src/api/client.ts @@ -340,7 +340,23 @@ export const deleteScalingPolicy = ( // Server export const getServerIp = () => - apiFetch<{ ip: string }>("/server/ip"); + apiFetch<{ ip: string; baseDomain: string; resolves: boolean; url: string }>("/server/ip"); + +// Auth +export const login = (username: string, password: string) => + apiFetch<{ ok: boolean; username: string; error?: string }>("/auth/login", { + method: "POST", + body: JSON.stringify({ username, password }), + }); + +export const logout = () => + apiFetch<{ ok: boolean }>("/auth/logout", { method: "POST" }); + +export const refreshSession = () => + apiFetch<{ ok: boolean; username: string }>("/auth/refresh", { method: "POST" }); + +export const getMe = () => + apiFetch<{ authenticated: boolean; username?: string }>("/auth/me"); // Prometheus export const queryPrometheus = (query: string) => diff --git a/apps/web/src/components/ConfigWarnings.tsx b/apps/web/src/components/ConfigWarnings.tsx index 3113b87..7a0b4f3 100644 --- a/apps/web/src/components/ConfigWarnings.tsx +++ b/apps/web/src/components/ConfigWarnings.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; import { useQuery } from '@tanstack/react-query'; import { Button } from './ui/button'; -import { Mail, GitBranch, type LucideIcon } from 'lucide-react'; +import { Globe, Mail, GitBranch, type LucideIcon } from 'lucide-react'; import * as api from '../api/client'; export function ConfigWarnings() { @@ -13,6 +13,9 @@ export function ConfigWarnings() { const github = useQuery({ queryKey: ['github-integration'], queryFn: () => api.getGithubIntegration(), }); + const serverInfo = useQuery({ + queryKey: ['server-ip'], queryFn: () => api.getServerIp(), + }); const missing: { key: string; icon: LucideIcon; title: string; desc: string }[] = []; if (smtp.data && !smtp.data.configured) { @@ -31,6 +34,14 @@ export function ConfigWarnings() { desc: 'Connect a GitHub OAuth App to enable the repo picker and auto-deploy.', }); } + if (serverInfo.data && !serverInfo.data.resolves) { + missing.push({ + key: 'base-domain', + icon: Globe, + title: 'Base domain misconfigured', + desc: `The base domain ${serverInfo.data.baseDomain} does not resolve to this server (${serverInfo.data.ip}). Access the dashboard at ${serverInfo.data.url}.`, + }); + } if (missing.length === 0) return null; diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index 542aeae..b705b3b 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -11,8 +11,33 @@ import { NotificationBanner } from "./layout/NotificationBanner"; export function Layout({ children }: { children: React.ReactNode }) { const location = useLocation(); const navigate = useNavigate(); - const { data: projects = [] } = useProjects(); + + const { data: me, isLoading: authLoading } = useQuery({ + queryKey: ["auth", "me"], + queryFn: () => api.getMe(), + retry: false, + }); + + useEffect(() => { + if (authLoading) return; + if (location.pathname === "/login") { + if (me?.authenticated) { + navigate({ to: "/" }); + } + return; + } + if (!me?.authenticated) { + navigate({ to: "/login" }); + } + }, [me, authLoading, location.pathname, navigate]); + + const { data: projects = [] } = useProjects({ enabled: !!me?.authenticated && location.pathname !== "/login" }); const [projectSelectorOpen, setProjectSelectorOpen] = useState(false); + const [sidebarOpen, setSidebarOpen] = useState(false); + + useEffect(() => { + setSidebarOpen(false); + }, [location.pathname, location.search]); const { data: metricsText } = useQuery({ queryKey: ["metrics"], @@ -69,12 +94,18 @@ export function Layout({ children }: { children: React.ReactNode }) { return () => clearTimeout(t); }, [notification]); + const isLoginPage = location.pathname === "/login"; + const match = location.pathname.match(/\/project\/([^/]+)/); const currentProjectId = match ? match[1] : null; const currentProject = projects.find((p) => p.id === currentProjectId); + if (isLoginPage) { + return
{children}
; + } + return ( -
+
-
+
setNotification(null)} /> -
{children}
+
{children}
); diff --git a/apps/web/src/components/github/RepoPicker.tsx b/apps/web/src/components/github/RepoPicker.tsx index 02ac518..277f19f 100644 --- a/apps/web/src/components/github/RepoPicker.tsx +++ b/apps/web/src/components/github/RepoPicker.tsx @@ -38,6 +38,7 @@ export function RepoPicker({ const [webhookActive, setWebhookActive] = useState(false); const [webhookLoading, setWebhookLoading] = useState(false); const [webhookChecked, setWebhookChecked] = useState(false); + const [webhookError, setWebhookError] = useState(""); const fetchRepos = async () => { setLoading(true); @@ -68,7 +69,8 @@ export function RepoPicker({ try { const [owner, repo] = selected.fullName.split("/"); const hooks = await getRepoHooks(owner, repo); - setWebhookActive(hooks.length > 0); + const expectedUrl = `${window.location.origin}/api/github/webhook`; + setWebhookActive(hooks.some((h) => h.url === expectedUrl)); } catch { setWebhookActive(false); } finally { @@ -81,6 +83,7 @@ export function RepoPicker({ const toggleWebhook = async () => { if (!selected) return; setWebhookLoading(true); + setWebhookError(""); try { const [owner, repo] = selected.fullName.split("/"); if (webhookActive) { @@ -90,8 +93,9 @@ export function RepoPicker({ await registerRepoHook(owner, repo); setWebhookActive(true); } - } catch { - setWebhookActive(false); + } catch (err) { + const message = err instanceof Error ? err.message : "Failed to update webhook"; + setWebhookError(message.includes("Not authenticated") ? "GitHub session expired. Reconnect GitHub, then try again." : message); } finally { setWebhookLoading(false); } @@ -167,11 +171,14 @@ export function RepoPicker({ Enable Auto-Deploy )} - - )} -
- ); - } + + )} + {webhookError && ( +

{webhookError}

+ )} +
+ ); + } return (
diff --git a/apps/web/src/components/layout/Header.tsx b/apps/web/src/components/layout/Header.tsx index 7b686c5..cce94ce 100644 --- a/apps/web/src/components/layout/Header.tsx +++ b/apps/web/src/components/layout/Header.tsx @@ -1,19 +1,29 @@ import { Link } from "@tanstack/react-router"; -import { ChevronRight } from "lucide-react"; +import { ChevronRight, Menu } from "lucide-react"; interface HeaderProps { currentProject: { name: string } | undefined; currentProjectId: string | null; location: { pathname: string; search: any }; + setSidebarOpen: (open: boolean) => void; } export function Header({ currentProject, currentProjectId, location, + setSidebarOpen, }: HeaderProps) { return ( -
+
+ + Workspace diff --git a/apps/web/src/components/layout/NotificationBanner.tsx b/apps/web/src/components/layout/NotificationBanner.tsx index 35755d6..cb1c6ec 100644 --- a/apps/web/src/components/layout/NotificationBanner.tsx +++ b/apps/web/src/components/layout/NotificationBanner.tsx @@ -18,7 +18,7 @@ export function NotificationBanner({ const Icon = notification.type === "success" ? CheckCircle2 : AlertCircle; return ( -
+
void; + sidebarOpen: boolean; + setSidebarOpen: (open: boolean) => void; } export function Sidebar({ @@ -29,43 +33,70 @@ export function Sidebar({ metrics, location, navigate, + sidebarOpen, + setSidebarOpen, }: SidebarProps) { return ( - + ); } diff --git a/apps/web/src/components/project/deployments/DeploymentsTab.tsx b/apps/web/src/components/project/deployments/DeploymentsTab.tsx index 69bd4ee..6350c29 100644 --- a/apps/web/src/components/project/deployments/DeploymentsTab.tsx +++ b/apps/web/src/components/project/deployments/DeploymentsTab.tsx @@ -8,85 +8,17 @@ import { useCancelDeployment, useDeleteDeployment, } from "../../../hooks/useDeployments"; -import { - Dialog, - DialogContent, - DialogHeader, - DialogTitle, - DialogDescription, - DialogFooter, -} from "../../ui/dialog"; -import { useDeploymentLogs } from "../../../hooks/useDeploymentLogs"; -import { StatusBadge } from "../../StatusBadge"; +import { getRepoHooks, registerRepoHook, removeRepoHook } from "../../../api/client"; import { Button } from "../../ui/button"; import { Input } from "../../ui/input"; -import { Badge } from "../../ui/badge"; import { Card, CardContent, CardHeader, CardTitle } from "../../ui/card"; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "../../ui/table"; -import { Rocket, Play, RefreshCw, RotateCcw, Terminal, ChevronLeft, ChevronRight, History } from "lucide-react"; -import { cn } from "../../../lib/utils"; - -function formatTimeAgo(dateStr: string) { - const diff = - Date.now() - new Date(dateStr).getTime(); - const mins = Math.floor(diff / 60000); - if (mins < 1) return "just now"; - if (mins < 60) return `${mins}m ago`; - const hours = Math.floor(mins / 60); - if (hours < 24) return `${hours}h ago`; - return `${Math.floor(hours / 24)}d ago`; -} - -function parseTimestamp(raw: string) { - if (!raw) return Date.now(); - const normalized = raw.includes(" ") && !raw.includes("T") ? raw.replace(" ", "T") : raw; - const d = new Date(normalized); - return Number.isNaN(d.getTime()) ? Date.now() : d.getTime(); -} - -function DeploymentDuration({ deployment }: { deployment: any }) { - const [duration, setDuration] = useState(""); - - useEffect(() => { - const calculate = () => { - const start = parseTimestamp(deployment.createdAt); - const status = deployment.status; - const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; - const end = isFinished ? parseTimestamp(deployment.updatedAt) : Date.now(); - - const diff = Math.max(0, end - start); - const secs = Math.floor(diff / 1000); - if (secs < 60) { - setDuration(`${secs}s`); - } else { - const mins = Math.floor(secs / 60); - const remainingSecs = secs % 60; - setDuration(`${mins}m ${remainingSecs}s`); - } - }; - - calculate(); - - const status = deployment.status; - const isFinished = status !== "pending" && status !== "building" && status !== "deploying"; - if (isFinished) return; - - const interval = setInterval(calculate, 1000); - return () => clearInterval(interval); - }, [deployment.createdAt, deployment.updatedAt, deployment.status]); - - return {duration}; -} +import { Rocket, Play, Webhook } from "lucide-react"; +import { ManualDeployDialog } from "./manual-deploy-dialog"; +import { DeploymentHistory } from "./deployment-history"; +import { ClearCacheToggle } from "./clear-cache-toggle"; const PAGE_SIZE = 5; -function depDisplayName(projectName: string | undefined, depId: string) { - const short = depId.slice(0, 8); - if (!projectName) return short; - const slug = projectName.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63); - return `${slug}-${short}`; -} - interface DeploymentsTabProps { projectId: string; } @@ -106,12 +38,6 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { const redeploy = useRedeployDeployment(); const cancel = useCancelDeployment(); const deleteDep = useDeleteDeployment(); - const [deleteConfirmId, setDeleteConfirmId] = useState< - string | null - >(null); - const [cancelConfirmId, setCancelConfirmId] = useState< - string | null - >(null); const [selectedId, setSelectedId] = useState< string | null >(null); @@ -133,6 +59,60 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { const autoDeployedRef = useRef(false); + const [webhookActive, setWebhookActive] = useState(false); + const [webhookLoading, setWebhookLoading] = useState(false); + const [webhookChecked, setWebhookChecked] = useState(false); + const [webhookError, setWebhookError] = useState(null); + + useEffect(() => { + if (!project?.repoUrl) return; + const parseRepo = (url: string): { owner: string; repo: string } | null => { + const match = url.replace(/\.git$/, "").match(/github\.com\/([^/]+)\/([^/]+)/); + return match ? { owner: match[1], repo: match[2] } : null; + }; + const repo = parseRepo(project.repoUrl); + if (!repo) return; + setWebhookChecked(false); + let cancelled = false; + getRepoHooks(repo.owner, repo.repo) + .then((hooks) => { + if (!cancelled) { + const expectedUrl = `${window.location.origin}/api/github/webhook`; + setWebhookActive(hooks.some((h) => h.url === expectedUrl)); + } + }) + .catch(() => { + if (!cancelled) setWebhookActive(false); + }) + .finally(() => { + if (!cancelled) setWebhookChecked(true); + }); + return () => { cancelled = true; }; + }, [project?.repoUrl]); + + const toggleWebhook = async () => { + if (!project?.repoUrl) return; + const match = project.repoUrl.replace(/\.git$/, "").match(/github\.com\/([^/]+)\/([^/]+)/); + if (!match) return; + const [, owner, repo] = match; + setWebhookLoading(true); + setWebhookError(null); + try { + if (webhookActive) { + await removeRepoHook(owner, repo); + setWebhookActive(false); + } else { + await registerRepoHook(owner, repo); + setWebhookActive(true); + } + } catch (err) { + const message = err instanceof Error ? err.message : "Failed to update webhook"; + setWebhookError(message.includes("Not authenticated") ? "GitHub session expired. Reconnect GitHub in Settings, then try again." : message); + } finally { + setWebhookLoading(false); + } + }; + useEffect(() => { if (!project) return; setGitUrl(project.repoUrl ?? ""); @@ -246,9 +226,13 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { setShowGitSwitch(false); }; - const selectedDeployment = deployments.find( - (d) => d.id === selectedId, - ); + const [showManualDeployDialog, setShowManualDeployDialog] = useState(false); + + const handleManualDeploy = async (form: FormData) => { + form.set("sourceType", "git"); + if (projectId) form.set("projectId", projectId); + await createDeployment.mutateAsync(form); + }; return (
@@ -288,132 +272,153 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) {
)} -
-
- {( - [ - "git", - "upload", - "compose", - ] as const - ).map((type) => ( + {sourceType === "git" ? ( +
+
+
+
+
Repository URL
+
{project?.repoUrl || "No repository configured"}
+
+
+
Branch
+
+ {project?.repoBranch || "main"} +
+
+
+ {webhookError && ( +

{webhookError}

+ )} +
+ {webhookChecked && ( + + )} - ))} +
+
- {sourceType === "git" ? ( + ) : ( +
- - setGitUrl( - e - .target - .value, - ) - } - className="flex-1" - disabled - /> - - setBranch( - e - .target - .value, - ) - } - className="w-32" - disabled - /> + {( + [ + "git", + "upload", + "compose", + ] as const + ).map((type) => ( + + ))}
- ) : ( - )} -
- - setEnvironment( - e.target - .value, - ) - } - className="flex-1" + - -
- {!canEditSource && ( -
- Source type locked - after first - deploy. Use Switch - to Git to change - source. +
+ + setEnvironment( + e.target + .value, + ) + } + className="flex-1" + /> +
- )} - + {!canEditSource && ( +
+ Source type locked + after first + deploy. Use Switch + to Git to change + source. +
+ )} + + )} @@ -486,342 +491,34 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { )} {totalDeployments > 0 && ( -
- - - - - Status - - - Deployment - - - Source - - - Branch - - - Duration - - - Age - - - - - - {deployments.map( - (dep) => ( - - setSelectedId( - selectedId === - dep.id - ? null - : dep.id, - ) - } - > - -
- - {dep.status === "running" && ( - - Active - - )} -
- - - {depDisplayName( - project?.name, - dep.id, - )} - - - { - dep.sourceType - } - - - {dep.branch ? ( - - { - dep.branch - } - - ) : ( - - — - - )} - - - - - - {formatTimeAgo( - dep.createdAt, - )} - - -
- {dep.status === "running" && dep.imageTag && dep.sourceType !== "image" && ( - - )} - {(dep.status === "pending" || dep.status === "building") && ( - - )} - {dep.imageTag && dep.status !== "running" && dep.status !== "pending" && dep.status !== "building" && dep.status !== "deploying" && ( - - )} - {dep.status !== "running" && ( - - )} -
- - - ), - )} - -
- {totalPages > 1 && ( -
- - {totalDeployments} total deployments - -
- - - {page + 1} / {totalPages} - - -
-
- )} -
- )} - - {selectedDeployment && ( - redeploy.mutate(id)} + onRollback={(id) => rollback.mutate(id)} + onCancel={(id) => cancel.mutate(id)} + onDelete={(id) => deleteDep.mutate(id)} /> )} - { if (!open) setCancelConfirmId(null); }}> - - - Cancel deployment? - - This will stop the current build/deploy process. The deployment will be marked as failed. This cannot be undone. - - - - - - - - - - { if (!open) setDeleteConfirmId(null); }}> - - - Delete deployment? - - This will remove the deployment record, its build logs, and stop its container. The Docker image is kept for potential rollback. This action cannot be undone. - - - - - - - - +
); } -function fmtLogTs(raw: string | undefined) { - if (!raw) return ""; - const d = new Date(raw); - if (Number.isNaN(d.getTime())) return raw; - const pad = (n: number) => String(n).padStart(2, "0"); - return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; -} -function DeploymentLogs({ - deployment, -}: { - deployment: any; -}) { - const { logs, isLoading } = useDeploymentLogs( - deployment.id, - ); - const endRef = useRef(null); - useEffect(() => { - endRef.current?.scrollIntoView({ - behavior: "smooth", - }); - }, [logs]); - - return ( - - - - - - Build Logs —{" "} - {deployment.id.slice(0, 8)} - - - Duration: - - - - - - {isLoading ? ( -
- Loading logs... -
- ) : logs.length === 0 ? ( -
- No build logs available - for this deployment. -
- ) : ( -
- {logs.map((log, i) => ( -
- - [{log.stage}]-[{fmtLogTs((log as any).timestamp || log.createdAt)}] - - - {log.message} - -
- ))} -
-
- )} - - - ); -} diff --git a/apps/web/src/components/project/deployments/clear-cache-toggle.tsx b/apps/web/src/components/project/deployments/clear-cache-toggle.tsx new file mode 100644 index 0000000..584eee5 --- /dev/null +++ b/apps/web/src/components/project/deployments/clear-cache-toggle.tsx @@ -0,0 +1,37 @@ +interface ClearCacheToggleProps { + checked: boolean; + onChange: (checked: boolean) => void; + id?: string; + label?: string; + description?: string; +} + +export function ClearCacheToggle({ + checked, + onChange, + id = "clearCache", + label = "Clear build cache for this deployment", + description, +}: ClearCacheToggleProps) { + return ( +
+ onChange(e.target.checked)} + className="h-4 w-4 rounded border-zinc-700 bg-zinc-800 text-amber-500 focus:ring-amber-500 focus:ring-offset-zinc-950 cursor-pointer" + /> +
onChange(!checked)}> + + {description && ( + + {description} + + )} +
+
+ ); +} diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx new file mode 100644 index 0000000..d3e8d3c --- /dev/null +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -0,0 +1,221 @@ +import React, { useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "../../ui/dialog"; +import { StatusBadge } from "../../StatusBadge"; +import { Button } from "../../ui/button"; +import { Badge } from "../../ui/badge"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "../../ui/table"; +import { RefreshCw, ChevronLeft, ChevronRight, History } from "lucide-react"; +import { cn } from "../../../lib/utils"; +import { formatTimeAgo, depDisplayName, DeploymentDuration, DeploymentLogs } from "./deployment-logs"; + +interface DeploymentHistoryProps { + deployments: any[]; + page: number; + totalPages: number; + totalDeployments: number; + projectName: string | undefined; + onPageChange: (page: number) => void; + onSelect: (id: string | null) => void; + selectedId: string | null; + onRedeploy: (id: string) => void; + onRollback: (id: string) => void; + onCancel: (id: string) => void; + onDelete: (id: string) => void; +} + +export function DeploymentHistory({ + deployments, + page, + totalPages, + totalDeployments, + projectName, + onPageChange, + onSelect, + selectedId, + onRedeploy, + onRollback, + onCancel, + onDelete, +}: DeploymentHistoryProps) { + const [deleteConfirmId, setDeleteConfirmId] = useState(null); + const [cancelConfirmId, setCancelConfirmId] = useState(null); + + const selectedDeployment = deployments.find((d) => d.id === selectedId); + + return ( + <> +
+ + + + Status + Deployment + Source + Branch + Duration + Age + + + + + {deployments.map((dep) => ( + + onSelect(selectedId === dep.id ? null : dep.id) + } + > + +
+ + {dep.status === "running" && ( + + Active + + )} +
+ + + {depDisplayName(projectName, dep.id)} + + + {dep.sourceType} + + + {dep.branch ? ( + {dep.branch} + ) : ( + — + )} + + + + + + {formatTimeAgo(dep.createdAt)} + + +
+ {dep.status === "running" && dep.imageTag && dep.sourceType !== "image" && ( + + )} + {(dep.status === "pending" || dep.status === "building") && ( + + )} + {dep.imageTag && dep.status !== "running" && dep.status !== "pending" && dep.status !== "building" && dep.status !== "deploying" && ( + + )} + {dep.status !== "running" && ( + + )} +
+ + + ))} + +
+ {totalPages > 1 && ( +
+ + {totalDeployments} total deployments + +
+ + + {page + 1} / {totalPages} + + +
+
+ )} +
+ + {selectedDeployment && } + + { if (!open) setCancelConfirmId(null); }}> + + + Cancel deployment? + + This will stop the current build/deploy process. The deployment will be marked as failed. This cannot be undone. + + + + + + + + + + { if (!open) setDeleteConfirmId(null); }}> + + + Delete deployment? + + This will remove the deployment record, its build logs, and stop its container. The Docker image is kept for potential rollback. This action cannot be undone. + + + + + + + + + + ); +} diff --git a/apps/web/src/components/project/deployments/deployment-logs.tsx b/apps/web/src/components/project/deployments/deployment-logs.tsx new file mode 100644 index 0000000..fe5396a --- /dev/null +++ b/apps/web/src/components/project/deployments/deployment-logs.tsx @@ -0,0 +1,203 @@ +import React, { + useState, + useEffect, + useRef, +} from "react"; +import { useDeploymentLogs } from "../../../hooks/useDeploymentLogs"; +import { + Card, + CardContent, + CardHeader, + CardTitle, +} from "../../ui/card"; +import { Terminal } from "lucide-react"; + +export function formatTimeAgo(dateStr: string) { + const diff = + Date.now() - new Date(dateStr).getTime(); + const mins = Math.floor(diff / 60000); + if (mins < 1) return "just now"; + if (mins < 60) return `${mins}m ago`; + const hours = Math.floor(mins / 60); + if (hours < 24) return `${hours}h ago`; + return `${Math.floor(hours / 24)}d ago`; +} + +export function parseTimestamp(raw: string) { + if (!raw) return Date.now(); + const normalized = + raw.includes(" ") && !raw.includes("T") + ? raw.replace(" ", "T") + : raw; + const d = new Date(normalized); + return Number.isNaN(d.getTime()) + ? Date.now() + : d.getTime(); +} + +export function depDisplayName( + projectName: string | undefined, + depId: string, +) { + const short = depId.slice(0, 8); + if (!projectName) return short; + const slug = projectName + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63); + return `${slug}-${short}`; +} + +export function DeploymentDuration({ + deployment, +}: { + deployment: any; +}) { + const [duration, setDuration] = useState(""); + + useEffect(() => { + const calculate = () => { + const start = parseTimestamp( + deployment.createdAt, + ); + const status = deployment.status; + const isFinished = + status !== "pending" && + status !== "building" && + status !== "deploying"; + const end = isFinished + ? parseTimestamp( + deployment.updatedAt, + ) + : Date.now(); + + const diff = Math.max(0, end - start); + const secs = Math.floor(diff / 1000); + if (secs < 60) { + setDuration(`${secs}s`); + } else { + const mins = Math.floor( + secs / 60, + ); + const remainingSecs = secs % 60; + setDuration( + `${mins}m ${remainingSecs}s`, + ); + } + }; + + calculate(); + + const status = deployment.status; + const isFinished = + status !== "pending" && + status !== "building" && + status !== "deploying"; + if (isFinished) return; + + const interval = setInterval( + calculate, + 1000, + ); + return () => clearInterval(interval); + }, [ + deployment.createdAt, + deployment.updatedAt, + deployment.status, + ]); + + return ( + + {duration} + + ); +} + +function fmtLogTs(raw: string | undefined) { + if (!raw) return ""; + const d = new Date(raw); + if (Number.isNaN(d.getTime())) return raw; + const pad = (n: number) => + String(n).padStart(2, "0"); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; +} + +export function DeploymentLogs({ + deployment, +}: { + deployment: any; +}) { + const { logs, isLoading } = useDeploymentLogs( + deployment.id, + ); + const endRef = useRef(null); + useEffect(() => { + endRef.current?.scrollIntoView({ + behavior: "smooth", + }); + }, [logs]); + + return ( + + + + + + Build Logs —{" "} + {deployment.id.slice( + 0, + 8, + )} + + + Duration: + + + + + + {isLoading ? ( +
+ Loading logs... +
+ ) : logs.length === 0 ? ( +
+ No build logs available + for this deployment. +
+ ) : ( +
+ {logs.map((log, i) => ( +
+ + [{log.stage} + ]-[ + {fmtLogTs( + ( + log as any + ) + .timestamp || + log.createdAt, + )} + ] + + + {log.message} + +
+ ))} +
+
+ )} + + + ); +} diff --git a/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx b/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx new file mode 100644 index 0000000..aa7ce6f --- /dev/null +++ b/apps/web/src/components/project/deployments/manual-deploy-dialog.tsx @@ -0,0 +1,163 @@ +import React, { useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "../../ui/dialog"; +import { Button } from "../../ui/button"; +import { Input } from "../../ui/input"; +import { Rocket } from "lucide-react"; +import { cn } from "../../../lib/utils"; +import { ClearCacheToggle } from "./clear-cache-toggle"; + +interface ManualDeployDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + projectId: string; + projectName: string; + repoUrl: string | null | undefined; + repoBranch: string | null | undefined; + isPending: boolean; + onDeploy: (form: FormData) => Promise; +} + +export function ManualDeployDialog({ + open, + onOpenChange, + projectName, + repoUrl, + repoBranch, + isPending, + onDeploy, +}: ManualDeployDialogProps) { + const [deployOption, setDeployOption] = useState<"latest" | "commit">("latest"); + const [commitSha, setCommitSha] = useState(""); + const [clearCache, setClearCache] = useState(false); + const [error, setError] = useState(null); + + const handleDeploy = async () => { + const form = new FormData(); + form.set("sourceType", "git"); + if (repoUrl) form.set("gitUrl", repoUrl); + if (repoBranch) form.set("branch", repoBranch); + + if (deployOption === "commit") { + if (!commitSha.trim()) return; + form.set("commitSha", commitSha.trim()); + } + + if (clearCache) { + form.set("clearCache", "true"); + } + + try { + await onDeploy(form); + setDeployOption("latest"); + setCommitSha(""); + setClearCache(false); + setError(null); + onOpenChange(false); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to start deployment"); + } + }; + + return ( + { if (!v) setError(null); onOpenChange(v); }}> + + + + + Manual Deployment + + + Build and deploy a new version of {projectName}. + + + +
+
+ +
+ + +
+
+ + {deployOption === "commit" && ( +
+ + setCommitSha(e.target.value)} + className="bg-[#121215] border-[#222227] text-zinc-200 focus:border-amber-500 text-xs font-mono h-9" + /> +
+ )} + + +
+ + {error && ( +
+
+ {error} +
+
+ )} + + + + + + + ); +} diff --git a/apps/web/src/components/project/domains/DomainsTab.tsx b/apps/web/src/components/project/domains/DomainsTab.tsx index 3cef41e..c2ab828 100644 --- a/apps/web/src/components/project/domains/DomainsTab.tsx +++ b/apps/web/src/components/project/domains/DomainsTab.tsx @@ -366,8 +366,8 @@ export function DomainsTab({ )} -
- +
+
diff --git a/apps/web/src/components/project/envtab/EnvVarTable.tsx b/apps/web/src/components/project/envtab/EnvVarTable.tsx index f6c4314..2c5dd58 100644 --- a/apps/web/src/components/project/envtab/EnvVarTable.tsx +++ b/apps/web/src/components/project/envtab/EnvVarTable.tsx @@ -77,8 +77,8 @@ export function EnvVarTable({ -
-
+
+
diff --git a/apps/web/src/components/project/volumes/VolumesTab.tsx b/apps/web/src/components/project/volumes/VolumesTab.tsx index d269a75..f67d4d4 100644 --- a/apps/web/src/components/project/volumes/VolumesTab.tsx +++ b/apps/web/src/components/project/volumes/VolumesTab.tsx @@ -233,8 +233,8 @@ export function VolumesTab({ -
-
+
+
diff --git a/apps/web/src/hooks/useProjects.ts b/apps/web/src/hooks/useProjects.ts index 23bd2ac..55bf629 100644 --- a/apps/web/src/hooks/useProjects.ts +++ b/apps/web/src/hooks/useProjects.ts @@ -2,8 +2,8 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { listProjects, createProject, deleteProject } from '../api/client'; import type { Project } from '../types'; -export function useProjects() { - return useQuery({ queryKey: ['projects'], queryFn: listProjects, refetchInterval: 10_000 }); +export function useProjects(options?: { enabled?: boolean }) { + return useQuery({ queryKey: ['projects'], queryFn: listProjects, refetchInterval: 10_000, ...options }); } export function useProject(id: string) { diff --git a/apps/web/src/index.css b/apps/web/src/index.css index 259e045..70d0249 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -78,6 +78,11 @@ opacity: 0.4; } +.log-line.error { + color: hsl(0 72% 56%); + opacity: 0.7; +} + .log-stage { color: hsl(24 95% 53%); flex-shrink: 0; @@ -152,6 +157,10 @@ .floating-chat-kofi-popup-iframe, .floating-chat-kofi-popup-iframe-mobi { position: fixed !important; + left: 16px !important; + bottom: 80px !important; + top: auto !important; + right: auto !important; z-index: 999999 !important; } diff --git a/apps/web/src/routes/Login.tsx b/apps/web/src/routes/Login.tsx new file mode 100644 index 0000000..094b918 --- /dev/null +++ b/apps/web/src/routes/Login.tsx @@ -0,0 +1,247 @@ +import { useState, type FormEvent } from 'react'; +import * as api from '../api/client'; +import { DequelLogo } from '../components/DequelLogo'; +import { Lock, User, Terminal, ArrowRight, Shield, Activity, RefreshCw } from 'lucide-react'; + +export function Login() { + const [username, setUsername] = useState(''); + const [password, setPassword] = useState(''); + const [error, setError] = useState(''); + const [loading, setLoading] = useState(false); + + const handleSubmit = async (e: FormEvent) => { + e.preventDefault(); + setError(''); + setLoading(true); + try { + const res = await api.login(username, password); + if (res.ok) { + window.location.href = '/'; + } else { + setError(res.error || 'Login failed'); + } + } catch (err: any) { + setError(err.message || 'Login failed'); + } finally { + setLoading(false); + } + }; + + return ( +
+ {/* Background glow using Dequel orange */} +
+ + {/* Center content container for closer alignment */} +
+ + {/* Left Column: Login Card & Heading */} +
+ {/* Logo Branding */} +
+ + + dequel + + + v{__DEQUEL_VERSION__} + +
+ + {/* Form */} +
+
+

+ Sign in to Dequel +

+

+ Enter your credentials to access your self-hosted deployment engine. +

+
+ +
+
+ + + Security Gateway + + + + Active + +
+ +
+
+ +
+ + setUsername(e.target.value)} + className="w-full pl-9 pr-4 py-2 rounded-lg border border-[#222227] bg-[#121214] text-zinc-200 text-xs placeholder-zinc-700 focus:outline-none focus:border-orange-500 focus:ring-1 focus:ring-orange-500/20 transition-all" + placeholder="Linux username" + autoFocus + required + /> +
+
+ +
+ +
+ + setPassword(e.target.value)} + className="w-full pl-9 pr-4 py-2 rounded-lg border border-[#222227] bg-[#121214] text-zinc-200 text-xs placeholder-zinc-700 focus:outline-none focus:border-orange-500 focus:ring-1 focus:ring-orange-500/20 transition-all" + placeholder="Linux password" + required + /> +
+
+ + {error && ( +

+ {error} +

+ )} + + + +
+
+ + {/* Footer Info */} +
+ © {new Date().getFullYear()} Dequel. +
+
+ + {/* Right Column: Clean, Low-contrast CSS Mockup of Dequel Dashboard */} +
+ {/* Header Mockup */} +
+
+ + + +
+
+ dequel.local/dashboard +
+
+
+ + {/* Content Layout Mockup */} +
+ {/* Sidebar Mockup */} +
+
+
+ + dequel +
+ +
+
+ + Overview +
+
+ + Logs +
+
+
+ + {/* Sidebar Footer Widget Mockup */} +
+
+ STATUS + + + Live + +
+
+ Services + 3 +
+
+
+ + {/* Main Area Mockup */} +
+
+
+

Overview

+

Manage and monitor cluster resources.

+
+
+ + New Project +
+
+ + {/* Metric stats card */} +
+
+
+
API Traffic
+
148 reqs
+
+ +
+ +
+
+
Deployments
+
3 active
+
+ +
+
+ + {/* Projects list */} +
+
+
+
+ A +
+
+
api-service
+
github.com/lftobs/api
+
+
+ + running + +
+
+
+
+
+ +
+
+ ); +} diff --git a/apps/web/src/routes/ProjectDetail.tsx b/apps/web/src/routes/ProjectDetail.tsx index f1e69db..79625f9 100644 --- a/apps/web/src/routes/ProjectDetail.tsx +++ b/apps/web/src/routes/ProjectDetail.tsx @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useEffect, useRef } from "react"; import { useProject } from "../hooks/useProjects"; import { useDeployments } from "../hooks/useDeployments"; import { useNavigate, useLocation } from "@tanstack/react-router"; @@ -32,6 +32,17 @@ export function ProjectDetail({ const navigate = useNavigate(); const location = useLocation(); const activeTab = new URLSearchParams(location.search).get("tab") || "deployments"; + const tabsListRef = useRef(null); + + useEffect(() => { + if (!tabsListRef.current) return; + const activeTrigger = tabsListRef.current.querySelector( + `[data-value="${activeTab}"]`, + ) as HTMLElement | null; + if (activeTrigger) { + activeTrigger.scrollIntoView({ behavior: "smooth", block: "nearest", inline: "center" }); + } + }, [activeTab]); if (isLoading) return ( @@ -106,32 +117,32 @@ export function ProjectDetail({
navigate({ search: { tab: val } as any })} className="space-y-4"> - - + + Deployments - + Env Vars - + Volumes - + Databases - + Domains - + Scaling - + Alerts - + Observability - + Logs diff --git a/apps/web/src/routes/Settings.tsx b/apps/web/src/routes/Settings.tsx index fb16b15..466b44c 100644 --- a/apps/web/src/routes/Settings.tsx +++ b/apps/web/src/routes/Settings.tsx @@ -72,8 +72,8 @@ function ServersSection() { {servers.length > 0 && ( -
-
+
+
NameHostStatus @@ -322,7 +322,7 @@ function ApiKeysSection() { {newKey} )} - +
setName(e.target.value)} className="w-56" /> @@ -330,8 +330,8 @@ function ApiKeysSection() { {apiKeys.length > 0 && ( -
-
+
+
NameKey HashCreated diff --git a/apps/web/src/routes/index.tsx b/apps/web/src/routes/index.tsx index e33483f..ffbfab3 100644 --- a/apps/web/src/routes/index.tsx +++ b/apps/web/src/routes/index.tsx @@ -1,6 +1,7 @@ import { createRootRoute, createRoute, createRouter, Outlet } from '@tanstack/react-router'; import { Layout } from '../components/Layout'; import { Dashboard } from './Dashboard'; +import { Login } from './Login'; import { Settings } from './Settings'; import { ProjectDetail } from './ProjectDetail'; @@ -18,6 +19,12 @@ const indexRoute = createRoute({ component: Dashboard, }); +const loginRoute = createRoute({ + getParentRoute: () => rootRoute, + path: '/login', + component: Login, +}); + const settingsRoute = createRoute({ getParentRoute: () => rootRoute, path: '/settings', @@ -38,7 +45,7 @@ const projectRoute = createRoute({ }), }); -const routeTree = rootRoute.addChildren([indexRoute, settingsRoute, projectRoute]); +const routeTree = rootRoute.addChildren([indexRoute, loginRoute, settingsRoute, projectRoute]); export const router = createRouter({ routeTree }); diff --git a/apps/web/vite-env.d.ts b/apps/web/vite-env.d.ts new file mode 100644 index 0000000..38b89f1 --- /dev/null +++ b/apps/web/vite-env.d.ts @@ -0,0 +1 @@ +declare const __DEQUEL_VERSION__: string; diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 95d423e..67b345d 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -1,9 +1,16 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; import path from 'path'; +import fs from 'fs'; + +const versionPath = [path.resolve(__dirname, 'VERSION'), path.resolve(__dirname, '../../VERSION')].find(fs.existsSync); +const version = process.env.DEQUEL_VERSION || (versionPath ? fs.readFileSync(versionPath, 'utf-8').trim() : '0.0.0'); export default defineConfig({ plugins: [react()], + define: { + __DEQUEL_VERSION__: JSON.stringify(version), + }, resolve: { alias: { '@': path.resolve(__dirname, './src'), diff --git a/docker-compose.yml b/docker-compose.yml index df049a8..e52127b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: buildkit: image: moby/buildkit:latest + restart: unless-stopped command: [ "--addr", @@ -18,11 +19,8 @@ services: healthcheck: test: [ - "CMD", - "wget", - "--spider", - "-q", - "tcp://localhost:1234", + "CMD-SHELL", + "wget -q -T 2 -O /dev/null http://localhost:1234 >/dev/null 2>&1; [ $$? -ne 4 ]", ] interval: 5s timeout: 3s @@ -31,8 +29,10 @@ services: api: image: ghcr.io/lftobs/dequel/api:latest + restart: unless-stopped # build: - # context: ./apps/api + # context: . + # dockerfile: ./apps/api/Dockerfile environment: PORT: 3001 DATABASE_PATH: /app/data/dequel.db @@ -45,11 +45,15 @@ services: DOCKER_BIN: /usr/bin/docker RAILPACK_VERBOSE: "1" RAILPACK_BUILD_TIMEOUT_MS: "1200000" + GRAFANA_URL: http://grafana:3000/grafana volumes: - ./data:/app/data - ./workspace:/app/workspace - ./infra/caddy/routes:/caddy/routes - /var/run/docker.sock:/var/run/docker.sock + - railpack-cache:/tmp/railpack + - /etc/passwd:/etc/passwd:ro + - /etc/group:/etc/group:ro depends_on: buildkit: condition: service_started @@ -70,10 +74,44 @@ services: aliases: - api + pam-auth: + image: python:3-slim + restart: unless-stopped + entrypoint: [] + command: + - sh + - -c + - | + apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq libpam-modules > /dev/null 2>&1 && exec python3 -u /app/pam-server.py + volumes: + - ./scripts/auth/pam-server.py:/app/pam-server.py:ro + - /etc/passwd:/etc/passwd:ro + - /etc/shadow:/etc/shadow:ro + - /etc/group:/etc/group:ro + networks: + dequel: + aliases: + - pam-auth + healthcheck: + test: + [ + "CMD", + "python3", + "-c", + "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:4567/health').status == 200 else 1)", + ] + interval: 5s + timeout: 3s + retries: 5 + start_period: 10s + web: image: ghcr.io/lftobs/dequel/web:latest + restart: unless-stopped # build: # context: ./apps/web + # args: + # DEQUEL_VERSION: ${DEQUEL_VERSION:-} healthcheck: test: [ @@ -93,6 +131,7 @@ services: caddy: image: caddy:2.8-alpine + restart: unless-stopped command: [ "caddy", @@ -104,7 +143,7 @@ services: "--watch", ] environment: - CADDY_EMAIL: ${CADDY_EMAIL:-} + CADDY_EMAIL: ${CADDY_EMAIL:-admin@dequel.local} CADDY_BASE_DOMAIN: ${CADDY_BASE_DOMAIN:-localhost} ports: - "80:80" @@ -122,6 +161,7 @@ services: redis: image: redis:7-alpine + restart: unless-stopped command: redis-server --appendonly yes volumes: - redis-data:/data @@ -137,6 +177,7 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:latest + restart: unless-stopped volumes: - /:/rootfs:ro - /var/run:/var/run:ro @@ -166,13 +207,14 @@ services: prometheus: image: prom/prometheus:latest + restart: unless-stopped + stop_grace_period: 60s + entrypoint: [] command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.path=/prometheus - - --web.console.libraries=/usr/share/prometheus/console_libraries - - --web.console.templates=/usr/share/prometheus/consoles - - --storage.tsdb.retention.time=30d + - sh + - /entrypoint.sh volumes: + - ./infra/monitoring/prometheus-entrypoint.sh:/entrypoint.sh:ro - ./infra/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus depends_on: @@ -198,6 +240,7 @@ services: loki: image: grafana/loki:3.0.0 + restart: unless-stopped command: - -config.file=/etc/loki/loki-config.yml volumes: @@ -223,12 +266,14 @@ services: promtail: image: grafana/promtail:3.0.0 + restart: unless-stopped command: - -config.file=/etc/promtail/promtail-config.yml volumes: - ./infra/monitoring/promtail-config.yml:/etc/promtail/promtail-config.yml:ro - /var/run/docker.sock:/var/run/docker.sock - /var/lib/docker/containers:/var/lib/docker/containers:ro + - promtail-data:/data depends_on: loki: condition: service_started @@ -237,6 +282,7 @@ services: grafana: image: grafana/grafana:latest + restart: unless-stopped environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin @@ -245,6 +291,7 @@ services: GF_INSTALL_PLUGINS: "" volumes: - ./infra/monitoring/grafana/datasources:/etc/grafana/provisioning/datasources:ro + - ./infra/monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro - grafana-data:/var/lib/grafana depends_on: prometheus: @@ -279,3 +326,5 @@ volumes: prometheus-data: loki-data: grafana-data: + railpack-cache: + promtail-data: diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index 6b34c41..4cd18df 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -1,5 +1,5 @@ { - email {$CADDY_EMAIL} + email {$CADDY_EMAIL:admin@dequel.local} } import /etc/caddy/routes/*.caddy @@ -13,7 +13,9 @@ import /etc/caddy/routes/*.caddy encode zstd gzip handle /api/* { - reverse_proxy api:3001 + reverse_proxy api:3001 { + trusted_proxies private_ranges + } } handle /metrics* { @@ -40,7 +42,9 @@ import /etc/caddy/routes/*.caddy encode zstd gzip handle /api/* { - reverse_proxy api:3001 + reverse_proxy api:3001 { + trusted_proxies private_ranges + } } handle /metrics* { diff --git a/infra/monitoring/grafana/dashboards/dashboards.yml b/infra/monitoring/grafana/dashboards/dashboards.yml new file mode 100644 index 0000000..dd16c25 --- /dev/null +++ b/infra/monitoring/grafana/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "Dequel" + orgId: 1 + folder: "" + type: file + disableDeletion: true + editable: false + options: + path: /etc/grafana/provisioning/dashboards diff --git a/infra/monitoring/grafana/dashboards/deployed-apps.json b/infra/monitoring/grafana/dashboards/deployed-apps.json new file mode 100644 index 0000000..9fd599f --- /dev/null +++ b/infra/monitoring/grafana/dashboards/deployed-apps.json @@ -0,0 +1,513 @@ +{ + "title": "Dequel — System & Apps Overview", + "uid": "dequel-deployed-apps", + "tags": ["dequel"], + "schemaVersion": 39, + "version": 2, + "timezone": "browser", + "refresh": "10s", + "templating": { + "list": [ + { + "name": "container", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(container_cpu_usage_seconds_total{job=\"cadvisor\"}, name)", + "refId": "container-var" + }, + "regex": "/([^/]+$)", + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "refresh": 1, + "hide": 0, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Key Metrics", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "id": 1, + "type": "gauge", + "title": "CPU Busy", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 85 } + ] + } + } + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[5m])) * 100 / max(machine_cpu_cores) or sum(rate(container_cpu_usage_seconds_total{name=~\".*\"}[5m])) * 100 / max(machine_cpu_cores)", + "refId": "A" + } + ] + }, + { + "id": 2, + "type": "gauge", + "title": "Used RAM Memory", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + } + } + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_working_set_bytes{id=\"/\"} * 100 / machine_memory_bytes or sum(container_memory_working_set_bytes{name=~\".*\"}) * 100 / max(machine_memory_bytes)", + "refId": "A" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Incoming Data (Total)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 4, "x": 12, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed" + }, + "fixedColor": "green" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_network_receive_bytes_total)", + "refId": "A" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Outgoing Data (Total)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 4, "x": 16, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed" + }, + "fixedColor": "green" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_network_transmit_bytes_total)", + "refId": "A" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Total Transferred Data", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 5, "w": 4, "x": 20, "y": 1 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed" + }, + "fixedColor": "green" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(container_network_receive_bytes_total) + sum(container_network_transmit_bytes_total)", + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "System Statistics / Ingress", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } + }, + { + "id": 6, + "type": "stat", + "title": "Bitrate Download", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "bps", + "color": { + "mode": "fixed" + }, + "fixedColor": "blue" + } + }, + "options": { + "graphMode": "line", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_network_receive_bytes_total[5m])) * 8", + "refId": "A" + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Bitrate Upload", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "bps", + "color": { + "mode": "fixed" + }, + "fixedColor": "blue" + } + }, + "options": { + "graphMode": "line", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(container_network_transmit_bytes_total[5m])) * 8", + "refId": "A" + } + ] + }, + { + "id": 8, + "type": "stat", + "title": "Running Projects", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "none", + "color": { + "mode": "fixed" + }, + "fixedColor": "orange" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "dequel_active_deployments", + "refId": "A" + } + ] + }, + { + "id": 9, + "type": "stat", + "title": "API Uptime", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "fixed" + }, + "fixedColor": "blue" + } + }, + "options": { + "graphMode": "none", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "dequel_uptime_seconds", + "refId": "A" + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "Total API Requests", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 7 }, + "fieldConfig": { + "defaults": { + "unit": "none", + "color": { + "mode": "fixed" + }, + "fixedColor": "purple" + } + }, + "options": { + "graphMode": "area", + "reduceOptions": { + "calcs": ["lastNotNull"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "dequel_requests_total", + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Resource History", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 } + }, + { + "id": 11, + "type": "timeseries", + "title": "CPU Usage (Cores)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 20, + "lineWidth": 1.5 + } + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(container_cpu_usage_seconds_total{name=~\".*${container:regex}$\"}[$__rate_interval])", + "legendFormat": "{{name}}", + "refId": "A" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Memory Usage", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 20, + "lineWidth": 1.5 + } + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "container_memory_working_set_bytes{name=~\".*${container:regex}$\"}", + "legendFormat": "{{name}}", + "refId": "A" + } + ] + }, + { + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 } + }, + { + "id": 13, + "type": "logs", + "title": "Container Logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 21 }, + "options": { + "showLabels": true, + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{container=~\"${container:regex}\"}", + "refId": "A" + } + ] + } + ] +} diff --git a/infra/monitoring/grafana/datasources/loki.yml b/infra/monitoring/grafana/datasources/loki.yml index 67ce8b7..9d8ab1e 100644 --- a/infra/monitoring/grafana/datasources/loki.yml +++ b/infra/monitoring/grafana/datasources/loki.yml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Loki + uid: loki type: loki access: proxy url: http://loki:3100 diff --git a/infra/monitoring/grafana/datasources/prometheus.yml b/infra/monitoring/grafana/datasources/prometheus.yml index bb009bb..00f9915 100644 --- a/infra/monitoring/grafana/datasources/prometheus.yml +++ b/infra/monitoring/grafana/datasources/prometheus.yml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Prometheus + uid: prometheus type: prometheus access: proxy url: http://prometheus:9090 diff --git a/infra/monitoring/prometheus-entrypoint.sh b/infra/monitoring/prometheus-entrypoint.sh new file mode 100755 index 0000000..f87ac6e --- /dev/null +++ b/infra/monitoring/prometheus-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +for block in /prometheus/01*/; do + if [ -d "$block" ]; then + missing="" + [ ! -f "$block/index" ] && missing="$missing index" + [ ! -f "$block/meta.json" ] && missing="$missing meta.json" + [ ! -d "$block/chunks" ] && missing="$missing chunks" + if [ -n "$missing" ]; then + echo "warning: corrupted block $(basename "$block") (missing:$missing), moving to quarantine" + mkdir -p /prometheus/quarantine + mv "$block" /prometheus/quarantine/ + fi + fi +done + +exec prometheus \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus \ + --web.console.libraries=/usr/share/prometheus/console_libraries \ + --web.console.templates=/usr/share/prometheus/consoles \ + --storage.tsdb.retention.time=30d \ + --storage.tsdb.wal-compression diff --git a/infra/monitoring/promtail-config.yml b/infra/monitoring/promtail-config.yml index 0bf72ff..e4f1be2 100644 --- a/infra/monitoring/promtail-config.yml +++ b/infra/monitoring/promtail-config.yml @@ -3,7 +3,7 @@ server: grpc_listen_port: 0 positions: - filename: /tmp/positions.yaml + filename: /data/positions.yaml clients: - url: http://loki:3100/loki/api/v1/push @@ -13,6 +13,9 @@ scrape_configs: docker_sd_configs: - host: unix:///var/run/docker.sock refresh_interval: 15s + filters: + - name: network + values: ["dequel_net"] relabel_configs: - source_labels: ['__meta_docker_container_name'] regex: '/(.*)' diff --git a/package.json b/package.json index abe5b72..3d67e7e 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,16 @@ { "name": "dequel", - "version": "0.1.0", "private": true, + "workspaces": [ + "apps/*" + ], "scripts": { + "tegami": "bun scripts/tegami.mts", + "release": "git tag v$(cat VERSION) && git push origin v$(cat VERSION)", "sync-versions": "node -e \"const v = require('fs').readFileSync('VERSION','utf8').trim(); ['apps/api/package.json','apps/web/package.json','apps/docs/package.json'].forEach(f => { const p = require('./'+f); p.version = v; require('fs').writeFileSync(f, JSON.stringify(p, null, 2) + '\\n'); }); console.log('Versions synced to', v);\"", "postinstall": "bun sync-versions" + }, + "devDependencies": { + "tegami": "^1.2.5" } } diff --git a/scripts/auth/pam-server.py b/scripts/auth/pam-server.py new file mode 100644 index 0000000..4bb96d8 --- /dev/null +++ b/scripts/auth/pam-server.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""HTTP sidecar for PAM authentication. Replaces direct /etc/shadow access from the API container.""" +import json +import os +import ctypes +import ctypes.util +import subprocess +from http.server import HTTPServer, BaseHTTPRequestHandler + +GRP_NAME = "dequel" +SERVICE = "dequel" + +PAM_PROMPT_ECHO_OFF = 1 +PAM_SUCCESS = 0 +PAM_END = -1 + + +class PamMessage(ctypes.Structure): + _fields_ = [("msg_style", ctypes.c_int), ("msg", ctypes.c_char_p)] + + +class PamResponse(ctypes.Structure): + _fields_ = [("resp", ctypes.c_void_p), ("resp_retcode", ctypes.c_int)] + + +CONV_FUNC = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.POINTER(PamMessage)), + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, +) + + +class PamConv(ctypes.Structure): + _fields_ = [("conv", CONV_FUNC), ("appdata_ptr", ctypes.c_void_p)] + + +def verify(username: str, password: str) -> dict: + lib_path = ctypes.util.find_library("pam") + if not lib_path: + return {"ok": False, "error": "PAM library not found on system"} + libpam = ctypes.cdll.LoadLibrary(lib_path) + libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c") or "libc.so.6") + libpam.pam_start.restype = ctypes.c_int + libpam.pam_authenticate.restype = ctypes.c_int + libpam.pam_acct_mgmt.restype = ctypes.c_int + libpam.pam_end.restype = ctypes.c_int + libpam.pam_strerror.restype = ctypes.c_char_p + libpam.pam_strerror.argtypes = [ctypes.c_void_p, ctypes.c_int] + libc.malloc.restype = ctypes.c_void_p + libc.strdup.restype = ctypes.c_void_p + + password_cpy = [password] + + def conv(nmsg, msg, out_resp, appdata): + count = nmsg + resp_size = ctypes.sizeof(PamResponse) * count + buf = libc.malloc(resp_size) + ctypes.memset(buf, 0, resp_size) + arr = ctypes.cast(buf, ctypes.POINTER(PamResponse)) + for i in range(count): + pm = ctypes.cast(msg[i], ctypes.POINTER(PamMessage))[0] + if pm.msg_style == PAM_PROMPT_ECHO_OFF: + pw = password_cpy[0] + pw_buf = libc.strdup(pw.encode()) + arr[i].resp = pw_buf + arr[i].resp_retcode = 0 + else: + arr[i].resp = None + arr[i].resp_retcode = PAM_END + out_resp[0] = buf + return PAM_SUCCESS + + cb = CONV_FUNC(conv) + conv_struct = PamConv(cb, None) + handle = ctypes.c_void_p() + + ret = libpam.pam_start(SERVICE.encode(), username.encode(), ctypes.byref(conv_struct), ctypes.byref(handle)) + if ret != PAM_SUCCESS: + err = libpam.pam_strerror(handle, ret) + return {"ok": False, "error": f"PAM start failed: {(err or b'').decode()}"} + + ret = libpam.pam_authenticate(handle, 0) + if ret != PAM_SUCCESS: + libpam.pam_end(handle, ret) + return {"ok": False, "error": "Authentication failed"} + + ret = libpam.pam_acct_mgmt(handle, 0) + libpam.pam_end(handle, ret) + if ret != PAM_SUCCESS: + return {"ok": False, "error": "Account expired or disabled"} + + result = subprocess.run( + ["getent", "group", GRP_NAME], + capture_output=True, text=True, timeout=10, + ) + if result.returncode != 0: + return {"ok": False, "error": f"Group '{GRP_NAME}' does not exist"} + + group_parts = result.stdout.strip().split(":") + members = group_parts[-1].split(",") if len(group_parts) >= 4 else [] + gid = group_parts[2] if len(group_parts) >= 3 else None + + if username in members: + return {"ok": True, "username": username} + + if gid: + pw_result = subprocess.run( + ["getent", "passwd", username], + capture_output=True, text=True, timeout=10, + ) + if pw_result.returncode == 0: + user_gid = pw_result.stdout.strip().split(":")[3] if ":" in pw_result.stdout else None + if user_gid == gid: + return {"ok": True, "username": username} + + return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} + + +class PamHandler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + except json.JSONDecodeError: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"error": "Invalid JSON"}).encode()) + return + + username = data.get("username", "").strip() + password = data.get("password", "") + + if not username or not password: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"error": "Username and password required"}).encode()) + return + + result = verify(username, password) + status = 200 if result.get("ok") else 401 + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(result).encode()) + + def do_GET(self): + if self.path == "/health": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"ok": True}).encode()) + return + self.send_response(404) + self.end_headers() + + +def main(): + port = int(os.environ.get("PORT", "4567")) + server = HTTPServer(("0.0.0.0", port), PamHandler) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/auth/pam-verify.py b/scripts/auth/pam-verify.py new file mode 100644 index 0000000..e7ca80b --- /dev/null +++ b/scripts/auth/pam-verify.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Authenticate a Linux user via PAM and check dequel group membership. + +Reads JSON { username, password } from stdin. +Exits 0 with { "ok": true, "username": "..." } on success. +Exits 1 with { "ok": false, "error": "..." } on failure. +""" +import json +import sys +import ctypes +import ctypes.util +import subprocess + +GRP_NAME = "dequel" +SERVICE = "dequel" + +PAM_PROMPT_ECHO_OFF = 1 +PAM_SUCCESS = 0 +PAM_END = -1 + + +class PamMessage(ctypes.Structure): + _fields_ = [("msg_style", ctypes.c_int), ("msg", ctypes.c_char_p)] + + +class PamResponse(ctypes.Structure): + _fields_ = [("resp", ctypes.c_void_p), ("resp_retcode", ctypes.c_int)] + + +CONV_FUNC = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.POINTER(PamMessage)), + ctypes.POINTER(ctypes.c_void_p), + ctypes.c_void_p, +) + + +class PamConv(ctypes.Structure): + _fields_ = [("conv", CONV_FUNC), ("appdata_ptr", ctypes.c_void_p)] + + +def verify(username: str, password: str) -> dict: + lib_path = ctypes.util.find_library("pam") + if not lib_path: + return {"ok": False, "error": "PAM library not found on system"} + libpam = ctypes.cdll.LoadLibrary(lib_path) + libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c") or "libc.so.6") + libpam.pam_start.restype = ctypes.c_int + libpam.pam_authenticate.restype = ctypes.c_int + libpam.pam_acct_mgmt.restype = ctypes.c_int + libpam.pam_end.restype = ctypes.c_int + libpam.pam_strerror.restype = ctypes.c_char_p + libpam.pam_strerror.argtypes = [ctypes.c_void_p, ctypes.c_int] + libc.malloc.restype = ctypes.c_void_p + libc.strdup.restype = ctypes.c_void_p + + password_cpy = [password] + + def conv(nmsg, msg, out_resp, appdata): + count = nmsg + resp_size = ctypes.sizeof(PamResponse) * count + buf = libc.malloc(resp_size) + ctypes.memset(buf, 0, resp_size) + arr = ctypes.cast(buf, ctypes.POINTER(PamResponse)) + for i in range(count): + pm = ctypes.cast(msg[i], ctypes.POINTER(PamMessage))[0] + if pm.msg_style == PAM_PROMPT_ECHO_OFF: + pw = password_cpy[0] + pw_buf = libc.strdup(pw.encode()) + arr[i].resp = pw_buf + arr[i].resp_retcode = 0 + else: + arr[i].resp = None + arr[i].resp_retcode = PAM_END + out_resp[0] = buf + return PAM_SUCCESS + + cb = CONV_FUNC(conv) + conv_struct = PamConv(cb, None) + handle = ctypes.c_void_p() + + ret = libpam.pam_start(SERVICE.encode(), username.encode(), ctypes.byref(conv_struct), ctypes.byref(handle)) + if ret != PAM_SUCCESS: + err = libpam.pam_strerror(handle, ret) + return {"ok": False, "error": f"PAM start failed: {(err or b'').decode()}"} + + ret = libpam.pam_authenticate(handle, 0) + if ret != PAM_SUCCESS: + libpam.pam_end(handle, ret) + return {"ok": False, "error": "Authentication failed"} + + ret = libpam.pam_acct_mgmt(handle, 0) + libpam.pam_end(handle, ret) + if ret != PAM_SUCCESS: + return {"ok": False, "error": "Account expired or disabled"} + + result = subprocess.run( + ["getent", "group", GRP_NAME], + capture_output=True, text=True, timeout=10, + ) + if result.returncode != 0: + return {"ok": False, "error": f"Group '{GRP_NAME}' does not exist"} + + group_parts = result.stdout.strip().split(":") + members = group_parts[-1].split(",") if len(group_parts) >= 4 else [] + gid = group_parts[2] if len(group_parts) >= 3 else None + + if username in members: + return {"ok": True, "username": username} + + if gid: + pw_result = subprocess.run( + ["getent", "passwd", username], + capture_output=True, text=True, timeout=10, + ) + if pw_result.returncode == 0: + user_gid = pw_result.stdout.strip().split(":")[3] if ":" in pw_result.stdout else None + if user_gid == gid: + return {"ok": True, "username": username} + + return {"ok": False, "error": f"User '{username}' is not in the '{GRP_NAME}' group"} + + +def main(): + try: + data = json.load(sys.stdin) + except json.JSONDecodeError as e: + print(json.dumps({"ok": False, "error": f"Invalid input: {e}"})) + sys.exit(1) + + username = data.get("username", "").strip() + password = data.get("password", "") + + if not username or not password: + print(json.dumps({"ok": False, "error": "Username and password required"})) + sys.exit(1) + + result = verify(username, password) + print(json.dumps(result)) + sys.exit(0 if result["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/dequel b/scripts/dequel index 811a722..d61d1a1 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -1,9 +1,12 @@ #!/usr/bin/env bash set -euo pipefail -DEQUEL_HOME="${DEQUEL_HOME:-$HOME/.dequel}" +SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || realpath "$0" 2>/dev/null || echo "$0")" +SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" +DEQUEL_HOME="${DEQUEL_HOME:-$SCRIPT_DIR}" COMPOSE_FILE="$DEQUEL_HOME/docker-compose.yml" -VERSION="0.1.0" +INSTALLED_VERSION="$(cat "$DEQUEL_HOME/VERSION" 2>/dev/null || echo "0.1.0")" +export DEQUEL_VERSION="$INSTALLED_VERSION" BOLD='\033[1m' DIM='\033[2m' @@ -15,6 +18,7 @@ NC='\033[0m' header() { printf "\n${BOLD}${AMBER}==>${NC}${BOLD} %s${NC}\n" "$*"; } info() { printf " ${DIM}%s${NC}\n" "$*"; } success() { printf " ${GREEN}✓${NC} %s\n" "$*"; } +warn() { printf " ${AMBER}⚠${NC} %s\n" "$*"; } fail() { printf " ${RED}✗${NC} %s\n" "$*"; exit 1; } cmd() { @@ -27,7 +31,12 @@ cmd() { cmd_start() { header "Starting Dequel" cmd up -d - success "Dequel is running at http://localhost" + local domain="${CADDY_BASE_DOMAIN:-localhost}" + if [ "$domain" = "localhost" ]; then + success "Dequel is running at http://localhost" + else + success "Dequel is running at https://$domain" + fi } cmd_stop() { @@ -53,9 +62,100 @@ cmd_logs() { cmd_update() { header "Updating Dequel" + + local repo="Lftobs/dequel" + local tag="" + local base_url="" + + info "Checking for latest release..." + tag=$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest" \ + | grep '"tag_name"' | head -1 | sed -E 's/.*"([^"]+)".*/\1/') || true + + if [ -z "$tag" ]; then + warn "Could not determine latest release, using main branch" + base_url="https://raw.githubusercontent.com/$repo/main" + else + base_url="https://raw.githubusercontent.com/$repo/$tag" + success "Latest release: $tag" + fi + + local tmp_dir + tmp_dir=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" EXIT + + info "Downloading updated configuration..." + curl -fsSL "$base_url/docker-compose.yml" -o "$tmp_dir/docker-compose.yml" + curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" + curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" + curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + + mkdir -p "$tmp_dir/infra/monitoring/grafana/datasources" \ + "$tmp_dir/infra/monitoring/grafana/dashboards" \ + "$tmp_dir/scripts/auth" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + curl -fsSL "$base_url/infra/monitoring/$f" -o "$tmp_dir/infra/monitoring/$f" + done + + for f in loki.yml prometheus.yml; do + curl -fsSL "$base_url/infra/monitoring/grafana/datasources/$f" -o "$tmp_dir/infra/monitoring/grafana/datasources/$f" + done + + for f in dashboards.yml deployed-apps.json; do + curl -fsSL "$base_url/infra/monitoring/grafana/dashboards/$f" -o "$tmp_dir/infra/monitoring/grafana/dashboards/$f" + done + + for f in pam-server.py pam-verify.py; do + curl -fsSL "$base_url/scripts/auth/$f" -o "$tmp_dir/scripts/auth/$f" + done + + python3 -c " +import re +with open('$tmp_dir/docker-compose.yml', 'r') as f: + content = f.read() +content = re.sub(r'#\s*(image:\s*ghcr\.io/lftobs/dequel/api:latest)', r'\1', content) +content = re.sub(r'#\s*(image:\s*ghcr\.io/lftobs/dequel/web:latest)', r'\1', content) +content = re.sub(r'(build:\s*\n\s*context:\s*\.\s*\n\s*dockerfile:\s*\./apps/api/Dockerfile)', lambda m: '\n'.join(' # ' + line.strip() for line in m.group(1).split('\n')), content) +content = re.sub(r'(build:\s*\n\s*context:\s*\./apps/web)', lambda m: '\n'.join(' # ' + line.strip() for line in m.group(1).split('\n')), content) +with open('$tmp_dir/docker-compose.yml', 'w') as f: + f.write(content) +" 2>/dev/null || true + + mkdir -p "$DEQUEL_HOME/infra/caddy" \ + "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ + "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" \ + "$DEQUEL_HOME/scripts/auth" + + mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" + mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + + for f in prometheus.yml loki-config.yml promtail-config.yml; do + mv "$tmp_dir/infra/monitoring/$f" "$DEQUEL_HOME/infra/monitoring/$f" + done + for f in loki.yml prometheus.yml; do + mv "$tmp_dir/infra/monitoring/grafana/datasources/$f" "$DEQUEL_HOME/infra/monitoring/grafana/datasources/$f" + done + for f in dashboards.yml deployed-apps.json; do + mv "$tmp_dir/infra/monitoring/grafana/dashboards/$f" "$DEQUEL_HOME/infra/monitoring/grafana/dashboards/$f" + done + for f in pam-server.py pam-verify.py; do + mv "$tmp_dir/scripts/auth/$f" "$DEQUEL_HOME/scripts/auth/$f" + done + + mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" + chmod +x "$tmp_dir/dequel" + mv "$tmp_dir/dequel" "$SCRIPT_PATH" + + success "Configuration updated" + + header "Pulling Docker images" cmd pull + + header "Recreating services" cmd up -d - success "Dequel updated." + + success "Dequel updated to ${tag:-main}" } cmd_uninstall() { @@ -110,7 +210,7 @@ cmd_uninstall() { } cmd_version() { - echo "dequel v$VERSION" + echo "dequel v$INSTALLED_VERSION" } cmd_help() { @@ -124,13 +224,13 @@ cmd_help() { echo " restart Restart all Dequel services" echo " status Show service status" echo " logs Follow service logs" - echo " update Pull latest images and recreate services" + echo " update Download latest config, pull images, and recreate services" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help" echo "" echo "Environment:" - echo " DEQUEL_HOME Config directory (default: \$HOME/.dequel)" + echo " DEQUEL_HOME Config directory (default: auto-detected from install path)" } case "${1:-}" in diff --git a/scripts/ensure-dashboards.py b/scripts/ensure-dashboards.py new file mode 100755 index 0000000..4c83a2d --- /dev/null +++ b/scripts/ensure-dashboards.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +import json +import os +import re +import subprocess +import sys + +GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost/grafana") +GRAFANA_USER = os.environ.get("GRAFANA_USER", "admin") +GRAFANA_PASS = os.environ.get("GRAFANA_PASS", "admin") +API_URL = os.environ.get("API_URL", "http://localhost/api") + +login = subprocess.run( + [ + "curl", + "-sk", + "-X", + "POST", + f"{GRAFANA_URL}/login", + "-H", + "Content-Type: application/json", + "-d", + json.dumps({"user": GRAFANA_USER, "password": GRAFANA_PASS}), + "-c", + "/tmp/grafana_cookies", + ], + capture_output=True, + text=True, +) +resp = json.loads(login.stdout) if login.stdout else {} +if "message" not in resp or resp.get("message") != "Logged in": + print(f"Grafana login failed: {login.stdout[:200]}") + sys.exit(1) +print("Logged into Grafana") + +result = subprocess.run( + ["curl", "-sk", f"{API_URL}/projects"], capture_output=True, text=True +) +if not result.stdout: + print("API returned empty response") + sys.exit(1) +projects = json.loads(result.stdout) +print(f"Found {len(projects)} projects") + +# Get running containers +running = set() +result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], capture_output=True, text=True +) +for c in result.stdout.strip().split("\n"): + if c: + running.add(c) + + +def grafana_post(path, data): + return subprocess.run( + [ + "curl", + "-sk", + "-X", + "POST", + "-b", + "/tmp/grafana_cookies", + "-H", + "Content-Type: application/json", + "-d", + json.dumps(data), + f"{GRAFANA_URL}/api{path}", + ], + capture_output=True, + text=True, + ) + + +created = 0 +for p in projects: + slug = p["name"].lower().replace(" ", "-").replace("_", "-") + slug = re.sub(r"[^a-z0-9-]", "", slug).strip("-")[:63] + + matching = sorted(c for c in running if c.startswith(slug + "-")) + if not matching: + print(f" Skip {p['name']}: no running containers") + continue + + container_regex = slug + "-.*" + print(f" {p['name']} (containers: {', '.join(matching)})") + + dashboard = { + "dashboard": { + "title": f"Dequel \u2014 {p['name']}", + "uid": f"dequel-project-{slug}", + "tags": ["dequel", "project", slug], + "schemaVersion": 39, + "version": 1, + "timezone": "browser", + "refresh": "30s", + "panels": [ + { + "type": "row", + "title": "Resource Usage", + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + }, + { + "id": 1, + "type": "timeseries", + "title": "CPU Usage", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 1}, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": {"mode": "normal"}, + "fillOpacity": 30, + "lineWidth": 1, + }, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": True, + }, + "tooltip": {"mode": "multi"}, + }, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": f'rate(container_cpu_usage_seconds_total{{name=~"{container_regex}"}}[$__rate_interval])', + "legendFormat": "{{name}}", + "refId": "A", + } + ], + }, + { + "id": 2, + "type": "timeseries", + "title": "Memory Usage", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 1}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "stacking": {"mode": "normal"}, + "fillOpacity": 30, + "lineWidth": 1, + }, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": True, + }, + "tooltip": {"mode": "multi"}, + }, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": f'container_memory_working_set_bytes{{name=~"{container_regex}"}}', + "legendFormat": "{{name}}", + "refId": "A", + } + ], + }, + { + "type": "row", + "title": "Request Metrics", + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 10}, + }, + { + "id": 4, + "type": "timeseries", + "title": "Request Rate", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 9, "w": 24, "x": 0, "y": 11}, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": {"fillOpacity": 30, "lineWidth": 1}, + }, + "overrides": [], + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": True, + }, + "tooltip": {"mode": "multi"}, + }, + "targets": [ + { + "datasource": {"type": "loki", "uid": "loki"}, + "expr": f'sum by(host) (count_over_time({{container=~"{container_regex}"}} | json [5m]))', + "legendFormat": "{{host}}", + "refId": "A", + } + ], + }, + { + "type": "row", + "title": "Logs", + "collapsed": False, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 20}, + }, + { + "id": 3, + "type": "logs", + "title": f"Container Logs", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 21}, + "options": { + "showLabels": True, + "showTime": True, + "wrapLogMessage": True, + "enableLogDetails": True, + "dedupStrategy": "none", + }, + "targets": [ + { + "datasource": {"type": "loki", "uid": "loki"}, + "expr": f'{{container=~"{container_regex}"}}', + "refId": "A", + } + ], + }, + ], + }, + "overwrite": True, + } + + r = grafana_post("/dashboards/db", dashboard) + try: + resp = json.loads(r.stdout) + if resp.get("status") == "success": + url = resp.get("url", "") + full_url = ( + f"https://localhost{url}" + if url.startswith("/grafana/") + else f"{GRAFANA_URL}{url}" + ) + print(f" OK: {full_url}") + created += 1 + else: + print(f" Error: {resp}") + except json.JSONDecodeError: + print(f" Failed: {r.stdout[:300]}") + +print(f"\nDone: {created} dashboard(s) created/updated") + +subprocess.run(["rm", "-f", "/tmp/grafana_cookies"], capture_output=True) diff --git a/scripts/ensure-dashboards.sh b/scripts/ensure-dashboards.sh new file mode 100755 index 0000000..a651554 --- /dev/null +++ b/scripts/ensure-dashboards.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +GRAFANA_URL="${1:-https://localhost/grafana}" +GRAFANA_USER="${2:-admin}" +GRAFANA_PASS="${3:-admin}" +API_URL="${4:-https://localhost/api}" + +export GRAFANA_URL GRAFANA_USER GRAFANA_PASS API_URL + +exec python3 "$SCRIPT_DIR/ensure-dashboards.py" \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh index 14dde29..a9047b3 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -34,16 +34,30 @@ download_if_missing() { check_prerequisites() { header "Checking prerequisites" - if command -v docker &>/dev/null; then + if command -v docker >/dev/null 2>&1; then success "Docker found: $(docker --version)" else fail "Docker is not installed. See https://docs.docker.com/engine/install/" fi - if docker compose version &>/dev/null; then + if docker info >/dev/null 2>&1; then + success "Docker socket accessible" + else + warn "Docker socket not accessible — attempting to add user to docker group" + if sudo usermod -aG docker "$USER" 2>/dev/null; then + success "Added $USER to docker group" + warn "Log out and back in (or run 'newgrp docker') for the change to take effect" + fail "Please log out and back in, then re-run the install script" + else + warn "Could not add user to docker group automatically" + warn "Run this manually: sudo usermod -aG docker $USER && newgrp docker" + fi + fi + + if docker compose version >/dev/null 2>&1; then success "Docker Compose found: $(docker compose version)" COMPOSE_CMD="docker compose" - elif docker-compose --version &>/dev/null; then + elif docker-compose --version >/dev/null 2>&1; then success "docker-compose found: $(docker-compose --version)" warn "Consider upgrading to 'docker compose' (Docker Compose v2)" COMPOSE_CMD="docker-compose" @@ -54,7 +68,7 @@ check_prerequisites() { setup_directories() { header "Setting up installation directory" - mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/workspace" "$INSTALL_DIR/infra/caddy/routes" "$INSTALL_DIR/infra/monitoring/grafana/datasources" + mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/workspace" "$INSTALL_DIR/infra/caddy/routes" "$INSTALL_DIR/infra/monitoring/grafana/datasources" "$INSTALL_DIR/infra/monitoring/grafana/dashboards" info "Installing to: $INSTALL_DIR" } @@ -113,28 +127,50 @@ download_configs() { for f in loki.yml prometheus.yml; do download_if_missing "$BASE_URL/infra/monitoring/grafana/datasources/$f" "$INSTALL_DIR/infra/monitoring/grafana/datasources/$f" done + + for f in dashboards.yml deployed-apps.json; do + download_if_missing "$BASE_URL/infra/monitoring/grafana/dashboards/$f" "$INSTALL_DIR/infra/monitoring/grafana/dashboards/$f" + done } prompt_config() { header "Configuration" - if [ ! -t 0 ]; then - warn "Non-interactive mode: skipping configuration prompt" - warn "Set CADDY_EMAIL and CADDY_BASE_DOMAIN manually in $INSTALL_DIR/.env" + local ADMIN_EMAIL="" + local HOSTNAME="" + + if [ -t 0 ]; then + read -r -p " Admin email (for SSL notifications, optional): " ADMIN_EMAIL + read -r -p " Base domain (e.g. dequel.example.com, optional): " HOSTNAME + elif (: /dev/null; then + read -r -p " Admin email (for SSL notifications, optional): " ADMIN_EMAIL < /dev/tty + read -r -p " Base domain (e.g. dequel.example.com, optional): " HOSTNAME < /dev/tty + else + warn "No terminal — skipping configuration prompt" + warn "Set CADDY_EMAIL and CADDY_BASE_DOMAIN in $INSTALL_DIR/.env after install" return fi - read -r -p " Admin email (for SSL notifications, optional): " ADMIN_EMAIL - read -r -p " Hostname (e.g. dequel.example.com, optional): " HOSTNAME + local ENC_KEY + ENC_KEY=$(openssl rand -hex 32 2>/dev/null || dd if=/dev/urandom bs=32 count=1 status=none 2>/dev/null | od -A n -t x1 | tr -d ' \n' || fail "Cannot generate encryption key — openssl and dd both failed") - if [ -n "$ADMIN_EMAIL" ] || [ -n "$HOSTNAME" ]; then - cat > "$INSTALL_DIR/.env" < "$INSTALL_DIR/data/dequel.json" < "$INSTALL_DIR/.env" + chmod 600 "$INSTALL_DIR/.env" + success "Created $INSTALL_DIR/.env" } pull_images() { diff --git a/scripts/tegami.mts b/scripts/tegami.mts new file mode 100644 index 0000000..8029ce1 --- /dev/null +++ b/scripts/tegami.mts @@ -0,0 +1,169 @@ +import { execSync } from "node:child_process"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { tegami } from "tegami"; +import { runCli } from "tegami/cli"; +import { github } from "tegami/plugins/github"; +import type { Draft, TegamiPlugin } from "tegami"; + +const REPO = "Lftobs/dequel"; + +function getPreviousTag(): string { + try { + const output = execSync( + "git tag -l 'v*' --sort=-v:refname", + { encoding: "utf-8" } + ); + const tags = output.trim().split("\n").filter(Boolean); + return tags[0] ?? ""; + } catch { + return ""; + } +} + +function getCommitMap(fromTag: string): Map { + const map = new Map(); + if (!fromTag) return map; + try { + const output = execSync( + `git log --oneline --format="%H %s" ${fromTag}..HEAD --no-merges`, + { encoding: "utf-8" } + ); + for (const line of output.trim().split("\n")) { + if (!line) continue; + const sha = line.slice(0, 40); + const msg = line.slice(41); + map.set(msg.toLowerCase(), sha); + } + } catch {} + return map; +} + +function formatChangelogBody(raw: string, commitMap: Map): string { + const lines = raw.replace(/^---[\s\S]*?---\n*/g, "").split("\n"); + const sectionTitles = new Map([ + ["New Features", "Features"], + ["Improvements", "Improvements"], + ["Bug Fixes", "Bug Fixes"], + ]); + + const result: string[] = []; + let sectionOpen = false; + + for (const line of lines) { + const trimmed = line.trimEnd(); + + const sectionMatch = trimmed.match(/^###\s+(.+)$/); + if (sectionMatch) { + if (sectionOpen) result.push(""); + const title = sectionTitles.get(sectionMatch[1]) ?? sectionMatch[1]; + result.push(`### ${title}`); + sectionOpen = true; + continue; + } + + const bulletMatch = trimmed.match(/^-\s+(.+)$/); + if (bulletMatch) { + const rawMsg = bulletMatch[1]; + const msg = rawMsg.replace(/^(feat|fix|refactor|chore|docs|style|perf|test|build|ci|revert)(\([^)]+\))?:\s*/, ""); + const display = msg.charAt(0).toUpperCase() + msg.slice(1); + const sha = commitMap.get(rawMsg) || commitMap.get(rawMsg.toLowerCase()); + if (sha) { + const shortSha = sha.slice(0, 7); + result.push(`- ${display} ([${shortSha}](https://github.com/${REPO}/commit/${sha}))`); + } else { + result.push(`- ${display}`); + } + continue; + } + + if (trimmed) result.push(trimmed); + } + + return result.join("\n") + "\n"; +} + +function docsChangelogPlugin(): TegamiPlugin { + return { + name: "docs-changelog", + enforce: "pre", + async applyDraft(this: any, draft: Draft) { + const logsDir = join(this.cwd, "apps/docs/src/content/changelogs"); + await mkdir(logsDir, { recursive: true }); + + const seen = new Set(); + const commitMap = getCommitMap(getPreviousTag()); + + for (const [pkgId, packageDraft] of draft.getPackageDrafts()) { + if (!packageDraft.changelogs?.length) continue; + if (pkgId === "npm:dequel") continue; + const pkg = this.graph.get(pkgId); + if (!pkg) continue; + + for (const entry of packageDraft.changelogs) { + if (seen.has(entry.id)) continue; + seen.add(entry.id); + + const pkgJson = JSON.parse(await readFile(join(pkg.path, "package.json"), "utf8")); + const currentVersion = pkgJson.version; + if (!currentVersion || !packageDraft.type) continue; + const { inc, parse } = await import("semver"); + const parsed = parse(currentVersion); + if (!parsed) continue; + const newVersion = inc(parsed, packageDraft.type); + if (!newVersion) continue; + + const raw = entry.getRawContent(); + const body = formatChangelogBody(raw, commitMap); + const today = new Date().toISOString().slice(0, 10); + const dest = join(logsDir, `v${newVersion}.md`); + + const content = `--- +version: ${newVersion} +date: "${today}" +--- + +${body} +`; + + await writeFile(dest, content); + } + } + + for (const [pkgId, packageDraft] of draft.getPackageDrafts()) { + if (pkgId === "npm:dequel") continue; + packageDraft.changelogs = []; + } + }, + }; +} + +const paper = tegami({ + npm: { + client: "bun", + }, + plugins: [ + docsChangelogPlugin(), + github({ + repo: REPO, + release: false, + versionPr: { + base: "main", + }, + }), + ], + groups: { + dequel: { + syncBump: true, + syncGitTag: true, + }, + }, + packages: { + "npm:dequel": { publish: false }, + "npm:dequel-api": { group: "dequel", publish: false }, + "npm:dequel-web": { group: "dequel", publish: false }, + "npm:dequel-docs": { group: "dequel", publish: false }, + }, +}); + +await runCli(paper); diff --git a/scripts/workflow/forbidden-pattern-scan.sh b/scripts/workflow/forbidden-pattern-scan.sh new file mode 100644 index 0000000..9e623fd --- /dev/null +++ b/scripts/workflow/forbidden-pattern-scan.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${1:-.}" +cd "$ROOT" + +FORBIDDEN=$'global[\'!\']' +EXCLUDE=${2:-":(exclude).github/workflows/forbidden-pattern-scan.yml"} + +matches=$(git grep -lF "$FORBIDDEN" -- . "$EXCLUDE" || true) +if [[ -n "$matches" ]]; then + echo "::error::Blocked literal pattern detected in repository files." >&2 + echo "Affected file(s):" >&2 + printf '%s\n' "$matches" >&2 + echo "" >&2 + git grep -nF "$FORBIDDEN" -- . "$EXCLUDE" >&2 || true + exit 1 +fi + +echo "OK: no files contain the forbidden pattern."