From 1ca4bf0c90979875624322787047f36657f44863 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 12 May 2026 12:39:52 +0000 Subject: [PATCH 1/2] feat: add TL;DR block to DBLab 4.1 post and restore preview envs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add blog/20260408-dblab-engine-4-1-released.md with the same component (Founders / Developers / DBAs / Managers / Ask AI) used in the invalid-index post. - Restore per-branch preview environments in .gitlab-ci.yml: every non-master / non-staging push now gets its own isolated deployment in the `review` namespace at https://., with an on_stop hook (`stop_review`) and a 1-week auto_stop_in safety net. Previously every non-master branch overwrote the shared staging namespace, which meant there were effectively no preview envs. - Wire deploy/configs/review.sh to the new review env (was an unused stub with only REPLICAS=1). - Document the GitLab↔GitHub mirror model in CLAUDE.md: master flows GitLab → GitHub, dev branches flow GitHub → GitLab via .github/workflows/mirror-to-gitlab.yml. https://claude.ai/code/session_015AC273dXNKXd8gG5QpC152 --- .gitlab-ci.yml | 62 +++++++ CLAUDE.md | 33 ++++ blog/20260408-dblab-engine-4-1-released.md | 182 +++++++++++++++++++++ deploy/configs/review.sh | 16 ++ 4 files changed, 293 insertions(+) create mode 100644 blog/20260408-dblab-engine-4-1-released.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86a7c4b2..0e6e8d07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,9 @@ variables: TAG_VERSION: "${REGISTRY}:${NAMESPACE}-${CI_PIPELINE_IID}" TAG_LATEST: "${REGISTRY}:${NAMESPACE}-latest" DOCS_NAME: "docs" + # Base domain for per-branch preview environments. + # Override via project CI/CD variable if your infrastructure uses a different host. + PREVIEW_BASE_DOMAIN: "preview-docs.postgres.ai" # Validate RSS/Atom feeds before building and deploying validate_feeds: @@ -117,6 +120,23 @@ validate_feeds: ENV: staging NAMESPACE: staging +# Per-branch preview environment. +# Each non-master branch gets its own isolated deployment in the `review` +# namespace, named after the branch slug, exposed at +# https://${CI_COMMIT_REF_SLUG}.${PREVIEW_BASE_DOMAIN}. +# The environment is automatically torn down when the branch is deleted +# (via `stop_review`, wired up through `environment.on_stop`). +.environment_template: &env_review + environment: + name: review/$CI_COMMIT_REF_SLUG + url: https://$CI_COMMIT_REF_SLUG.$PREVIEW_BASE_DOMAIN + on_stop: stop_review + auto_stop_in: 1 week + variables: + ENV: review + NAMESPACE: review + DOCS_NAME: "docs-$CI_COMMIT_REF_SLUG" + # Stages with branch policies. build_and_push_production: <<: *build_and_push_definition @@ -124,11 +144,22 @@ build_and_push_production: only: - master +# Static staging environment (v2.postgres.ai). Only the `staging` branch +# deploys here; other development branches get their own preview env below. build_and_push_staging: <<: *build_and_push_definition <<: *env_staging + only: + - staging + +# Per-branch preview environments — automatic build on every push to a +# development branch. +build_and_push_review: + <<: *build_and_push_definition + <<: *env_review except: - master + - staging deploy_production: <<: *deploy_definition @@ -140,5 +171,36 @@ deploy_staging: <<: *deploy_definition <<: *env_staging when: manual + only: + - staging + +deploy_review: + <<: *deploy_definition + <<: *env_review + except: + - master + - staging + +# Triggered automatically when the branch is deleted (or manually from the +# Environments page). Tears down the per-branch Deployment and Service so +# stale previews do not pile up. +stop_review: + stage: deploy + image: dtzar/helm-kubectl:2.14.1 + variables: + ENV: review + NAMESPACE: review + DOCS_NAME: "docs-$CI_COMMIT_REF_SLUG" + GIT_STRATEGY: none + environment: + name: review/$CI_COMMIT_REF_SLUG + action: stop + when: manual + script: + - kubectl config get-contexts + - kubectl config use-context postgres-ai/docs:k8s-cluster-docs + - kubectl delete deployment "$DOCS_NAME" -n "$NAMESPACE" --ignore-not-found + - kubectl delete service "$DOCS_NAME" -n "$NAMESPACE" --ignore-not-found except: - master + - staging diff --git a/CLAUDE.md b/CLAUDE.md index 5692478a..7f4e1396 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,3 +3,36 @@ Read and follow all rules in `.cursor/rules/` directory. @.cursor/rules + +## Repository mirroring: GitLab ↔ GitHub + +This repo lives in **two** places and they are mirrored in opposite +directions depending on the branch: + +- **`master` / `main`** is mirrored **GitLab → GitHub**. + GitLab is the source of truth for the main branch; GitHub gets the + read-only copy. + +- **Development branches** (everything except `master` / `main`) are + mirrored **GitHub → GitLab** by `.github/workflows/mirror-to-gitlab.yml` + on every push. + +The intent is: developers (including Claude Code's GitHub integration) +can start work on a development branch on GitHub; the branch is +auto-pushed to GitLab; the rest of the workflow — code review, +preview-environment CI, and the merge into `master` — happens on +GitLab. After the GitLab merge, the updated `master` flows back to +GitHub via the GitLab → GitHub mirror. + +Practical implications when using Claude Code here: + +- Push development branches to **GitHub** (`origin`). The mirror workflow + handles GitLab; do not push directly to GitLab from this clone. +- Open PRs on GitHub for visibility, but the **merge** happens on + GitLab (a corresponding MR is opened on the GitLab side). +- Do **not** push to `master` from any branch — `master` only updates + via the GitLab → GitHub mirror after a GitLab MR is merged. +- Preview environments (`https://.preview-docs.postgres.ai`) + are provisioned by GitLab CI on every push to a non-`master` branch. + See `.gitlab-ci.yml` (`build_and_push_review` / `deploy_review` / + `stop_review`). diff --git a/blog/20260408-dblab-engine-4-1-released.md b/blog/20260408-dblab-engine-4-1-released.md new file mode 100644 index 00000000..27232b97 --- /dev/null +++ b/blog/20260408-dblab-engine-4-1-released.md @@ -0,0 +1,182 @@ +--- +authors: denis +date: 2026-04-08 00:00:00 +publishDate: 2026-04-08 00:00:00 +linktitle: "DBLab 4.1: protection leases, Teleport, Prometheus, and more" +title: "DBLab 4.1: protection leases, Teleport, Prometheus, and more" +weight: 0 +image: /assets/thumbnails/dblab-4.0-blog.png +tags: + - Product announcements + - DBLab Engine + - Database Lab Engine +--- + +import { BlogFooter } from '@site/src/components/BlogFooter' +import { TldrTabs } from '@site/src/components/TldrTabs' +import { denis } from '@site/src/config/authors' + +Building on DBLab 4.0's instant database branching, version 4.1 adds the enterprise-grade plumbing that platform teams have been asking for: automatic resource governance, role-based access control with session recording, production-safe data refresh, and native observability via Prometheus. + +If 4.0 was about making branching *possible* at O(1) cost, 4.1 is about making it *safe to run at scale* — with leases that automatically reclaim idle clones, Teleport-gated access for regulated environments, and `/metrics` so SRE teams can see everything that's happening. + + + +` on `dblab clone create` — auto-cleanup when your CI job exits", + "Webhooks fire before lease expiration so long-running experiments can extend automatically", + "Database rename at snapshot time — connect to `myapp` locally, even though source is `myapp_production`", + "Native ARM64 / Apple Silicon builds via Colima — full branching on a MacBook, no Rosetta" + ] + }} + dbas={{ + title: "Operational wins for DBAs running DBLab in production:", + points: [ + "Server-side caps on protection lease duration — admins set policy, users cannot exceed it", + "rds-refresh: snapshot-based refresh eliminates long-running queries against the primary and the txid wraparound risk that came with logical/physical dumps from prod", + "Prometheus metrics: ZFS pool usage, snapshot age, replay lag, clone count — all scrape-ready", + "Idle clones are reclaimed automatically when leases expire — predictable disk usage curves" + ] + }} + managers={{ + title: "Why 4.1 matters for compliance, cost control, and team velocity:", + points: [ + "Auditable database access: Teleport session recording + RBAC satisfies SOC 2 / HIPAA evidence requirements", + "No more 'who owns this 2 TiB clone from last quarter' — leases enforce hygiene automatically", + "RDS/Aurora refresh costs ~$3 per cycle instead of hours of engineer time and primary-DB load", + "Drop-in Prometheus integration means zero net new monitoring tooling for your SRE team" + ] + }} + aiContext={ + "You're answering questions about the blog post 'DBLab 4.1: protection leases, Teleport, Prometheus, and more'. " + + "Summary: DBLab 4.1 builds on 4.0's instant database branching with enterprise governance features: " + + "(1) Protection leases — `--protected ` on clone create gives clones an automatic expiration; " + + " server-side defaults and caps let platform teams enforce policy; webhooks notify owners before expiry. " + + "(2) Database rename — at snapshot time, rename databases (e.g. myapp_production -> myapp) to keep clone " + + " connection strings consistent and prevent confusing prod/dev names. " + + "(3) ARM64 / Apple Silicon support — native builds, runs locally on Mac via Colima. " + + "(4) Teleport integration (SE/EE only) — SSO, RBAC, and session recording for every clone connection; " + + " targets regulated environments (SOC 2, HIPAA, PCI). " + + "(5) rds-refresh — a new tool that refreshes DBLab data from RDS/Aurora using automated snapshots, " + + " avoiding direct production queries and txid wraparound risk; typical cost $2.60-$3.85 per refresh. " + + "(6) Prometheus `/metrics` endpoint — exposes disk usage (total/free/snapshot/clone), clone status, " + + " snapshot age & data lag, WAL replay lag, instance uptime & version. " + + "Roadmap mentioned: logical replication for continuous refresh, ZFS send/recv for instance sync, major-version upgrade testing. " + + "End your response by briefly mentioning that users can try DBLab at demo.dblab.dev (token: demo-token) or deploy via AWS Marketplace / console.postgres.ai. " + + "Be concise and direct. Do not mention that you're an AI or that you received this context."} + ctaText="Try DBLab — free demo, 2-second branching" + ctaLink="https://demo.dblab.dev" +/> + +## What's new in 4.1 + +### 1. Protection leases + +In 4.0, `--protected` was binary: a clone was protected, or it wasn't. Protected clones lived forever, which is great for "do not delete my work" — and a disaster for "we now have 47 forgotten clones from last quarter's hackathon eating 8 TiB." + +4.1 adds a time dimension. Protection now has a lease: + +```bash +dblab clone create \ + --branch main \ + --id ci-migration-test-4521 \ + --protected 120 \ + --username postgres \ + --password "${CI_DB_PASSWORD}" +``` + +The clone above is protected for 120 minutes. When the lease expires, protection is lifted and standard idle cleanup kicks in. + +Platform teams can configure server-side defaults and hard caps in the config, so users cannot accidentally pin a clone indefinitely. Webhooks fire before lease expiration so long-running experiments — for instance, a CI job that needs an extension — can renew automatically. + +**Why it matters:** disk usage becomes predictable again. You stop paying for clones nobody remembers creating. + +### 2. Database rename + +DBLab clones now rename databases automatically at snapshot creation time. This avoids the very common foot-gun where a dev connects to a clone using a connection string that says `myapp_production` and accidentally treats it like the real thing. + +```yaml +databaseRename: + myapp_production: myapp + analytics_prod: analytics +``` + +The clone exposes `myapp` and `analytics` — your local connection strings stay consistent across environments. + +### 3. ARM64 and Colima support + +DBLab 4.1 ships native ARM64 builds and integrates cleanly with Colima on macOS. Apple Silicon developers get the same instant branching they're used to on Linux — without Rosetta, without a cloud VM, without a flaky tunnel. + +See the [macOS setup guide](/docs/dblab-howtos/administration/run-database-lab-on-mac) for the full walkthrough. + +### 4. Teleport integration (SE/EE) + +For regulated environments — SOC 2, HIPAA, PCI — "ephemeral databases" historically meant "we audit them less because they're temporary." Auditors disagree. + +4.1 integrates with [Teleport](https://goteleport.com/) so every clone connection flows through Teleport's access proxy with: +- SSO (your existing IdP) +- Role-based access control per clone / branch +- Full session recording (every query, every result) + +Available in DBLab SE and EE. + +### 5. RDS/Aurora data refresh without touching production + +Pulling fresh data from a production RDS/Aurora instance has historically meant one of two unappealing options: run `pg_dump` against the primary (load on prod, txid wraparound risk on long-running snapshots) or stand up a read replica just to dump from it. + +4.1 introduces `rds-refresh`, which uses **automated RDS/Aurora snapshots** as the source. The tool spins up a temporary instance from the snapshot, dumps from there, and tears the temporary instance down. + +```bash +dblab rds-refresh \ + --source-instance prod-db \ + --snapshot-strategy latest-automated +``` + +Typical cost: **$2.60–$3.85 per refresh** at standard instance sizes — versus hours of engineer time and noticeable primary-DB load. + +### 6. Prometheus metrics + +DBLab Engine now exposes a `/metrics` endpoint in Prometheus format: + +- **Disk** — total, free, breakdown by snapshot vs clone +- **Clones** — count by status, per-clone resource consumption +- **Snapshots** — age, data lag +- **Replication** — WAL replay lag, sync state +- **Instance** — uptime, version + +Scrape it with your existing Prometheus setup. Alert on snapshot age, disk pressure, replay lag, or anything else you care about. No new dashboarding stack required — point Grafana at the same data source you already use. + +## Roadmap + +Work in progress: + +1. **Logical replication for continuous refresh** — keep clones near-current without snapshot/restore cycles +2. **ZFS send/recv for instance synchronization** — sync between DBLab instances, including local laptops +3. **Major version upgrade testing as a first-class feature** — test PG `N → N+1` migrations against full-size data + +## Where to start + +1. **Try the demo:** [demo.dblab.dev](https://demo.dblab.dev) (token: `demo-token`) +2. **Deploy DBLab SE:** [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-wlmm2satykuec) or [Postgres AI Console](https://console.postgres.ai) +3. **Install open source:** [How-to](https://postgres.ai/docs/dblab-howtos/administration/install-dle-manually) +4. **Enterprise inquiries:** [team@postgres.ai](mailto:team@postgres.ai) + +--- + +DBLab 4.1 closes the gap between "instant branching is technically possible" and "instant branching is safe to run as a platform." Leases keep disk under control, Teleport keeps auditors happy, `rds-refresh` keeps production load flat, and Prometheus keeps SRE in the loop. + +[Get started](https://postgres.ai/docs/database-lab) | [GitLab](https://gitlab.com/postgres-ai/database-lab) | [Slack](https://slack.postgres.ai) + + diff --git a/deploy/configs/review.sh b/deploy/configs/review.sh index 25d74ee7..3cecce8b 100644 --- a/deploy/configs/review.sh +++ b/deploy/configs/review.sh @@ -1 +1,17 @@ export REPLICAS=1 + +# Per-branch preview environment. +# URL is derived from the GitLab branch slug + PREVIEW_BASE_DOMAIN +# (see `.environment_template: &env_review` in .gitlab-ci.yml). +export URL="https://${CI_COMMIT_REF_SLUG}.${PREVIEW_BASE_DOMAIN}" +export BASE_URL="/" + +# Previews share the staging API / auth / bot backends — there is no separate +# per-branch backend, only the docs frontend is built per branch. +export SIGN_IN_URL="https://console-v2.postgres.ai/signin" +export BOT_WS_URL="wss://v2.postgres.ai/ai-bot-ws/" +export API_URL_PREFIX="https://v2.postgres.ai/api/general" + +# No analytics on preview environments. +export UMAMI_WEBSITE_ID="" +export UMAMI_SCRIPT_URL="" From ae27d10c8b1597cdfb8b22e5d5e47450f7641619 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 12 May 2026 13:44:42 +0000 Subject: [PATCH 2/2] ci: simplify review apps to match historical 2020 pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit reinvented review apps with a bunch of new machinery (PREVIEW_BASE_DOMAIN, stop_review job, auto_stop_in, on_stop wiring). The existing infrastructure already has the `review` namespace, the ingress, and the Cloudflare wildcard for *.pgai.green — review apps just need to deploy `docs-` into the `review` namespace and the rest is handled outside this repo. This commit replaces the over-engineered version with the same minimal shape that lived here before commit 43d434c (Oct 2020, "fix: remove review CI"), adapted to the current TAG_VERSION/TAG_LATEST naming and with the correct external URL (https://docs-.pgai.green). Also drops the per-branch overwrite of the staging deployment by scoping `build_and_push_staging` / `deploy_staging` to the `staging` branch only — that bug is the actual reason previews stopped working: every dev push was clobbering the shared staging Deployment instead of getting an isolated review env. https://claude.ai/code/session_015AC273dXNKXd8gG5QpC152 --- .gitlab-ci.yml | 53 +++++++++------------------------------- CLAUDE.md | 9 ++++--- deploy/configs/review.sh | 11 +++++---- 3 files changed, 23 insertions(+), 50 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e6e8d07..0902573d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,9 +21,6 @@ variables: TAG_VERSION: "${REGISTRY}:${NAMESPACE}-${CI_PIPELINE_IID}" TAG_LATEST: "${REGISTRY}:${NAMESPACE}-latest" DOCS_NAME: "docs" - # Base domain for per-branch preview environments. - # Override via project CI/CD variable if your infrastructure uses a different host. - PREVIEW_BASE_DOMAIN: "preview-docs.postgres.ai" # Validate RSS/Atom feeds before building and deploying validate_feeds: @@ -121,21 +118,21 @@ validate_feeds: NAMESPACE: staging # Per-branch preview environment. -# Each non-master branch gets its own isolated deployment in the `review` -# namespace, named after the branch slug, exposed at -# https://${CI_COMMIT_REF_SLUG}.${PREVIEW_BASE_DOMAIN}. -# The environment is automatically torn down when the branch is deleted -# (via `stop_review`, wired up through `environment.on_stop`). +# Restores the review-apps pattern that existed before commit 43d434c +# ("fix: remove review CI", Oct 2020). The cluster's existing ingress + +# Cloudflare wildcard *.pgai.green route to the per-branch Service named +# `docs-` in the `review` namespace — no extra ingress object needed. .environment_template: &env_review environment: - name: review/$CI_COMMIT_REF_SLUG - url: https://$CI_COMMIT_REF_SLUG.$PREVIEW_BASE_DOMAIN - on_stop: stop_review - auto_stop_in: 1 week + name: review/$CI_COMMIT_REF_NAME + url: https://docs-${CI_COMMIT_REF_SLUG}.pgai.green variables: ENV: review NAMESPACE: review - DOCS_NAME: "docs-$CI_COMMIT_REF_SLUG" + # Branch-scoped tags so concurrent previews don't fight over the same one. + TAG_VERSION: "${REGISTRY}:review-${CI_COMMIT_REF_SLUG}-${CI_PIPELINE_IID}" + TAG_LATEST: "${REGISTRY}:review-${CI_COMMIT_REF_SLUG}-latest" + DOCS_NAME: "docs-${CI_COMMIT_REF_SLUG}" # Stages with branch policies. build_and_push_production: @@ -144,16 +141,14 @@ build_and_push_production: only: - master -# Static staging environment (v2.postgres.ai). Only the `staging` branch -# deploys here; other development branches get their own preview env below. +# Static staging env (v2.postgres.ai). Only the `staging` branch deploys here; +# other dev branches get their own preview env below so they don't trample it. build_and_push_staging: <<: *build_and_push_definition <<: *env_staging only: - staging -# Per-branch preview environments — automatic build on every push to a -# development branch. build_and_push_review: <<: *build_and_push_definition <<: *env_review @@ -180,27 +175,3 @@ deploy_review: except: - master - staging - -# Triggered automatically when the branch is deleted (or manually from the -# Environments page). Tears down the per-branch Deployment and Service so -# stale previews do not pile up. -stop_review: - stage: deploy - image: dtzar/helm-kubectl:2.14.1 - variables: - ENV: review - NAMESPACE: review - DOCS_NAME: "docs-$CI_COMMIT_REF_SLUG" - GIT_STRATEGY: none - environment: - name: review/$CI_COMMIT_REF_SLUG - action: stop - when: manual - script: - - kubectl config get-contexts - - kubectl config use-context postgres-ai/docs:k8s-cluster-docs - - kubectl delete deployment "$DOCS_NAME" -n "$NAMESPACE" --ignore-not-found - - kubectl delete service "$DOCS_NAME" -n "$NAMESPACE" --ignore-not-found - except: - - master - - staging diff --git a/CLAUDE.md b/CLAUDE.md index 7f4e1396..06f57696 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,8 @@ Practical implications when using Claude Code here: GitLab (a corresponding MR is opened on the GitLab side). - Do **not** push to `master` from any branch — `master` only updates via the GitLab → GitHub mirror after a GitLab MR is merged. -- Preview environments (`https://.preview-docs.postgres.ai`) - are provisioned by GitLab CI on every push to a non-`master` branch. - See `.gitlab-ci.yml` (`build_and_push_review` / `deploy_review` / - `stop_review`). +- Preview environments (`https://docs-.pgai.green`) are + provisioned by GitLab CI on every push to a non-`master` / + non-`staging` branch. See `build_and_push_review` and `deploy_review` + in `.gitlab-ci.yml`. The cluster ingress + Cloudflare wildcard for + `*.pgai.green` already route to the per-branch Service. diff --git a/deploy/configs/review.sh b/deploy/configs/review.sh index 3cecce8b..88232c70 100644 --- a/deploy/configs/review.sh +++ b/deploy/configs/review.sh @@ -1,13 +1,14 @@ export REPLICAS=1 # Per-branch preview environment. -# URL is derived from the GitLab branch slug + PREVIEW_BASE_DOMAIN -# (see `.environment_template: &env_review` in .gitlab-ci.yml). -export URL="https://${CI_COMMIT_REF_SLUG}.${PREVIEW_BASE_DOMAIN}" +# DOCS_NAME / CI_COMMIT_REF_SLUG come from .gitlab-ci.yml's &env_review. +# The cluster's existing Cloudflare + ingress route docs-.pgai.green +# to the Service named ${DOCS_NAME} in the `review` namespace. +export URL="https://${DOCS_NAME}.pgai.green" export BASE_URL="/" -# Previews share the staging API / auth / bot backends — there is no separate -# per-branch backend, only the docs frontend is built per branch. +# Previews share the staging API / auth / bot backends — only the docs +# frontend is built per branch. export SIGN_IN_URL="https://console-v2.postgres.ai/signin" export BOT_WS_URL="wss://v2.postgres.ai/ai-bot-ws/" export API_URL_PREFIX="https://v2.postgres.ai/api/general"