From 49807d61b1babd48eb7cc7f022618d08a8207da5 Mon Sep 17 00:00:00 2001 From: Jeff McCollum <16550786+jeffmccollum@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:45:55 -0500 Subject: [PATCH 1/2] update example to use role label instead of name --- braintrust/README.md | 17 +++++++++++++++-- .../examples/google-standard/values.yaml | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 9fa8f31..8b96663 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -152,6 +152,8 @@ brainstore: For Standard mode clusters, create node pools with local SSDs, then deploy: +Use the stable `braintrust/node-pool` role label in each node selector. Terraform node pool replacements can use generated GKE pool names. + **Configure the Helm chart:** ```yaml cloud: "google" @@ -159,10 +161,18 @@ For Standard mode clusters, create node pools with local SSDs, then deploy: google: mode: "standard" + api: + nodeSelector: + braintrust/node-pool: "services" + + aiGateway: + nodeSelector: + braintrust/node-pool: "services" + brainstore: reader: nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" # Target your node pool + braintrust/node-pool: "brainstore" resources: requests: cpu: "44" @@ -179,9 +189,12 @@ For Standard mode clusters, create node pools with local SSDs, then deploy: - brainstore-reader - brainstore-writer topologyKey: kubernetes.io/hostname + fastreader: + nodeSelector: + braintrust/node-pool: "brainstore" writer: nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" + braintrust/node-pool: "brainstore" resources: requests: cpu: "44" diff --git a/braintrust/examples/google-standard/values.yaml b/braintrust/examples/google-standard/values.yaml index a1ee0d9..99d7677 100644 --- a/braintrust/examples/google-standard/values.yaml +++ b/braintrust/examples/google-standard/values.yaml @@ -43,7 +43,7 @@ api: # Native GCS authentication via Workload Identity. Set to true to use GCS natively instead of S3-compatible access. enableGcsAuth: false nodeSelector: - cloud.google.com/gke-nodepool: "api" + braintrust/node-pool: "services" resources: requests: cpu: "4" @@ -56,6 +56,10 @@ api: - name: AWS_REGION value: "us-central1" +aiGateway: + nodeSelector: + braintrust/node-pool: "services" + # Brainstore configuration (split into reader and writer) brainstore: serviceAccount: @@ -78,7 +82,7 @@ brainstore: port: 4000 portName: http nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" # Target your node pool + braintrust/node-pool: "brainstore" resources: requests: cpu: "16" @@ -104,7 +108,12 @@ brainstore: volume: size: "200Gi" extraEnvVars: - + + # Brainstore Fast Reader configuration + fastreader: + nodeSelector: + braintrust/node-pool: "brainstore" + # Brainstore Writer configuration writer: name: "brainstore-writer" @@ -115,7 +124,7 @@ brainstore: port: 4000 portName: http nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" + braintrust/node-pool: "brainstore" resources: requests: cpu: "32" @@ -141,4 +150,3 @@ brainstore: volume: size: "200Gi" extraEnvVars: - \ No newline at end of file From 0c8d7e4548653028d71507f7ccf8625dfc12461a Mon Sep 17 00:00:00 2001 From: Jeff McCollum <16550786+jeffmccollum@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:05:59 -0500 Subject: [PATCH 2/2] add pdq --- AGENTS.md | 33 +++++ README.md | 10 ++ braintrust/README.md | 76 ++++++++-- .../examples/google-standard/values.yaml | 25 +++- braintrust/templates/api-pdb.yaml | 4 + braintrust/templates/brainstore-pdb.yaml | 25 ++++ braintrust/tests/brainstore-pdb_test.yaml | 138 ++++++++++++++++++ braintrust/values.yaml | 10 ++ 8 files changed, 306 insertions(+), 15 deletions(-) create mode 100644 braintrust/templates/brainstore-pdb.yaml create mode 100644 braintrust/tests/brainstore-pdb_test.yaml diff --git a/AGENTS.md b/AGENTS.md index 8c39cb8..755942b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,18 @@ Always use the namespace helper: namespace: {{ include "braintrust.namespace" . }} ``` +## GKE mode recommendation + +GKE Autopilot is the preferred solution for new Braintrust deployments. +Standard is supported when customer requirements prevent Autopilot use. +The separate Standard example does not change this preference. +Existing deployments must keep their initial cluster mode. +A mode change replaces the cluster, causes downtime, and requires Helm release redeployment. + +Recommend Autopilot first for new deployments. +Present Standard as a supported alternative for customer constraints. +Do not imply that existing Standard customers must switch modes. + ## Critical Safety Constraints These constraints exist because of real incidents and confirmed engineering guidance. Do not "simplify" or "clean up" code that implements them. @@ -82,6 +94,27 @@ These constraints apply to customers migrating from Data Plane 1.x to 2.0. New d The three brainstore configmaps (`brainstore-reader-configmap.yaml`, `brainstore-writer-configmap.yaml`, `brainstore-fastreader-configmap.yaml`) must have identical environment variable logic for `BRAINSTORE_RESPONSE_CACHE_URI`, `BRAINSTORE_CODE_BUNDLE_URI`, `BRAINSTORE_ASYNC_SCORING_OBJECTS`, and `BRAINSTORE_LOG_AUTOMATIONS_OBJECTS`. If you modify one, you must update all three. +### Disruption budgets and GKE Standard + +Brainstore readers, fast readers, and writers each expose an optional `podDisruptionBudget`. +Budgets default to disabled on every cloud. An enabled Brainstore budget defaults to `maxUnavailable: 1`. +The API retains `minAvailable` unless an explicit `maxUnavailable` takes precedence. +The GKE Standard example enables separate budgets for the API and all Brainstore roles. +A single writer can stop briefly during eviction. With multiple writers, the budget permits one unavailable replica. +Each role has an independent budget, so different roles can lose a replica simultaneously. +Readiness probes determine healthy replicas. Deployment rollout settings and `minReadySeconds` do not control node eviction. +Stable `braintrust/node-pool` labels connect Helm selectors to Terraform pools. +The Terraform pool map key sets this label. Automatic replacement preserves the label. +GKE deletion protection must exist on the source pool before hardware replacement. Its PDB protection expires after one hour. + +- Keep PDB templates cloud-independent and disabled by default. +- Enable the budgets in the GKE Standard example. +- Preserve the single-writer interruption exception. +- Use stable workload labels instead of generated GKE pool names in selectors. +- Document the Helm deployment and source pool protection steps before hardware replacement. +- Do not describe PDBs as an unconditional zero-downtime guarantee. +- Test role isolation, API compatibility, and cross-cloud behavior after PDB changes. + ### Version Numbers Chart version numbers are semantically meaningful for the upgrade path: diff --git a/README.md b/README.md index 5ec78fd..8ed6caa 100644 --- a/README.md +++ b/README.md @@ -24,3 +24,13 @@ helm upgrade --install \ Before installing the Braintrust Helm chart, ensure you have run the appropriate braintrust terraform module [Google](https://github.com/braintrustdata/terraform-google-braintrust-data-plane) or [Azure](https://github.com/braintrustdata/terraform-azure-braintrust-data-plane) to deploy the base infrastructure. See the [Braintrust Helm Chart](./braintrust/README.md) for more details. + +## GKE deployment modes + +GKE Autopilot is the preferred solution for new Braintrust deployments. GKE Standard is supported when customer requirements prevent Autopilot use. + +## GKE Standard node pool changes + +The [Standard example](braintrust/examples/google-standard/values.yaml) uses stable workload selectors and enables optional API and Brainstore disruption budgets. +The chart defaults remain unchanged for other deployments. +The [disruption budget guidance](braintrust/README.md#optional-disruption-budgets) explains the single-writer exception and the preparation steps for existing GKE pools. diff --git a/braintrust/README.md b/braintrust/README.md index 8b96663..a2bab7f 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -99,6 +99,8 @@ Kubernetes limits each Secret to 1 MiB. A normal combined CA bundle is typically ## GKE with Local SSDs +GKE Autopilot is the preferred solution for new Braintrust deployments. GKE Standard is supported when customer requirements prevent Autopilot use. + Braintrust requires local SSDs for maximum disk performance. Configuration varies depending on whether you're using GKE Autopilot or Standard mode. ### GKE Autopilot @@ -150,9 +152,13 @@ brainstore: ### GKE Standard Mode -For Standard mode clusters, create node pools with local SSDs, then deploy: +The Terraform module creates separate services and Brainstore pools. Brainstore uses a bundled Local SSD machine type without a disk count input. + +The stable `braintrust/node-pool` label connects Helm selectors to replacement pools with generated GKE names. +The Terraform map key sets the label. The `services` and `brainstore` keys match the selectors below. -Use the stable `braintrust/node-pool` role label in each node selector. Terraform node pool replacements can use generated GKE pool names. +The example enables PDBs for node eviction. Existing pools need deletion protection before a hardware replacement. +See [Optional disruption budgets](#optional-disruption-budgets) for the preparation sequence and limits. **Configure the Helm chart:** ```yaml @@ -162,6 +168,9 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor mode: "standard" api: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "services" @@ -171,6 +180,9 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor brainstore: reader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" resources: @@ -190,9 +202,15 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor - brainstore-writer topologyKey: kubernetes.io/hostname fastreader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" writer: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" resources: @@ -213,10 +231,13 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor topologyKey: kubernetes.io/hostname ``` -**What happens:** -- Pods are scheduled on your pre-configured node pools -- Local SSDs are automatically available via emptyDir volumes -- Pod anti-affinity ensures readers and writers don't share nodes (each pod gets dedicated node access) +The configuration has these effects: + +- Pods use the stable labels to select eligible nodes. +- Brainstore uses Local SSD storage through `emptyDir` volumes. +- Anti-affinity separates readers and writers across nodes. It does not exclude other workloads from those nodes. +- Separate PDBs limit voluntary evictions for each role. +- A single writer can stop briefly during a drain. ## AWS EKS Local Storage @@ -376,10 +397,45 @@ creating replacements. This causes a complete role outage and, with the default single writer, pauses background processing until the replacement becomes Ready. -These settings pace Deployment-managed rollouts only. The chart does not -currently create PodDisruptionBudgets for Brainstore, so these controls do not -limit voluntary disruptions such as node drains or protect against involuntary -pod or node failures. +These settings pace Deployment rollouts only. Optional PodDisruptionBudgets protect voluntary evictions such as node drains. +Neither mechanism protects against node failure. + +## Optional disruption budgets + +Brainstore readers, fast readers, and writers each support an optional PDB. +PDBs default to disabled on every cloud. Each enabled role defaults to `maxUnavailable: 1`. +The budget permits a single writer to stop briefly. Replicated roles retain all but one healthy replica during permitted evictions. + +```yaml +brainstore: + reader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 + fastreader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 + writer: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +api: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +``` + +The API retains its existing `minAvailable` behavior unless `maxUnavailable` is set. An explicit maximum takes precedence over the inherited minimum. +Each role has an independent budget. Evictions can proceed simultaneously across roles. +Readiness probes determine healthy replicas for PDBs. Deployment `minReadySeconds` does not delay PDB eviction permission. +The GKE Standard example enables these budgets. Existing EKS and AKS defaults remain unchanged. +GKE requires deletion protection on the source pool before a replacement, and its PDB protection expires after one hour. + +1. Deploy the Helm budgets before a node pool replacement. +2. Enable GKE deletion protection on existing pools in a separate Terraform apply before hardware changes. +3. Verify sufficient replicas and replacement capacity. + ## Testing diff --git a/braintrust/examples/google-standard/values.yaml b/braintrust/examples/google-standard/values.yaml index 99d7677..149d97a 100644 --- a/braintrust/examples/google-standard/values.yaml +++ b/braintrust/examples/google-standard/values.yaml @@ -1,10 +1,12 @@ +# GKE Autopilot is preferred for new deployments. +# Use this supported Standard example when customer requirements prevent Autopilot use. # Sample values for GKE Standard deployment # -# GKE Standard requires manual node pool configuration: -# - Create a dedicated node pool with local NVMe SSDs for Brainstore workloads -# - Recommended machine types: c4-standard-32 or higher with local SSDs -# - Configure local SSDs: Use 4x375GB local SSDs (1500GB total) or more -# - Total local SSD capacity should exceed the volume.size configured below +# Terraform creates the services and brainstore node pools. +# Brainstore requires a bundled Local SSD machine type, such as c4a-standard-48-lssd. +# GKE selects the fixed SSD count from the machine type. +# Enable PDB protection on existing pools before a hardware replacement. +# Deploy these PDBs before that replacement. # Global configs global: @@ -28,6 +30,9 @@ api: annotations: service: networking.gke.io/load-balancer-type: "Internal" + podDisruptionBudget: + enabled: true + maxUnavailable: 1 replicas: 4 # Uncomment the following section to use a different image or tag from the version in the Helm release #image: @@ -74,6 +79,9 @@ brainstore: # Brainstore Reader configuration reader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 name: "brainstore-reader" replicas: 2 service: @@ -111,11 +119,18 @@ brainstore: # Brainstore Fast Reader configuration fastreader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" # Brainstore Writer configuration + # One writer can stop briefly during a node drain. writer: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 name: "brainstore-writer" replicas: 1 service: diff --git a/braintrust/templates/api-pdb.yaml b/braintrust/templates/api-pdb.yaml index 3fe35ff..769b991 100644 --- a/braintrust/templates/api-pdb.yaml +++ b/braintrust/templates/api-pdb.yaml @@ -18,7 +18,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if hasKey $api.podDisruptionBudget "maxUnavailable" }} + maxUnavailable: {{ $api.podDisruptionBudget.maxUnavailable | toYaml | trim }} + {{- else }} minAvailable: {{ $api.podDisruptionBudget.minAvailable }} + {{- end }} selector: matchLabels: app: {{ $api.name }} diff --git a/braintrust/templates/brainstore-pdb.yaml b/braintrust/templates/brainstore-pdb.yaml new file mode 100644 index 0000000..1a70a8e --- /dev/null +++ b/braintrust/templates/brainstore-pdb.yaml @@ -0,0 +1,25 @@ +{{- $root := . -}} +{{- $rendered := 0 -}} +{{- range $role := list "reader" "fastreader" "writer" -}} +{{- $config := index $root.Values.brainstore $role -}} +{{- if $config.podDisruptionBudget.enabled }} +{{- if gt $rendered 0 }} +--- +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $config.name }} + namespace: {{ include "braintrust.namespace" $root }} + {{- with (merge (deepCopy $config.labels) $root.Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + maxUnavailable: {{ $config.podDisruptionBudget.maxUnavailable | toYaml | trim }} + selector: + matchLabels: + app: {{ $config.name }} +{{- $rendered = add1 $rendered -}} +{{- end }} +{{- end }} diff --git a/braintrust/tests/brainstore-pdb_test.yaml b/braintrust/tests/brainstore-pdb_test.yaml new file mode 100644 index 0000000..ab30029 --- /dev/null +++ b/braintrust/tests/brainstore-pdb_test.yaml @@ -0,0 +1,138 @@ +suite: Brainstore disruption budgets +templates: + - brainstore-pdb.yaml + - api-pdb.yaml +tests: + - it: leaves all PDBs absent by default + template: brainstore-pdb.yaml + asserts: + - hasDocuments: + count: 0 + - it: enables only the selected role and preserves a zero budget + template: brainstore-pdb.yaml + release: + namespace: custom-namespace + set: + brainstore.writer.name: custom-writer + brainstore.writer.podDisruptionBudget.enabled: true + brainstore.writer.podDisruptionBudget.maxUnavailable: 0 + global.namespace: custom-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.namespace + value: custom-namespace + - equal: + path: spec.selector.matchLabels.app + value: custom-writer + - equal: + path: spec.maxUnavailable + value: 0 + - notExists: + path: spec.minAvailable + - it: keeps labels isolated between roles + template: brainstore-pdb.yaml + set: + global.labels.shared: global + brainstore.reader.labels.readerOnly: reader + brainstore.writer.labels.writerOnly: writer + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.writer.podDisruptionBudget.enabled: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: metadata.labels.readerOnly + value: reader + documentIndex: 0 + - notExists: + path: metadata.labels.writerOnly + documentIndex: 0 + - equal: + path: metadata.labels.writerOnly + value: writer + documentIndex: 1 + - notExists: + path: metadata.labels.readerOnly + documentIndex: 1 + - equal: + path: metadata.labels.shared + value: global + documentIndex: 1 + - it: enables all roles in the GKE Standard example + template: brainstore-pdb.yaml + values: + - ../examples/google-standard/values.yaml + asserts: + - hasDocuments: + count: 3 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 0 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 1 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 2 + - it: uses the API maximum budget in the GKE Standard example + template: api-pdb.yaml + values: + - ../examples/google-standard/values.yaml + asserts: + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + - it: preserves the existing API minimum budget + template: api-pdb.yaml + set: + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.minAvailable: 2 + asserts: + - equal: + path: spec.minAvailable + value: 2 + - notExists: + path: spec.maxUnavailable + - it: supports optional budgets on aws + template: brainstore-pdb.yaml + set: + cloud: aws + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: "25%" + - it: supports optional budgets on azure + template: brainstore-pdb.yaml + set: + cloud: azure + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: "25%" + - it: supports optional budgets on google + template: brainstore-pdb.yaml + set: + cloud: google + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: "25%" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index d702372..361d483 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -214,6 +214,7 @@ api: maxUnavailable: 0 podDisruptionBudget: enabled: false + # An explicit maxUnavailable overrides this minimum. minAvailable: 1 topologySpread: enabled: false @@ -402,6 +403,9 @@ brainstore: deployment: {} service: {} pod: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 replicas: 2 # Configure rollout pacing for this Brainstore role. Defaults preserve the # rollout behavior from earlier chart versions. @@ -462,6 +466,9 @@ brainstore: deployment: {} service: {} pod: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 replicas: 2 minReadySeconds: 0 progressDeadlineSeconds: 600 @@ -520,6 +527,9 @@ brainstore: deployment: {} service: {} pod: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 replicas: 1 minReadySeconds: 0 progressDeadlineSeconds: 600