diff --git a/README.md b/README.md index 16ea42e..1dadf6f 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,12 @@ The following table lists the configurable parameters and their default values. | Parameter | Description | Default | |-----------|-------------|---------| -| `replicaCount` | Number of replicas | `1` | +| `replicaCount` | Number of replicas (omitted from the Deployment when `autoscaling.enabled`) | `1` | +| `autoscaling.enabled` | Render a HorizontalPodAutoscaler for the control-layer deployment | `false` | +| `autoscaling.minReplicas` | HPA minimum replicas | `1` | +| `autoscaling.maxReplicas` | HPA maximum replicas | `3` | +| `autoscaling.metrics` | Raw `autoscaling/v2` metrics list, rendered verbatim | `[]` | +| `autoscaling.behavior` | Raw `autoscaling/v2` behavior block, rendered verbatim | `{}` | | `image.repository` | Container image repository | `ghcr.io/doublewordai/control-layer` | | `image.tag` | Container image tag | Chart appVersion | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | @@ -191,16 +196,19 @@ The fusillade daemon handles background batch processing tasks. By default, it r | Parameter | Description | Default | |-----------|-------------|---------| | `fusillade.enabled` | Deploy fusillade as a separate deployment | `false` | -| `fusillade.replicaCount` | Number of fusillade replicas | `1` | +| `fusillade.replicaCount` | Number of fusillade replicas (omitted from a Deployment whose effective autoscaling is enabled) | `1` | +| `fusillade.autoscaling` | HPA defaults for the fusillade daemon and both split roles (`enabled`, `minReplicas`, `maxReplicas`, `metrics`, `behavior`) | `enabled: false` | | `fusillade.mode` | Optional daemon mode for the standard fusillade deployment (`both`, `request_only`, `batch_only`); empty uses the application default | `""` | | `fusillade.split.enabled` | Render separate request-only and batch-only daemon deployments | `false` | | `fusillade.split.request.enabled` | Render the request-only daemon deployment | `true` | | `fusillade.split.request.replicaCount` | Number of request daemon replicas | inherits `fusillade.replicaCount` | +| `fusillade.split.request.autoscaling` | HPA override for the request daemon | merged over `fusillade.autoscaling` | | `fusillade.split.request.resources` | CPU/Memory requests/limits for request daemon pods | inherits `fusillade.resources` | | `fusillade.split.request.env` | Additional environment variables for request daemon pods | merged after `env` and `fusillade.env` | | `fusillade.split.request.database` | Database pool overrides for request daemon pods | merged over `fusillade.database` | | `fusillade.split.batch.enabled` | Render the batch-only daemon deployment | `true` | | `fusillade.split.batch.replicaCount` | Number of batch daemon replicas | inherits `fusillade.replicaCount` | +| `fusillade.split.batch.autoscaling` | HPA override for the batch daemon | merged over `fusillade.autoscaling` | | `fusillade.split.batch.resources` | CPU/Memory requests/limits for batch daemon pods | inherits `fusillade.resources` | | `fusillade.split.batch.env` | Additional environment variables for batch daemon pods | merged after `env` and `fusillade.env` | | `fusillade.split.batch.database` | Database pool overrides for batch daemon pods | merged over `fusillade.database` | diff --git a/templates/deployment.yaml b/templates/deployment.yaml index 8966f57..34fec87 100644 --- a/templates/deployment.yaml +++ b/templates/deployment.yaml @@ -28,7 +28,11 @@ metadata: labels: {{- include "control-layer.labels" . | nindent 4 }} spec: + {{- /* Argo CD reconciles continuously; a rendered replicas field would revert + every HPA scale-up on the next sync, so omit it when the HPA owns the count. */}} + {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} + {{- end }} {{- if .Values.rollout.enabled }} strategy: {{- toYaml .Values.rollout.strategy | nindent 4 }} diff --git a/templates/fusillade/deployment.yaml b/templates/fusillade/deployment.yaml index a667a04..c57b61e 100644 --- a/templates/fusillade/deployment.yaml +++ b/templates/fusillade/deployment.yaml @@ -10,6 +10,7 @@ {{- $database := mergeOverwrite (deepCopy ($fusillade.database | default dict)) (get $config "database" | default dict) -}} {{- $podAnnotations := mergeOverwrite (deepCopy ($fusillade.podAnnotations | default dict)) (get $config "podAnnotations" | default dict) -}} {{- $podLabels := mergeOverwrite (deepCopy ($fusillade.podLabels | default dict)) (get $config "podLabels" | default dict) -}} +{{- $autoscaling := mergeOverwrite (deepCopy ($fusillade.autoscaling | default dict)) (get $config "autoscaling" | default dict) -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -17,7 +18,11 @@ metadata: labels: {{- include "control-layer.fusillade.labelsFor" (dict "root" $root "component" $component) | nindent 4 }} spec: + {{- /* Argo CD reconciles continuously; a rendered replicas field would revert + every HPA scale-up on the next sync, so omit it when the HPA owns the count. */}} + {{- if not $autoscaling.enabled }} replicas: {{- if hasKey $config "replicaCount" }} {{ get $config "replicaCount" }}{{- else }} {{ $fusillade.replicaCount }}{{- end }} + {{- end }} selector: matchLabels: {{- include "control-layer.fusillade.selectorLabelsFor" (dict "root" $root "component" $component) | nindent 6 }} diff --git a/templates/fusillade/hpa.yaml b/templates/fusillade/hpa.yaml new file mode 100644 index 0000000..12e430b --- /dev/null +++ b/templates/fusillade/hpa.yaml @@ -0,0 +1,58 @@ +{{- /* +One HorizontalPodAutoscaler per fusillade workload whose effective autoscaling +is enabled. Iterates the same workloads as fusillade/deployment.yaml (single +mode: -fusillade; split mode: -fusillade-request / -fusillade-batch). Effective +config = fusillade.autoscaling overlaid with the per-role autoscaling block. +*/}} +{{- define "control-layer.fusillade.hpa" -}} +{{- $root := .root -}} +{{- $fusillade := $root.Values.fusillade -}} +{{- $config := .config | default dict -}} +{{- $component := .component -}} +{{- $suffix := .suffix -}} +{{- $autoscaling := mergeOverwrite (deepCopy ($fusillade.autoscaling | default dict)) (get $config "autoscaling" | default dict) -}} +{{- if $autoscaling.enabled }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "control-layer.fullname" $root }}{{ $suffix }} + labels: + {{- include "control-layer.fusillade.labelsFor" (dict "root" $root "component" $component) | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "control-layer.fullname" $root }}{{ $suffix }} + minReplicas: {{ $autoscaling.minReplicas }} + maxReplicas: {{ $autoscaling.maxReplicas }} + {{- with $autoscaling.metrics }} + metrics: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} + +{{- if .Values.fusillade.enabled }} +{{- $split := .Values.fusillade.split | default dict -}} +{{- if $split.enabled }} +{{- $request := $split.request | default dict -}} +{{- $batch := $split.batch | default dict -}} +{{- $requestEnabled := true -}} +{{- $batchEnabled := true -}} +{{- if hasKey $request "enabled" }}{{- $requestEnabled = $request.enabled -}}{{- end -}} +{{- if hasKey $batch "enabled" }}{{- $batchEnabled = $batch.enabled -}}{{- end -}} +{{- if $requestEnabled }} +{{- include "control-layer.fusillade.hpa" (dict "root" . "suffix" "-fusillade-request" "component" "fusillade-request" "config" $request) }} +{{- end }} +{{- if $batchEnabled }} +{{- include "control-layer.fusillade.hpa" (dict "root" . "suffix" "-fusillade-batch" "component" "fusillade-batch" "config" $batch) }} +{{- end }} +{{- else }} +{{- include "control-layer.fusillade.hpa" (dict "root" . "suffix" "-fusillade" "component" "fusillade" "config" dict) }} +{{- end }} +{{- end }} diff --git a/templates/hpa.yaml b/templates/hpa.yaml new file mode 100644 index 0000000..246337b --- /dev/null +++ b/templates/hpa.yaml @@ -0,0 +1,23 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "control-layer.fullname" . }} + labels: + {{- include "control-layer.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "control-layer.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + {{- with .Values.autoscaling.metrics }} + metrics: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/tests/control_layer_deployment_test.yaml b/tests/control_layer_deployment_test.yaml index 2d2760d..7de6b88 100644 --- a/tests/control_layer_deployment_test.yaml +++ b/tests/control_layer_deployment_test.yaml @@ -220,3 +220,19 @@ tests: asserts: - failedTemplate: errorMessage: rollout.connectionDrainTimeoutSeconds must be 3600 to match the control-layer request timeout + + - it: should render replicas by default (no autoscaling) + template: deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 1 + + - it: should omit replicas when autoscaling is enabled + template: deployment.yaml + set: + autoscaling.enabled: true + asserts: + - isNull: + path: spec.replicas + diff --git a/tests/fusillade_deployment_test.yaml b/tests/fusillade_deployment_test.yaml index a806384..88d2f26 100644 --- a/tests/fusillade_deployment_test.yaml +++ b/tests/fusillade_deployment_test.yaml @@ -371,3 +371,20 @@ tests: content: name: DWCTL_BACKGROUND_SERVICES__BATCH_DAEMON__MODE value: "request_only" + + - it: should keep replicas in split mode by default + template: fusillade/deployment.yaml + set: + fusillade: + enabled: true + split: + enabled: true + asserts: + - equal: + path: spec.replicas + value: 1 + documentIndex: 0 + - equal: + path: spec.replicas + value: 1 + documentIndex: 1 diff --git a/tests/hpa_test.yaml b/tests/hpa_test.yaml new file mode 100644 index 0000000..ef4c4e2 --- /dev/null +++ b/tests/hpa_test.yaml @@ -0,0 +1,242 @@ +suite: hpa +templates: + - hpa.yaml + - fusillade/hpa.yaml + - deployment.yaml + - fusillade/deployment.yaml + - configmap.yaml + - secret.yaml +tests: + - it: renders no HPA by default + template: hpa.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders no fusillade HPA by default + template: fusillade/hpa.yaml + set: + fusillade.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: renders main HPA and omits replicas when enabled + set: + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + template: hpa.yaml + asserts: + - hasDocuments: + count: 1 + - isAPIVersion: + of: autoscaling/v2 + - isKind: + of: HorizontalPodAutoscaler + - equal: + path: metadata.name + value: RELEASE-NAME-control-layer + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer + - equal: + path: spec.scaleTargetRef.kind + value: Deployment + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 4 + - equal: + path: spec.metrics[0].resource.target.averageUtilization + value: 75 + - equal: + path: spec.behavior.scaleDown.stabilizationWindowSeconds + value: 300 + + - it: omits metrics and behavior keys when empty + set: + autoscaling.enabled: true + template: hpa.yaml + asserts: + - isNull: + path: spec.metrics + - isNull: + path: spec.behavior + + - it: omits replicas from main deployment when autoscaling enabled + set: + autoscaling.enabled: true + template: deployment.yaml + asserts: + - isNull: + path: spec.replicas + + - it: renders single-mode fusillade HPA from fusillade default + set: + fusillade: + enabled: true + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + template: fusillade/hpa.yaml + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: RELEASE-NAME-control-layer-fusillade + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer-fusillade + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: fusillade + - equal: + path: spec.maxReplicas + value: 5 + + - it: omits replicas from single-mode fusillade deployment when autoscaling enabled + set: + fusillade: + enabled: true + autoscaling: + enabled: true + template: fusillade/deployment.yaml + asserts: + - isNull: + path: spec.replicas + + - it: renders one HPA per split role from fusillade default + set: + fusillade: + enabled: true + autoscaling: + enabled: true + split: + enabled: true + template: fusillade/hpa.yaml + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer-fusillade-request + documentIndex: 0 + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: fusillade-request + documentIndex: 0 + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer-fusillade-batch + documentIndex: 1 + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: fusillade-batch + documentIndex: 1 + + - it: per-role override beats fusillade default + set: + fusillade: + enabled: true + autoscaling: + enabled: false + split: + enabled: true + request: + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 3 + template: fusillade/hpa.yaml + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer-fusillade-request + - equal: + path: spec.maxReplicas + value: 3 + + - it: per-role override can disable autoscaling for one role and merges over defaults + set: + fusillade: + enabled: true + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 6 + split: + enabled: true + batch: + autoscaling: + enabled: false + template: fusillade/hpa.yaml + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer-fusillade-request + - equal: + path: spec.minReplicas + value: 2 + - equal: + path: spec.maxReplicas + value: 6 + + - it: omits replicas only from split roles whose autoscaling is enabled + set: + fusillade: + enabled: true + replicaCount: 2 + split: + enabled: true + request: + autoscaling: + enabled: true + template: fusillade/deployment.yaml + asserts: + - hasDocuments: + count: 2 + - isNull: + path: spec.replicas + documentIndex: 0 + - equal: + path: spec.replicas + value: 2 + documentIndex: 1 + + - it: skips HPA for a disabled split role + set: + fusillade: + enabled: true + autoscaling: + enabled: true + split: + enabled: true + request: + enabled: false + template: fusillade/hpa.yaml + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-control-layer-fusillade-batch diff --git a/values.yaml b/values.yaml index 85b36a8..46e3c49 100644 --- a/values.yaml +++ b/values.yaml @@ -57,9 +57,25 @@ secrets: POSTGRES_USER: "clay" POSTGRES_PASSWORD: "clay_password" -# This will set the replicaset count +# This will set the replicaset count. Ignored (and omitted from the rendered +# Deployment) when autoscaling.enabled is true so the HPA owns the count. replicaCount: 1 +# Horizontal Pod Autoscaler for the main control-layer (API) deployment. +# metrics and behavior are raw autoscaling/v2 fields rendered verbatim. +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + metrics: [] + # - type: Resource + # resource: + # name: cpu + # target: + # type: Utilization + # averageUtilization: 75 + behavior: {} + # Opt-in rollout controls for request-serving control-layer pods. Enabling this # keeps replacement capacity available while existing requests drain. rollout: @@ -215,9 +231,21 @@ fusillade: # Set to true to deploy fusillade daemon separately from the control layer enabled: false - # Number of fusillade daemon replicas + # Number of fusillade daemon replicas. Ignored (and omitted from the rendered + # Deployment) for any fusillade workload whose effective autoscaling is enabled. replicaCount: 1 + # Horizontal Pod Autoscaler defaults for fusillade workloads: applies to the + # single daemon deployment and, in split mode, to both the request and batch + # deployments. Override per role via fusillade.split..autoscaling + # (merged over this block). Same schema as the top-level autoscaling block. + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + metrics: [] + behavior: {} + # Optional daemon mode for the standard single fusillade deployment. # Leave empty to use the application's default mode (both). # Split mode overrides this per deployment. @@ -232,11 +260,14 @@ fusillade: # settings such as replicaCount, image, resources, pod metadata, security # context, scheduling, probes, and database pools can be overridden per daemon. # env is merged after global/fusillade env, while volumes and volumeMounts are - # appended to the top-level lists. + # appended to the top-level lists. autoscaling is merged over + # fusillade.autoscaling (keys set here win, e.g. autoscaling.enabled: true + # for one role only). split: enabled: false request: enabled: true + autoscaling: {} image: {} resources: {} podAnnotations: {} @@ -255,6 +286,7 @@ fusillade: volumeMounts: [] batch: enabled: true + autoscaling: {} image: {} resources: {} podAnnotations: {}