From 6ff8aad280e9e34955a427876774bad82f2c84d0 Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:14:03 -0400 Subject: [PATCH 1/2] fix(migrate): strip oversized last-applied-configuration annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client-side apply writes the entire object into the `kubectl.kubernetes.io/last-applied-configuration` annotation. For an object carrying a large embedded schema — kyverno's policy CRDs, the ESO/cnpg/Gateway-API CRDs, grafana's dashboard ConfigMaps — that runs 150-250KB and tips past the apiserver's 262144-byte annotations cap, after which *every* subsequent patch to the object fails: CustomResourceDefinition.apiextensions.k8s.io "clusterpolicies.kyverno.io" is invalid: metadata.annotations: Too long: may not be more than 262144 bytes The owning Applications never reach Synced and platform-bootstrap stalls. This reproduces reliably on reused/long-lived clusters and not on fresh ones, which is the tell: core Applications already sync with `ServerSideApply=true`, so nothing rewrites the annotation today — but an object that picked one up under an *earlier* client-side apply keeps it forever, and keeps spending the annotation budget on it. Add a migration that sweeps CRDs and ConfigMaps and removes the annotation wherever it exceeds 100KB. Under server-side apply the annotation is inert, so removing it costs nothing and unwedges clusters carrying one from before. The sweep paginates (100 objects per page) rather than listing cluster-wide in one call, and a failed patch on one object is logged and skipped rather than failing the migration. Refs #3449, #3421 Bumps specVersion to 72. --- helmfile.d/snippets/defaults.yaml | 2 +- src/cmd/migrate.test.ts | 91 ++++++++++++++++++ src/cmd/migrate.ts | 110 ++++++++++++++++++++++ tests/fixtures/env/settings/versions.yaml | 2 +- values-changes.yaml | 3 + 5 files changed, 206 insertions(+), 2 deletions(-) diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index 7d9d569960..872daadadc 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -1165,4 +1165,4 @@ environments: aiEnabled: false users: [] versions: - specVersion: 71 + specVersion: 72 diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 46a4d6ea50..3c9bdfd809 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1,11 +1,13 @@ import { applyChanges, Changes, + escapeJsonPointer, filterChanges, preservePvcStorageClassInRawValues, processDeletionEntry, removeSopsArtifacts, sopsMigration, + stripOversizedLastAppliedConfiguration, } from 'src/cmd/migrate' import { terminal } from '../common/debug' import { env } from '../common/envalid' @@ -589,3 +591,92 @@ describe('preservePvcStorageClassInRawValues', () => { expect(values.databases.keycloak.storageClass).toBe('preset-db-sc') }) }) + +describe('stripOversizedLastAppliedConfiguration', () => { + type StripDeps = NonNullable[1]> + + const annotation = 'kubectl.kubernetes.io/last-applied-configuration' + const oversized = { [annotation]: 'x'.repeat(200 * 1024) } + const small = { [annotation]: 'x'.repeat(1024) } + + const makeDeps = (overrides: Partial = {}): StripDeps => ({ + listCrdPage: jest.fn(async () => ({ items: [] })), + listConfigMapPage: jest.fn(async () => ({ items: [] })), + removeCrdLastApplied: jest.fn(async () => {}), + removeConfigMapLastApplied: jest.fn(async () => {}), + ...overrides, + }) + + it('should strip the annotation only from objects over the threshold', async () => { + const deps = makeDeps({ + listCrdPage: jest.fn(async () => ({ + items: [ + { metadata: { name: 'clusterpolicies.kyverno.io', annotations: oversized } }, + { metadata: { name: 'small.example.io', annotations: small } }, + { metadata: { name: 'unannotated.example.io' } }, + ], + })), + listConfigMapPage: jest.fn(async () => ({ + items: [ + { metadata: { name: 'grafana-dashboards-k8s-admin', namespace: 'grafana', annotations: oversized } }, + { metadata: { name: 'ordinary', namespace: 'grafana', annotations: small } }, + ], + })), + }) + + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(deps.removeCrdLastApplied).toHaveBeenCalledTimes(1) + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('clusterpolicies.kyverno.io') + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledTimes(1) + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'grafana-dashboards-k8s-admin') + }) + + it('should follow pagination until the continue token is exhausted', async () => { + const listCrdPage = jest.fn(async (cont?: string) => + cont === 'page-2' + ? { items: [{ metadata: { name: 'second.example.io', annotations: oversized } }] } + : { + items: [{ metadata: { name: 'first.example.io', annotations: oversized } }], + metadata: { _continue: 'page-2' }, + }, + ) + + const deps = makeDeps({ listCrdPage }) + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(listCrdPage).toHaveBeenCalledTimes(2) + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('first.example.io') + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('second.example.io') + }) + + it('should keep sweeping when one object cannot be patched', async () => { + const deps = makeDeps({ + listCrdPage: jest.fn(async () => ({ + items: [ + { metadata: { name: 'fails.example.io', annotations: oversized } }, + { metadata: { name: 'succeeds.example.io', annotations: oversized } }, + ], + })), + removeCrdLastApplied: jest.fn(async (name: string) => { + if (name === 'fails.example.io') throw new Error('boom') + }), + }) + + await expect(stripOversizedLastAppliedConfiguration({}, deps)).resolves.toBeUndefined() + + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('succeeds.example.io') + }) +}) + +describe('escapeJsonPointer', () => { + it('should escape the slashes in the annotation key per RFC 6901', () => { + expect(escapeJsonPointer('kubectl.kubernetes.io/last-applied-configuration')).toBe( + 'kubectl.kubernetes.io~1last-applied-configuration', + ) + }) + + it('should escape tildes before slashes', () => { + expect(escapeJsonPointer('a~b/c')).toBe('a~0b~1c') + }) +}) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index b0ee0a0ce2..a0d8f39591 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -808,9 +808,119 @@ const removeIngressNginxValues = async (values: Record) => { } } +const LAST_APPLIED_ANNOTATION = 'kubectl.kubernetes.io/last-applied-configuration' +// The apiserver caps metadata.annotations at 262144 bytes in total. Anything holding a +// last-applied-configuration this large is one CRD schema bump away from tipping over it, after +// which *every* patch to that object fails. Well under the cap, well above an ordinary annotation. +const LAST_APPLIED_SIZE_THRESHOLD = 100 * 1024 + +const CRD_PARAMS = { group: 'apiextensions.k8s.io', version: 'v1', plural: 'customresourcedefinitions' } + +type AnnotatedObject = { metadata?: { name?: string; namespace?: string; annotations?: Record } } +type ObjectPage = { items?: AnnotatedObject[]; metadata?: { _continue?: string } } + +const listCrdPage = async (cont?: string): Promise => + (await k8s.custom().listClusterCustomObject({ ...CRD_PARAMS, limit: 100, _continue: cont })) as ObjectPage + +const listConfigMapPage = async (cont?: string): Promise => + (await k8s.core().listConfigMapForAllNamespaces({ limit: 100, _continue: cont })) as ObjectPage + +const removeCrdLastApplied = async (name: string): Promise => { + await k8s.custom().patchClusterCustomObject( + { + ...CRD_PARAMS, + name, + body: [{ op: 'remove', path: `/metadata/annotations/${escapeJsonPointer(LAST_APPLIED_ANNOTATION)}` }], + }, + setHeaderOptions('Content-Type', PatchStrategy.JsonPatch), + ) +} + +const removeConfigMapLastApplied = async (namespace: string, name: string): Promise => { + await k8s.core().patchNamespacedConfigMap( + { + namespace, + name, + body: [{ op: 'remove', path: `/metadata/annotations/${escapeJsonPointer(LAST_APPLIED_ANNOTATION)}` }], + }, + setHeaderOptions('Content-Type', PatchStrategy.JsonPatch), + ) +} + +// RFC 6901: '~' -> '~0', '/' -> '~1'. The annotation key contains slashes. +export const escapeJsonPointer = (token: string): string => token.replace(/~/g, '~0').replace(/\//g, '~1') + +export const isOversized = (object: AnnotatedObject): boolean => + (object.metadata?.annotations?.[LAST_APPLIED_ANNOTATION]?.length ?? 0) > LAST_APPLIED_SIZE_THRESHOLD + +/** + * Client-side apply writes the whole object into the last-applied-configuration annotation. For an + * object with a large embedded schema — kyverno/ESO/cnpg CRDs, grafana dashboard ConfigMaps — that + * runs 150-250KB and tips past the 262144-byte annotations cap, after which every subsequent patch + * to the object fails and the owning Application never reaches Synced. + * + * Core Applications sync with ServerSideApply, which never writes the annotation, so on those + * objects it is inert leftovers from before — but leftovers that still consume the budget. Strip + * them so clusters carrying the annotation from an earlier client-side apply stop wedging. + */ +export const stripOversizedLastAppliedConfiguration = async ( + _values: Record, + deps = { + listCrdPage, + listConfigMapPage, + removeCrdLastApplied, + removeConfigMapLastApplied, + }, +): Promise => { + const d = terminal('stripOversizedLastAppliedConfiguration') + + const parsedArgs = getParsedArgs() + if (parsedArgs?.dryRun || parsedArgs?.local || env.DISABLE_SYNC) { + d.info('Skipping last-applied-configuration cleanup in dry-run/local/dev mode') + return + } + + let stripped = 0 + + const sweep = async ( + kind: string, + listPage: (cont?: string) => Promise, + strip: (object: AnnotatedObject) => Promise, + ): Promise => { + let cont: string | undefined + do { + const page = await listPage(cont) + for (const object of page.items || []) { + if (!isOversized(object)) continue + const name = [object.metadata?.namespace, object.metadata?.name].filter(Boolean).join('/') + try { + await strip(object) + stripped += 1 + d.info(`Removed oversized ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}`) + } catch (error) { + // A 404/422 here means someone else already removed it — never fail the migration over it. + if (error instanceof ApiException && (error.code === 404 || error.code === 422)) continue + d.error(`Could not strip ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}: ${error}`) + } + } + cont = page.metadata?._continue || undefined + } while (cont) + } + + await sweep('CustomResourceDefinition', deps.listCrdPage, (object) => + deps.removeCrdLastApplied(object.metadata!.name!), + ) + await sweep('ConfigMap', deps.listConfigMapPage, (object) => + deps.removeConfigMapLastApplied(object.metadata!.namespace!, object.metadata!.name!), + ) + + d.info(`Stripped ${stripped} oversized ${LAST_APPLIED_ANNOTATION} annotation(s)`) +} + const customMigrationFunctions: Record = { valkeyAndOauth2RedisPVCMigration, preservePvcStorageClassInRawValues, + stripOversizedLastAppliedConfiguration, addLinodeNBAnnotations, sopsMigration, setIngressDefault, diff --git a/tests/fixtures/env/settings/versions.yaml b/tests/fixtures/env/settings/versions.yaml index 625d15e8ea..98e731dc09 100644 --- a/tests/fixtures/env/settings/versions.yaml +++ b/tests/fixtures/env/settings/versions.yaml @@ -2,4 +2,4 @@ kind: AplVersion metadata: name: versions spec: - specVersion: 71 + specVersion: 72 diff --git a/values-changes.yaml b/values-changes.yaml index 31958656ca..fea84ec2ea 100644 --- a/values-changes.yaml +++ b/values-changes.yaml @@ -507,3 +507,6 @@ changes: - version: 71 customFunctions: - preservePvcStorageClassInRawValues + - version: 72 + customFunctions: + - stripOversizedLastAppliedConfiguration From 4f17a338d7c32e8a326aabae7ef5fa81b4abcc0e Mon Sep 17 00:00:00 2001 From: Adam Weingarten <6517820+aweingarten@users.noreply.github.com> Date: Fri, 7 Aug 2026 08:09:13 -0400 Subject: [PATCH 2/2] fix(migrate): page CRDs correctly, and size the annotation in bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CustomObjectsApi deserializes as "any", so a CRD list page keeps the raw `continue` key — only typed models get it renamed to `_continue`. Reading only `_continue` stopped the CRD sweep after the first 100, which is most of them on a cluster that has this problem. Also compare Buffer.byteLength rather than String.length (the apiserver cap is bytes) and hand the error object to the logger instead of interpolating it. --- src/cmd/migrate.test.ts | 38 ++++++++++++++++++++++++++++++++++++-- src/cmd/migrate.ts | 21 ++++++++++++++++----- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 3c9bdfd809..48461288b2 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -632,13 +632,14 @@ describe('stripOversizedLastAppliedConfiguration', () => { expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'grafana-dashboards-k8s-admin') }) - it('should follow pagination until the continue token is exhausted', async () => { + // CustomObjectsApi deserializes as "any", so a CRD page keeps the raw `continue` key. + it('should follow CRD pagination on the raw continue token', async () => { const listCrdPage = jest.fn(async (cont?: string) => cont === 'page-2' ? { items: [{ metadata: { name: 'second.example.io', annotations: oversized } }] } : { items: [{ metadata: { name: 'first.example.io', annotations: oversized } }], - metadata: { _continue: 'page-2' }, + metadata: { continue: 'page-2' }, }, ) @@ -650,6 +651,39 @@ describe('stripOversizedLastAppliedConfiguration', () => { expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('second.example.io') }) + // Typed list models rename `continue` to `_continue` via the generated attributeTypeMap. + it('should follow ConfigMap pagination on the renamed _continue token', async () => { + const listConfigMapPage = jest.fn(async (cont?: string) => + cont === 'page-2' + ? { items: [{ metadata: { name: 'second', namespace: 'grafana', annotations: oversized } }] } + : { + items: [{ metadata: { name: 'first', namespace: 'grafana', annotations: oversized } }], + metadata: { _continue: 'page-2' }, + }, + ) + + const deps = makeDeps({ listConfigMapPage }) + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(listConfigMapPage).toHaveBeenCalledTimes(2) + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'first') + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'second') + }) + + it('should size the annotation in bytes, not UTF-16 code units', async () => { + // 60k multi-byte characters: 60k UTF-16 units (under the 100KiB threshold) but 180k bytes. + const multiByte = { [annotation]: '€'.repeat(60 * 1024) } + const deps = makeDeps({ + listCrdPage: jest.fn(async () => ({ + items: [{ metadata: { name: 'unicode.example.io', annotations: multiByte } }], + })), + }) + + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('unicode.example.io') + }) + it('should keep sweeping when one object cannot be patched', async () => { const deps = makeDeps({ listCrdPage: jest.fn(async () => ({ diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index a0d8f39591..99d180c883 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -817,7 +817,14 @@ const LAST_APPLIED_SIZE_THRESHOLD = 100 * 1024 const CRD_PARAMS = { group: 'apiextensions.k8s.io', version: 'v1', plural: 'customresourcedefinitions' } type AnnotatedObject = { metadata?: { name?: string; namespace?: string; annotations?: Record } } -type ObjectPage = { items?: AnnotatedObject[]; metadata?: { _continue?: string } } +// Typed list models (V1ConfigMapList) are deserialized through the generated attributeTypeMap, +// which renames the reserved word `continue` to `_continue`. CustomObjectsApi deserializes as +// "any", so a CRD page arrives as raw JSON and keeps `continue`. Read both or CRD pagination stops +// after the first page — and a cluster carrying this annotation has well over 100 CRDs. +type ObjectPage = { items?: AnnotatedObject[]; metadata?: { _continue?: string; continue?: string } } + +const continueToken = (page: ObjectPage): string | undefined => + page.metadata?._continue || page.metadata?.continue || undefined const listCrdPage = async (cont?: string): Promise => (await k8s.custom().listClusterCustomObject({ ...CRD_PARAMS, limit: 100, _continue: cont })) as ObjectPage @@ -850,8 +857,12 @@ const removeConfigMapLastApplied = async (namespace: string, name: string): Prom // RFC 6901: '~' -> '~0', '/' -> '~1'. The annotation key contains slashes. export const escapeJsonPointer = (token: string): string => token.replace(/~/g, '~0').replace(/\//g, '~1') -export const isOversized = (object: AnnotatedObject): boolean => - (object.metadata?.annotations?.[LAST_APPLIED_ANNOTATION]?.length ?? 0) > LAST_APPLIED_SIZE_THRESHOLD +// Byte length, not String.length — the apiserver cap is in bytes, and a CRD schema full of +// non-ASCII descriptions would otherwise be undercounted and skipped. +export const isOversized = (object: AnnotatedObject): boolean => { + const annotation = object.metadata?.annotations?.[LAST_APPLIED_ANNOTATION] + return annotation !== undefined && Buffer.byteLength(annotation, 'utf8') > LAST_APPLIED_SIZE_THRESHOLD +} /** * Client-side apply writes the whole object into the last-applied-configuration annotation. For an @@ -900,10 +911,10 @@ export const stripOversizedLastAppliedConfiguration = async ( } catch (error) { // A 404/422 here means someone else already removed it — never fail the migration over it. if (error instanceof ApiException && (error.code === 404 || error.code === 422)) continue - d.error(`Could not strip ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}: ${error}`) + d.error(`Could not strip ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}:`, error) } } - cont = page.metadata?._continue || undefined + cont = continueToken(page) } while (cont) }