diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index 7d9d569960..872daadadc 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -1165,4 +1165,4 @@ environments: aiEnabled: false users: [] versions: - specVersion: 71 + specVersion: 72 diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 46a4d6ea50..48461288b2 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1,11 +1,13 @@ import { applyChanges, Changes, + escapeJsonPointer, filterChanges, preservePvcStorageClassInRawValues, processDeletionEntry, removeSopsArtifacts, sopsMigration, + stripOversizedLastAppliedConfiguration, } from 'src/cmd/migrate' import { terminal } from '../common/debug' import { env } from '../common/envalid' @@ -589,3 +591,126 @@ describe('preservePvcStorageClassInRawValues', () => { expect(values.databases.keycloak.storageClass).toBe('preset-db-sc') }) }) + +describe('stripOversizedLastAppliedConfiguration', () => { + type StripDeps = NonNullable[1]> + + const annotation = 'kubectl.kubernetes.io/last-applied-configuration' + const oversized = { [annotation]: 'x'.repeat(200 * 1024) } + const small = { [annotation]: 'x'.repeat(1024) } + + const makeDeps = (overrides: Partial = {}): StripDeps => ({ + listCrdPage: jest.fn(async () => ({ items: [] })), + listConfigMapPage: jest.fn(async () => ({ items: [] })), + removeCrdLastApplied: jest.fn(async () => {}), + removeConfigMapLastApplied: jest.fn(async () => {}), + ...overrides, + }) + + it('should strip the annotation only from objects over the threshold', async () => { + const deps = makeDeps({ + listCrdPage: jest.fn(async () => ({ + items: [ + { metadata: { name: 'clusterpolicies.kyverno.io', annotations: oversized } }, + { metadata: { name: 'small.example.io', annotations: small } }, + { metadata: { name: 'unannotated.example.io' } }, + ], + })), + listConfigMapPage: jest.fn(async () => ({ + items: [ + { metadata: { name: 'grafana-dashboards-k8s-admin', namespace: 'grafana', annotations: oversized } }, + { metadata: { name: 'ordinary', namespace: 'grafana', annotations: small } }, + ], + })), + }) + + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(deps.removeCrdLastApplied).toHaveBeenCalledTimes(1) + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('clusterpolicies.kyverno.io') + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledTimes(1) + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'grafana-dashboards-k8s-admin') + }) + + // CustomObjectsApi deserializes as "any", so a CRD page keeps the raw `continue` key. + it('should follow CRD pagination on the raw continue token', async () => { + const listCrdPage = jest.fn(async (cont?: string) => + cont === 'page-2' + ? { items: [{ metadata: { name: 'second.example.io', annotations: oversized } }] } + : { + items: [{ metadata: { name: 'first.example.io', annotations: oversized } }], + metadata: { continue: 'page-2' }, + }, + ) + + const deps = makeDeps({ listCrdPage }) + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(listCrdPage).toHaveBeenCalledTimes(2) + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('first.example.io') + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('second.example.io') + }) + + // Typed list models rename `continue` to `_continue` via the generated attributeTypeMap. + it('should follow ConfigMap pagination on the renamed _continue token', async () => { + const listConfigMapPage = jest.fn(async (cont?: string) => + cont === 'page-2' + ? { items: [{ metadata: { name: 'second', namespace: 'grafana', annotations: oversized } }] } + : { + items: [{ metadata: { name: 'first', namespace: 'grafana', annotations: oversized } }], + metadata: { _continue: 'page-2' }, + }, + ) + + const deps = makeDeps({ listConfigMapPage }) + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(listConfigMapPage).toHaveBeenCalledTimes(2) + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'first') + expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'second') + }) + + it('should size the annotation in bytes, not UTF-16 code units', async () => { + // 60k multi-byte characters: 60k UTF-16 units (under the 100KiB threshold) but 180k bytes. + const multiByte = { [annotation]: '€'.repeat(60 * 1024) } + const deps = makeDeps({ + listCrdPage: jest.fn(async () => ({ + items: [{ metadata: { name: 'unicode.example.io', annotations: multiByte } }], + })), + }) + + await stripOversizedLastAppliedConfiguration({}, deps) + + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('unicode.example.io') + }) + + it('should keep sweeping when one object cannot be patched', async () => { + const deps = makeDeps({ + listCrdPage: jest.fn(async () => ({ + items: [ + { metadata: { name: 'fails.example.io', annotations: oversized } }, + { metadata: { name: 'succeeds.example.io', annotations: oversized } }, + ], + })), + removeCrdLastApplied: jest.fn(async (name: string) => { + if (name === 'fails.example.io') throw new Error('boom') + }), + }) + + await expect(stripOversizedLastAppliedConfiguration({}, deps)).resolves.toBeUndefined() + + expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('succeeds.example.io') + }) +}) + +describe('escapeJsonPointer', () => { + it('should escape the slashes in the annotation key per RFC 6901', () => { + expect(escapeJsonPointer('kubectl.kubernetes.io/last-applied-configuration')).toBe( + 'kubectl.kubernetes.io~1last-applied-configuration', + ) + }) + + it('should escape tildes before slashes', () => { + expect(escapeJsonPointer('a~b/c')).toBe('a~0b~1c') + }) +}) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index b0ee0a0ce2..99d180c883 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -808,9 +808,130 @@ const removeIngressNginxValues = async (values: Record) => { } } +const LAST_APPLIED_ANNOTATION = 'kubectl.kubernetes.io/last-applied-configuration' +// The apiserver caps metadata.annotations at 262144 bytes in total. Anything holding a +// last-applied-configuration this large is one CRD schema bump away from tipping over it, after +// which *every* patch to that object fails. Well under the cap, well above an ordinary annotation. +const LAST_APPLIED_SIZE_THRESHOLD = 100 * 1024 + +const CRD_PARAMS = { group: 'apiextensions.k8s.io', version: 'v1', plural: 'customresourcedefinitions' } + +type AnnotatedObject = { metadata?: { name?: string; namespace?: string; annotations?: Record } } +// Typed list models (V1ConfigMapList) are deserialized through the generated attributeTypeMap, +// which renames the reserved word `continue` to `_continue`. CustomObjectsApi deserializes as +// "any", so a CRD page arrives as raw JSON and keeps `continue`. Read both or CRD pagination stops +// after the first page — and a cluster carrying this annotation has well over 100 CRDs. +type ObjectPage = { items?: AnnotatedObject[]; metadata?: { _continue?: string; continue?: string } } + +const continueToken = (page: ObjectPage): string | undefined => + page.metadata?._continue || page.metadata?.continue || undefined + +const listCrdPage = async (cont?: string): Promise => + (await k8s.custom().listClusterCustomObject({ ...CRD_PARAMS, limit: 100, _continue: cont })) as ObjectPage + +const listConfigMapPage = async (cont?: string): Promise => + (await k8s.core().listConfigMapForAllNamespaces({ limit: 100, _continue: cont })) as ObjectPage + +const removeCrdLastApplied = async (name: string): Promise => { + await k8s.custom().patchClusterCustomObject( + { + ...CRD_PARAMS, + name, + body: [{ op: 'remove', path: `/metadata/annotations/${escapeJsonPointer(LAST_APPLIED_ANNOTATION)}` }], + }, + setHeaderOptions('Content-Type', PatchStrategy.JsonPatch), + ) +} + +const removeConfigMapLastApplied = async (namespace: string, name: string): Promise => { + await k8s.core().patchNamespacedConfigMap( + { + namespace, + name, + body: [{ op: 'remove', path: `/metadata/annotations/${escapeJsonPointer(LAST_APPLIED_ANNOTATION)}` }], + }, + setHeaderOptions('Content-Type', PatchStrategy.JsonPatch), + ) +} + +// RFC 6901: '~' -> '~0', '/' -> '~1'. The annotation key contains slashes. +export const escapeJsonPointer = (token: string): string => token.replace(/~/g, '~0').replace(/\//g, '~1') + +// Byte length, not String.length — the apiserver cap is in bytes, and a CRD schema full of +// non-ASCII descriptions would otherwise be undercounted and skipped. +export const isOversized = (object: AnnotatedObject): boolean => { + const annotation = object.metadata?.annotations?.[LAST_APPLIED_ANNOTATION] + return annotation !== undefined && Buffer.byteLength(annotation, 'utf8') > LAST_APPLIED_SIZE_THRESHOLD +} + +/** + * Client-side apply writes the whole object into the last-applied-configuration annotation. For an + * object with a large embedded schema — kyverno/ESO/cnpg CRDs, grafana dashboard ConfigMaps — that + * runs 150-250KB and tips past the 262144-byte annotations cap, after which every subsequent patch + * to the object fails and the owning Application never reaches Synced. + * + * Core Applications sync with ServerSideApply, which never writes the annotation, so on those + * objects it is inert leftovers from before — but leftovers that still consume the budget. Strip + * them so clusters carrying the annotation from an earlier client-side apply stop wedging. + */ +export const stripOversizedLastAppliedConfiguration = async ( + _values: Record, + deps = { + listCrdPage, + listConfigMapPage, + removeCrdLastApplied, + removeConfigMapLastApplied, + }, +): Promise => { + const d = terminal('stripOversizedLastAppliedConfiguration') + + const parsedArgs = getParsedArgs() + if (parsedArgs?.dryRun || parsedArgs?.local || env.DISABLE_SYNC) { + d.info('Skipping last-applied-configuration cleanup in dry-run/local/dev mode') + return + } + + let stripped = 0 + + const sweep = async ( + kind: string, + listPage: (cont?: string) => Promise, + strip: (object: AnnotatedObject) => Promise, + ): Promise => { + let cont: string | undefined + do { + const page = await listPage(cont) + for (const object of page.items || []) { + if (!isOversized(object)) continue + const name = [object.metadata?.namespace, object.metadata?.name].filter(Boolean).join('/') + try { + await strip(object) + stripped += 1 + d.info(`Removed oversized ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}`) + } catch (error) { + // A 404/422 here means someone else already removed it — never fail the migration over it. + if (error instanceof ApiException && (error.code === 404 || error.code === 422)) continue + d.error(`Could not strip ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}:`, error) + } + } + cont = continueToken(page) + } while (cont) + } + + await sweep('CustomResourceDefinition', deps.listCrdPage, (object) => + deps.removeCrdLastApplied(object.metadata!.name!), + ) + await sweep('ConfigMap', deps.listConfigMapPage, (object) => + deps.removeConfigMapLastApplied(object.metadata!.namespace!, object.metadata!.name!), + ) + + d.info(`Stripped ${stripped} oversized ${LAST_APPLIED_ANNOTATION} annotation(s)`) +} + const customMigrationFunctions: Record = { valkeyAndOauth2RedisPVCMigration, preservePvcStorageClassInRawValues, + stripOversizedLastAppliedConfiguration, addLinodeNBAnnotations, sopsMigration, setIngressDefault, diff --git a/tests/fixtures/env/settings/versions.yaml b/tests/fixtures/env/settings/versions.yaml index 625d15e8ea..98e731dc09 100644 --- a/tests/fixtures/env/settings/versions.yaml +++ b/tests/fixtures/env/settings/versions.yaml @@ -2,4 +2,4 @@ kind: AplVersion metadata: name: versions spec: - specVersion: 71 + specVersion: 72 diff --git a/values-changes.yaml b/values-changes.yaml index 31958656ca..fea84ec2ea 100644 --- a/values-changes.yaml +++ b/values-changes.yaml @@ -507,3 +507,6 @@ changes: - version: 71 customFunctions: - preservePvcStorageClassInRawValues + - version: 72 + customFunctions: + - stripOversizedLastAppliedConfiguration