fix(migrate): strip oversized last-applied-configuration annotations

helmfile.d/snippets/defaults.yaml

@@ -1165,4 +1165,4 @@ environments:
           aiEnabled: false
         users: []
         versions:
-          specVersion: 71
+          specVersion: 72

src/cmd/migrate.test.ts

@@ -1,11 +1,13 @@
 import {
   applyChanges,
   Changes,
+  escapeJsonPointer,
   filterChanges,
   preservePvcStorageClassInRawValues,
   processDeletionEntry,
   removeSopsArtifacts,
   sopsMigration,
+  stripOversizedLastAppliedConfiguration,
 } from 'src/cmd/migrate'
 import { terminal } from '../common/debug'
 import { env } from '../common/envalid'
@@ -589,3 +591,126 @@ describe('preservePvcStorageClassInRawValues', () => {
     expect(values.databases.keycloak.storageClass).toBe('preset-db-sc')
   })
 })
+
+describe('stripOversizedLastAppliedConfiguration', () => {
+  type StripDeps = NonNullable<Parameters<typeof stripOversizedLastAppliedConfiguration>[1]>
+
+  const annotation = 'kubectl.kubernetes.io/last-applied-configuration'
+  const oversized = { [annotation]: 'x'.repeat(200 * 1024) }
+  const small = { [annotation]: 'x'.repeat(1024) }
+
+  const makeDeps = (overrides: Partial<StripDeps> = {}): StripDeps => ({
+    listCrdPage: jest.fn(async () => ({ items: [] })),
+    listConfigMapPage: jest.fn(async () => ({ items: [] })),
+    removeCrdLastApplied: jest.fn(async () => {}),
+    removeConfigMapLastApplied: jest.fn(async () => {}),
+    ...overrides,
+  })
+
+  it('should strip the annotation only from objects over the threshold', async () => {
+    const deps = makeDeps({
+      listCrdPage: jest.fn(async () => ({
+        items: [
+          { metadata: { name: 'clusterpolicies.kyverno.io', annotations: oversized } },
+          { metadata: { name: 'small.example.io', annotations: small } },
+          { metadata: { name: 'unannotated.example.io' } },
+        ],
+      })),
+      listConfigMapPage: jest.fn(async () => ({
+        items: [
+          { metadata: { name: 'grafana-dashboards-k8s-admin', namespace: 'grafana', annotations: oversized } },
+          { metadata: { name: 'ordinary', namespace: 'grafana', annotations: small } },
+        ],
+      })),
+    })
+
+    await stripOversizedLastAppliedConfiguration({}, deps)
+
+    expect(deps.removeCrdLastApplied).toHaveBeenCalledTimes(1)
+    expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('clusterpolicies.kyverno.io')
+    expect(deps.removeConfigMapLastApplied).toHaveBeenCalledTimes(1)
+    expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'grafana-dashboards-k8s-admin')
+  })
+
+  // CustomObjectsApi deserializes as "any", so a CRD page keeps the raw `continue` key.
+  it('should follow CRD pagination on the raw continue token', async () => {
+    const listCrdPage = jest.fn(async (cont?: string) =>
+      cont === 'page-2'
+        ? { items: [{ metadata: { name: 'second.example.io', annotations: oversized } }] }
+        : {
+            items: [{ metadata: { name: 'first.example.io', annotations: oversized } }],
+            metadata: { continue: 'page-2' },
+          },
+    )
+
+    const deps = makeDeps({ listCrdPage })
+    await stripOversizedLastAppliedConfiguration({}, deps)
+
+    expect(listCrdPage).toHaveBeenCalledTimes(2)
+    expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('first.example.io')
+    expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('second.example.io')
+  })
+
+  // Typed list models rename `continue` to `_continue` via the generated attributeTypeMap.
+  it('should follow ConfigMap pagination on the renamed _continue token', async () => {
+    const listConfigMapPage = jest.fn(async (cont?: string) =>
+      cont === 'page-2'
+        ? { items: [{ metadata: { name: 'second', namespace: 'grafana', annotations: oversized } }] }
+        : {
+            items: [{ metadata: { name: 'first', namespace: 'grafana', annotations: oversized } }],
+            metadata: { _continue: 'page-2' },
+          },
+    )
+
+    const deps = makeDeps({ listConfigMapPage })
+    await stripOversizedLastAppliedConfiguration({}, deps)
+
+    expect(listConfigMapPage).toHaveBeenCalledTimes(2)
+    expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'first')
+    expect(deps.removeConfigMapLastApplied).toHaveBeenCalledWith('grafana', 'second')
+  })
+
+  it('should size the annotation in bytes, not UTF-16 code units', async () => {
+    // 60k multi-byte characters: 60k UTF-16 units (under the 100KiB threshold) but 180k bytes.
+    const multiByte = { [annotation]: '€'.repeat(60 * 1024) }
+    const deps = makeDeps({
+      listCrdPage: jest.fn(async () => ({
+        items: [{ metadata: { name: 'unicode.example.io', annotations: multiByte } }],
+      })),
+    })
+
+    await stripOversizedLastAppliedConfiguration({}, deps)
+
+    expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('unicode.example.io')
+  })
+
+  it('should keep sweeping when one object cannot be patched', async () => {
+    const deps = makeDeps({
+      listCrdPage: jest.fn(async () => ({
+        items: [
+          { metadata: { name: 'fails.example.io', annotations: oversized } },
+          { metadata: { name: 'succeeds.example.io', annotations: oversized } },
+        ],
+      })),
+      removeCrdLastApplied: jest.fn(async (name: string) => {
+        if (name === 'fails.example.io') throw new Error('boom')
+      }),
+    })
+
+    await expect(stripOversizedLastAppliedConfiguration({}, deps)).resolves.toBeUndefined()
+
+    expect(deps.removeCrdLastApplied).toHaveBeenCalledWith('succeeds.example.io')
+  })
+})
+
+describe('escapeJsonPointer', () => {
+  it('should escape the slashes in the annotation key per RFC 6901', () => {
+    expect(escapeJsonPointer('kubectl.kubernetes.io/last-applied-configuration')).toBe(
+      'kubectl.kubernetes.io~1last-applied-configuration',
+    )
+  })
+
+  it('should escape tildes before slashes', () => {
+    expect(escapeJsonPointer('a~b/c')).toBe('a~0b~1c')
+  })
+})

src/cmd/migrate.ts

@@ -808,9 +808,130 @@ const removeIngressNginxValues = async (values: Record<string, any>) => {
   }
 }
 
+const LAST_APPLIED_ANNOTATION = 'kubectl.kubernetes.io/last-applied-configuration'
+// The apiserver caps metadata.annotations at 262144 bytes in total. Anything holding a
+// last-applied-configuration this large is one CRD schema bump away from tipping over it, after
+// which *every* patch to that object fails. Well under the cap, well above an ordinary annotation.
+const LAST_APPLIED_SIZE_THRESHOLD = 100 * 1024
+
+const CRD_PARAMS = { group: 'apiextensions.k8s.io', version: 'v1', plural: 'customresourcedefinitions' }
+
+type AnnotatedObject = { metadata?: { name?: string; namespace?: string; annotations?: Record<string, string> } }
+// Typed list models (V1ConfigMapList) are deserialized through the generated attributeTypeMap,
+// which renames the reserved word `continue` to `_continue`. CustomObjectsApi deserializes as
+// "any", so a CRD page arrives as raw JSON and keeps `continue`. Read both or CRD pagination stops
+// after the first page — and a cluster carrying this annotation has well over 100 CRDs.
+type ObjectPage = { items?: AnnotatedObject[]; metadata?: { _continue?: string; continue?: string } }
+
+const continueToken = (page: ObjectPage): string | undefined =>
+  page.metadata?._continue || page.metadata?.continue || undefined
+
+const listCrdPage = async (cont?: string): Promise<ObjectPage> =>
+  (await k8s.custom().listClusterCustomObject({ ...CRD_PARAMS, limit: 100, _continue: cont })) as ObjectPage
+
+const listConfigMapPage = async (cont?: string): Promise<ObjectPage> =>
+  (await k8s.core().listConfigMapForAllNamespaces({ limit: 100, _continue: cont })) as ObjectPage
+
+const removeCrdLastApplied = async (name: string): Promise<void> => {
+  await k8s.custom().patchClusterCustomObject(
+    {
+      ...CRD_PARAMS,
+      name,
+      body: [{ op: 'remove', path: `/metadata/annotations/${escapeJsonPointer(LAST_APPLIED_ANNOTATION)}` }],
+    },
+    setHeaderOptions('Content-Type', PatchStrategy.JsonPatch),
+  )
+}
+
+const removeConfigMapLastApplied = async (namespace: string, name: string): Promise<void> => {
+  await k8s.core().patchNamespacedConfigMap(
+    {
+      namespace,
+      name,
+      body: [{ op: 'remove', path: `/metadata/annotations/${escapeJsonPointer(LAST_APPLIED_ANNOTATION)}` }],
+    },
+    setHeaderOptions('Content-Type', PatchStrategy.JsonPatch),
+  )
+}
+
+// RFC 6901: '~' -> '~0', '/' -> '~1'. The annotation key contains slashes.
+export const escapeJsonPointer = (token: string): string => token.replace(/~/g, '~0').replace(/\//g, '~1')
+
+// Byte length, not String.length — the apiserver cap is in bytes, and a CRD schema full of
+// non-ASCII descriptions would otherwise be undercounted and skipped.
+export const isOversized = (object: AnnotatedObject): boolean => {
+  const annotation = object.metadata?.annotations?.[LAST_APPLIED_ANNOTATION]
+  return annotation !== undefined && Buffer.byteLength(annotation, 'utf8') > LAST_APPLIED_SIZE_THRESHOLD
+}
+
+/**
+ * Client-side apply writes the whole object into the last-applied-configuration annotation. For an
+ * object with a large embedded schema — kyverno/ESO/cnpg CRDs, grafana dashboard ConfigMaps — that
+ * runs 150-250KB and tips past the 262144-byte annotations cap, after which every subsequent patch
+ * to the object fails and the owning Application never reaches Synced.
+ *
+ * Core Applications sync with ServerSideApply, which never writes the annotation, so on those
+ * objects it is inert leftovers from before — but leftovers that still consume the budget. Strip
+ * them so clusters carrying the annotation from an earlier client-side apply stop wedging.
+ */
+export const stripOversizedLastAppliedConfiguration = async (
+  _values: Record<string, any>,
+  deps = {
+    listCrdPage,
+    listConfigMapPage,
+    removeCrdLastApplied,
+    removeConfigMapLastApplied,
+  },
+): Promise<void> => {
+  const d = terminal('stripOversizedLastAppliedConfiguration')
+
+  const parsedArgs = getParsedArgs()
+  if (parsedArgs?.dryRun || parsedArgs?.local || env.DISABLE_SYNC) {
+    d.info('Skipping last-applied-configuration cleanup in dry-run/local/dev mode')
+    return
+  }
+
+  let stripped = 0
+
+  const sweep = async (
+    kind: string,
+    listPage: (cont?: string) => Promise<ObjectPage>,
+    strip: (object: AnnotatedObject) => Promise<void>,
+  ): Promise<void> => {
+    let cont: string | undefined
+    do {
+      const page = await listPage(cont)
+      for (const object of page.items || []) {
+        if (!isOversized(object)) continue
+        const name = [object.metadata?.namespace, object.metadata?.name].filter(Boolean).join('/')
+        try {
+          await strip(object)
+          stripped += 1
+          d.info(`Removed oversized ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}`)
+        } catch (error) {
+          // A 404/422 here means someone else already removed it — never fail the migration over it.
+          if (error instanceof ApiException && (error.code === 404 || error.code === 422)) continue
+          d.error(`Could not strip ${LAST_APPLIED_ANNOTATION} from ${kind} ${name}:`, error)
+        }
+      }
+      cont = continueToken(page)
+    } while (cont)
+  }
+
+  await sweep('CustomResourceDefinition', deps.listCrdPage, (object) =>
+    deps.removeCrdLastApplied(object.metadata!.name!),
+  )
+  await sweep('ConfigMap', deps.listConfigMapPage, (object) =>
+    deps.removeConfigMapLastApplied(object.metadata!.namespace!, object.metadata!.name!),
+  )
+
+  d.info(`Stripped ${stripped} oversized ${LAST_APPLIED_ANNOTATION} annotation(s)`)
+}
+
 const customMigrationFunctions: Record<string, CustomMigrationFunction> = {
   valkeyAndOauth2RedisPVCMigration,
   preservePvcStorageClassInRawValues,
+  stripOversizedLastAppliedConfiguration,
   addLinodeNBAnnotations,
   sopsMigration,
   setIngressDefault,

tests/fixtures/env/settings/versions.yaml

@@ -2,4 +2,4 @@ kind: AplVersion
 metadata:
     name: versions
 spec:
-    specVersion: 71
+    specVersion: 72

values-changes.yaml

@@ -507,3 +507,6 @@ changes:
   - version: 71
     customFunctions:
       - preservePvcStorageClassInRawValues
+  - version: 72
+    customFunctions:
+      - stripOversizedLastAppliedConfiguration