feat: ensure server side apply to mitigate the last applied annotation overflow

src/cmd/apply-as-apps.test.ts

@@ -1,19 +1,20 @@
+import { statSync } from 'fs'
+import { glob } from 'glob'
+import { ARGOCD_APP_PARAMS } from '../common/constants'
+import { env } from '../common/envalid'
+import { getNames } from '../common/utils'
 import {
   addGitOpsApps,
   applyArgocdApp,
   applyGitOpsApps,
   ArgocdAppManifest,
   calculateGitOpsAppsSyncState,
+  checkArgoCdController,
   getApplications,
+  getArgocdCoreAppManifest,
   getArgocdGitopsManifest,
-  checkArgoCdController,
   removeGitOpsApps,
 } from './apply-as-apps'
-import { glob } from 'glob'
-import { env } from '../common/envalid'
-import { statSync } from 'fs'
-import { ARGOCD_APP_PARAMS } from '../common/constants'
-import { getNames } from '../common/utils'
 
 jest.mock('glob')
 jest.mock('fs', () => ({
@@ -687,3 +688,34 @@ describe('checkArgoCdController', () => {
     expect(mockRestartStatefulSet).not.toHaveBeenCalled()
   })
 })
+
+describe('getArgocdCoreAppManifest', () => {
+  const release = {
+    name: 'kyverno',
+    namespace: 'kyverno',
+    enabled: true,
+    installed: true,
+    labels: '',
+    chart: '../charts/kyverno',
+    version: '1.0.0',
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    ;(env as any).APPS_REPO_URL = 'https://charts.example.com'
+    ;(env as any).APPS_REVISION = undefined
+  })
+
+  it('should include ServerSideApply=true in syncOptions', () => {
+    const manifest = getArgocdCoreAppManifest(release, {}, '1.0.0')
+
+    expect(manifest.spec.syncPolicy.syncOptions).toContain('ServerSideApply=true')
+  })
+
+  it('should preserve ServerSideApply=true when app has a patch with only ignoreDifferences', () => {
+    const istioBase = { ...release, name: 'istio-base', namespace: 'istio-system' }
+    const manifest = getArgocdCoreAppManifest(istioBase, {}, '1.0.0')
+
+    expect(manifest.spec.syncPolicy.syncOptions).toContain('ServerSideApply=true')
+  })
+})

src/cmd/apply-as-apps.ts

@@ -113,13 +113,14 @@ const getArgoCdAppManifest = (name: string, appLabel: string, spec: Record<strin
   }
 }
 
-const getArgocdCoreAppManifest = (
+export const getArgocdCoreAppManifest = (
   release: HelmRelease,
   values: Record<string, any>,
   otomiVersion: string,
 ): ArgocdAppManifest => {
   const name = getAppName(release)
   const patch = (appPatches[name] || genericPatch) as Record<string, any>
+
   return getArgoCdAppManifest(name, ARGOCD_APP_DEFAULT_LABEL, {
     syncPolicy: ARGOCD_APP_DEFAULT_SYNC_POLICY,
     project: 'default',

src/common/hf.ts

@@ -19,7 +19,7 @@ export const HF_DEFAULT_SYNC_ARGS = [
   '--concurrency=1',
   '--sync-args',
   // These two need to be in same string as is passed as single argument to --sync-args
-  '--disable-openapi-validation --qps=20',
+  '--disable-openapi-validation --qps=20 --serverSide=true',
 ]
 
 export const HF_DEFAULT_SYNC_ON_INITIAL_INSTALL_ARGS = [
@@ -28,7 +28,7 @@ export const HF_DEFAULT_SYNC_ON_INITIAL_INSTALL_ARGS = [
   '--concurrency=1',
   '--sync-args',
   // These two need to be in same string as is passed as single argument to --sync-args
-  '--disable-openapi-validation --qps=20',
+  '--disable-openapi-validation --qps=20 --serverSide=true',
 ]
 
 type HFParams = {

src/common/runtime-upgrades/runtime-upgrades.ts

@@ -1,6 +1,7 @@
 import { OtomiDebugger } from '../debug'
 import { k8s } from '../k8s'
 import { detectAndRestartOutdatedIstioSidecars } from './restart-istio-sidecars'
+import { stripOversizedLastAppliedAnnotations } from './v6.2.0'
 
 export interface RuntimeUpgradeContext {
   debug: OtomiDebugger
@@ -35,4 +36,10 @@ export const runtimeUpgrades: RuntimeUpgrades = [
       },
     },
   },
+  {
+    version: '6.2.0',
+    pre: async ({ debug }) => {
+      await stripOversizedLastAppliedAnnotations().catch((e) => debug.warn('Failed to strip oversized annotations:', e))
+    },
+  },
 ]

src/common/runtime-upgrades/v6.2.0.test.ts

@@ -0,0 +1,82 @@
+import { stripOversizedLastAppliedAnnotations } from './v6.2.0'
+
+jest.mock('../debug', () => ({
+  ...jest.requireActual('../debug'),
+  terminal: jest.fn(() => ({
+    info: jest.fn(),
+    warn: jest.fn(),
+    debug: jest.fn(),
+    error: jest.fn(),
+    stream: { log: process.stdout, error: process.stderr },
+  })),
+}))
+
+describe('stripOversizedLastAppliedAnnotations', () => {
+  const annotation = 'kubectl.kubernetes.io/last-applied-configuration'
+  const oversizedValue = 'x'.repeat(262145)
+
+  const mockListCRDs = jest.fn()
+  const mockPatchCRD = jest.fn()
+  const mockListConfigMaps = jest.fn()
+  const mockPatchConfigMap = jest.fn()
+
+  const mockDeps = {
+    getCrdApi: () => ({ listCustomResourceDefinition: mockListCRDs, patchCustomResourceDefinition: mockPatchCRD }),
+    getCoreApi: () => ({
+      listConfigMapForAllNamespaces: mockListConfigMaps,
+      patchNamespacedConfigMap: mockPatchConfigMap,
+    }),
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    mockListCRDs.mockResolvedValue({ items: [] })
+    mockListConfigMaps.mockResolvedValue({ items: [] })
+    mockPatchCRD.mockResolvedValue({})
+    mockPatchConfigMap.mockResolvedValue({})
+  })
+
+  it('removes last-applied-configuration from a CRD whose annotation exceeds the limit', async () => {
+    mockListCRDs.mockResolvedValue({
+      items: [{ metadata: { name: 'clusterpolicies.kyverno.io', annotations: { [annotation]: oversizedValue } } }],
+    })
+
+    await stripOversizedLastAppliedAnnotations(mockDeps as any)
+
+    expect(mockPatchCRD).toHaveBeenCalledWith(
+      expect.objectContaining({ name: 'clusterpolicies.kyverno.io' }),
+      expect.anything(),
+    )
+  })
+
+  it('removes last-applied-configuration from a ConfigMap whose annotation exceeds the limit', async () => {
+    mockListConfigMaps.mockResolvedValue({
+      items: [
+        {
+          metadata: {
+            name: 'grafana-dashboards-k8s-admin',
+            namespace: 'grafana',
+            annotations: { [annotation]: oversizedValue },
+          },
+        },
+      ],
+    })
+
+    await stripOversizedLastAppliedAnnotations(mockDeps as any)
+
+    expect(mockPatchConfigMap).toHaveBeenCalledWith(
+      expect.objectContaining({ name: 'grafana-dashboards-k8s-admin', namespace: 'grafana' }),
+      expect.anything(),
+    )
+  })
+
+  it('does not patch a ConfigMap without the annotation', async () => {
+    mockListConfigMaps.mockResolvedValue({
+      items: [{ metadata: { name: 'some-config', namespace: 'default', annotations: {} } }],
+    })
+
+    await stripOversizedLastAppliedAnnotations(mockDeps as any)
+
+    expect(mockPatchConfigMap).not.toHaveBeenCalled()
+  })
+})

src/common/runtime-upgrades/v6.2.0.ts

@@ -0,0 +1,49 @@
+import { ApiextensionsV1Api, CoreV1Api, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node'
+import { terminal } from '../debug'
+import { k8s } from '../k8s'
+
+const LAST_APPLIED_ANNOTATION = 'kubectl.kubernetes.io/last-applied-configuration'
+// JSON Patch requires '/' in key names to be escaped as '~1'
+const LAST_APPLIED_PATCH_PATH = '/metadata/annotations/kubectl.kubernetes.io~1last-applied-configuration'
+
+export const stripOversizedLastAppliedAnnotations = async (
+  deps = {
+    getCrdApi: (): ApiextensionsV1Api => k8s.kc().makeApiClient(ApiextensionsV1Api),
+    getCoreApi: (): CoreV1Api => k8s.core(),
+  },
+): Promise<void> => {
+  const log = terminal('common:runtime-upgrades:v6.2.0:stripOversized')
+  const patchHeaders = setHeaderOptions('Content-Type', PatchStrategy.JsonPatch)
+  const removePatch = [{ op: 'remove', path: LAST_APPLIED_PATCH_PATH }]
+
+  const crdApi = deps.getCrdApi()
+  const { items: crds } = await crdApi.listCustomResourceDefinition()
+  await Promise.allSettled(
+    crds
+      .filter((crd) => {
+        const value = crd.metadata?.annotations?.[LAST_APPLIED_ANNOTATION]
+        return value !== undefined
+      })
+      .map(async (crd) => {
+        const name = crd.metadata!.name!
+        log.info(`Stripping oversized last-applied-configuration from CRD ${name}`)
+        await crdApi.patchCustomResourceDefinition({ name, body: removePatch }, patchHeaders)
+      }),
+  )
+
+  const coreApi = deps.getCoreApi()
+  const { items: configMaps } = await coreApi.listConfigMapForAllNamespaces()
+  await Promise.allSettled(
+    configMaps
+      .filter((cm) => {
+        const value = cm.metadata?.annotations?.[LAST_APPLIED_ANNOTATION]
+        return value !== undefined
+      })
+      .map(async (cm) => {
+        const name = cm.metadata!.name!
+        const namespace = cm.metadata!.namespace!
+        log.info(`Stripping oversized last-applied-configuration from ConfigMap ${namespace}/${name}`)
+        await coreApi.patchNamespacedConfigMap({ name, namespace, body: removePatch }, patchHeaders)
+      }),
+  )
+}