From c7fecbf5753322525713c13eab9073cfaa6b1f6d Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Mon, 7 Sep 2026 11:53:18 +0200 Subject: [PATCH] experiment(ci): test higher singleWriterShards/sqlitePoolSize headroom Follow-up to the sharded single-writer validation (kubescape/storage#397): default 8 shards / 10 pool connections reduced failures substantially (15/30 vs 18-19/30 baseline, and no more sustained 20min hangs) but not to near-zero. Residual failures show occasional "sqlite: step: interrupted" errors and context-deadline timeouts under concurrent write bursts, suggesting 8 shards / pool size 10 may not give enough headroom for CI's actual burst size. Testing 16 shards / 24 pool connections via config.json overrides (no new storage image needed -- both are config-driven) to see if more headroom resolves the residual flakiness or if it points to a separate contributing cause. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LLN3CgGftcnAikV13qD6yJ --- .github/workflows/component-tests.yaml | 7 +++++-- tests/chart/templates/storage/configmap.yaml | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index b46c571537..182c15d445 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -153,8 +153,11 @@ jobs: kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus -n monitoring --timeout=300s - name: Install Node Agent Chart run: | - STORAGE_TAG=$(./tests/scripts/storage-tag.sh) - echo "Storage tag that will be used: ${STORAGE_TAG}" + # EXPERIMENT: validating kubescape/storage#397 (sharded single-writer + # fix) with higher shard/pool headroom before merge. Pins storage to + # a manually-built test image. Do not merge this override. + STORAGE_TAG=test-sharded-writer-397b + echo "Storage tag that will be used: ${STORAGE_TAG} (EXPERIMENT override)" helm upgrade --install kubescape ./tests/chart --set clusterName=`kubectl config current-context` --set nodeAgent.image.tag=${{ needs.build-and-push-image.outputs.image_tag }} --set nodeAgent.image.repository=${{ needs.build-and-push-image.outputs.image_repo }} --set storage.image.tag=${STORAGE_TAG} -n kubescape --create-namespace --wait --timeout 10m --debug # Check that the node-agent pod is running kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=node-agent -n kubescape --timeout=600s diff --git a/tests/chart/templates/storage/configmap.yaml b/tests/chart/templates/storage/configmap.yaml index 0dbf866090..0e1809ef1d 100644 --- a/tests/chart/templates/storage/configmap.yaml +++ b/tests/chart/templates/storage/configmap.yaml @@ -9,5 +9,7 @@ metadata: data: config.json: | { - "cleanupInterval": "{{ .Values.storage.cleanupInterval }}" + "cleanupInterval": "{{ .Values.storage.cleanupInterval }}", + "singleWriterShards": 16, + "sqlitePoolSize": 24 }