Restore fork infra (rebased onto current main) — supersedes #72

.arclint

@@ -20,6 +20,7 @@
     "(^private\/credentials\/.*\\.yaml)",
     "(^src/operator/client/versioned/)",
     "(^src/operator/apis/px.dev/v1alpha1/zz_generated.deepcopy.go)",
+    "(^src/e2e_test/adaptive_export_loadtest/tools/loadgen/)",
     "(^src/stirling/bpf_tools/bcc_bpf/system-headers)",
     "(^src/stirling/mysql/testing/.*\\.json$)",
     "(^src/stirling/obj_tools/testdata/go/test_go_binary.go)",

.github/workflows/e2e_calibrate_soc.yaml

@@ -0,0 +1,111 @@
+---
+name: e2e-calibration-soc
+on:
+  workflow_dispatch:
+    inputs:
+      dx_image:
+        description: dx-daemon image to test (default = .image-tags pin)
+        required: false
+        default: ""
+      soc_ref:
+        description: k8sstormcenter/soc branch
+        required: false
+        default: "main"
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: oracle-vm-16cpu-64gb-x86-64   
+    timeout-minutes: 90
+    env:
+      KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+      HARNESS: src/e2e_test/adaptive_export_loadtest/harness
+    steps:
+    - name: Checkout pixie (harness scripts)
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+
+    - name: Install k3s
+      run: |
+        curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
+        for i in $(seq 1 60); do kubectl get nodes --no-headers 2>/dev/null | grep -q ' Ready' && break; sleep 5; done
+        kubectl get nodes
+
+    - name: Deploy the SOC stack (Pixie + kubescape + ClickHouse + AE + dx + chain)
+      env:
+        PX_CLOUD_ADDR: pixie.austrianopencloudcommunity.org
+        PX_DEPLOY_KEY: ${{ secrets.PX_DEPLOY_KEY }}
+        PX_API_KEY: ${{ secrets.PX_API_KEY }}
+        TS_AUTHKEY: ${{ secrets.TAILSCALE_AUTH_KEY }}          
+        CLICKHOUSE_ANALYST_PASSWORD: ${{ secrets.CLICKHOUSE_ANALYST_PASSWORD }}
+        CLICKHOUSE_INGEST_PASSWORD: ${{ secrets.CLICKHOUSE_INGEST_PASSWORD }}
+        CLICKHOUSE_PIXIE_PASSWORD: ${{ secrets.CLICKHOUSE_PIXIE_PASSWORD }}
+      run: |
+        set -euo pipefail
+        sudo apt-get update -qq && sudo apt-get install -y python3-yaml
+        git clone --depth 1 -b "${{ inputs.soc_ref }}" https://github.com/k8sstormcenter/soc soc
+        cd soc
+        make pixie                              # vizier + AE
+        make kubescape || true                  # node-agent (netStreaming)
+        bash tree/clickhouse-lab/install.sh      # forensic_db
+        make java-poc                          
+        if [ -n "${{ inputs.dx_image }}" ]; then
+          kubectl -n honey set image ds/dx-daemon dx-daemon="${{ inputs.dx_image }}" || true
+        fi
+        # optimal config + enable pprof for the real-life profile (DX_TELEMETRY_CACHE/DX_BENCH
+        # are defaults in main, set here too in case the kit's manifest predates them)
+        kubectl -n honey set env ds/dx-daemon DX_PPROF_ADDR=0.0.0.0:6060 DX_TELEMETRY_CACHE=1 DX_BENCH=pemdirect
+        kubectl -n honey rollout status ds/dx-daemon --timeout=120s
+
+    - name: Wait for stack healthy
+      run: |
+        set -euo pipefail
+        kubectl wait --for=condition=Ready pod -l name=adaptive-export -n pl --timeout=300s
+        kubectl wait --for=condition=Ready pod -l app=dx-daemon -n honey --timeout=300s
+        kubectl -n pl get pods; kubectl -n honey get pods
+        # dx must be non-blind on pemdirect (the optimal default from #29/#33)
+        kubectl -n honey logs ds/dx-daemon | grep -E "bench=pemdirect|telemetry cache ENABLED" | head
+
+    - name: Run canonical harness scripts — assert each actually runs
+      run: |
+        set -uo pipefail
+        mkdir -p /tmp/evidence; fail=0
+        for s in javapoc_fire exp_matrix nfr exp_row_reconcile; do
+          echo "::group::$s"
+          if bash "$HARNESS/$s.sh" > "/tmp/evidence/$s.log" 2>&1; then
+            echo "PASS $s"; tail -5 "/tmp/evidence/$s.log"
+          else
+            echo "FAIL $s (exit $?)"; tail -30 "/tmp/evidence/$s.log"; fail=1
+          fi
+          echo "::endgroup::"
+        done
+        kubectl -n honey logs ds/dx-daemon | grep -iE "RULE IN|ruled_in" | tee /tmp/evidence/dx_ruleins.txt
+
+
+    - name: Profile dx in real life (pprof + metrics)
+      if: always()
+      run: |
+        set -uo pipefail
+        POD=$(kubectl -n honey get pod -l app=dx-daemon -o jsonpath='{.items[0].metadata.name}')
+        kubectl -n honey port-forward "$POD" 6060:6060 9095:9095 & PF=$!; sleep 5
+        # 30s CPU profile under a fresh fire + heap, served by DX_PPROF_ADDR=:6060
+        ( bash "$HARNESS/javapoc_fire.sh" >/dev/null 2>&1 || true ) &
+        curl -s --max-time 40 -o /tmp/evidence/dx_cpu.pprof \
+          "http://127.0.0.1:6060/debug/pprof/profile?seconds=30" || true
+        curl -s "http://127.0.0.1:6060/debug/pprof/heap" -o /tmp/evidence/dx_heap.pprof || true
+        curl -s "http://127.0.0.1:9095/metrics" -o /tmp/evidence/dx_metrics.txt || true
+        go tool pprof -top -nodecount=25 /tmp/evidence/dx_cpu.pprof > /tmp/evidence/dx_cpu_top.txt 2>&1 || true
+        kill $PF 2>/dev/null || true
+        echo "=== dx CPU top ==="; head -30 /tmp/evidence/dx_cpu_top.txt
+        echo "=== verdict latency ==="
+        grep -E \
+          "dx_(time_to_verdict|bench_query_duration)_seconds_(sum|count)" \
+          /tmp/evidence/dx_metrics.txt || true
+
+    - name: Upload evidence + profiles
+      if: always()
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882  # v4.4.3
+      with:
+        name: e2e-calibration-soc
+        path: /tmp/evidence/
+        retention-days: 14

.github/workflows/filename_linter.yaml

@@ -15,13 +15,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-    - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36  # v3.0.2
-      id: changes
       with:
-        filters: |
-          private:
-          - '**/*private*/**'
-          - '**/*private*'
-    - name: Fail on private
-      if: ${{ steps.changes.outputs.private == 'true' }}
-      run: echo "This repo disallows dirnames or filenames with 'private' in it." && exit 1
+        fetch-depth: 0
+    - name: Fail on disallowed 'private' paths
+      env:
+        BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+      # Upstream forbids any path containing 'private' (copybara-excluded in OSS).
+      # This fork intentionally keeps fork-only infra under private/ and
+      # tools/private/, so those roots are allowed; the guard still blocks
+      # accidental new 'private' paths introduced anywhere else.
+      run: |
+        offending="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" \
+          | grep -i 'private' \
+          | grep -vE '^(private/|tools/private/)' || true)"
+        if [ -n "$offending" ]; then
+          echo "This repo disallows dirnames or filenames with 'private' in it"
+          echo "(outside the fork infra roots private/ and tools/private/):"
+          echo "$offending"
+          exit 1
+        fi
+        echo "OK: no disallowed 'private' paths."

.sops.yaml

@@ -0,0 +1,5 @@
+---
+# az login -t 1e8a0c86-2410-4f48-91c3-dc1d164680ca
+creation_rules:
+- path_regex: terraform/credentials/cockpit
+  azure_keyvault: https://kv-pixie-cloud.vault.azure.net/keys/sops-key

private/cockpit/artifact_config.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pl-artifact-config
+data:
+  PL_ARTIFACT_MANIFEST_URL: https://k8sstormcenter.github.io/pixie/artifacts/manifest.json
+  PL_ARTIFACT_BUCKET: ""
+  PL_SA_KEY_PATH: ""

private/cockpit/auth_deployment.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: auth-server
+  labels:
+    db: pgsql
+spec:
+  template:
+    spec:
+      containers:
+      - name: auth-server
+        env:
+        - name: PL_AUTH0_CLIENT_ID
+          valueFrom:
+            secretKeyRef:
+              name: cloud-auth0-secrets
+              key: auth0-client-id
+              optional: true
+        - name: PL_AUTH0_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: cloud-auth0-secrets
+              key: auth0-client-secret

private/cockpit/cloud_ingress.yaml

@@ -0,0 +1,79 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: cloud-ingress
+  namespace: plc
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: >-
+      test.austrianopencloudcommunity.org,work.test.austrianopencloudcommunity.org
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+spec:
+  tls:
+  - hosts:
+    - test.austrianopencloudcommunity.org
+    - work.test.austrianopencloudcommunity.org
+    secretName: cloud-proxy-tls-certs
+  rules:
+  - host: test.austrianopencloudcommunity.org
+    http:
+      paths:
+      - path: /px.services
+        pathType: Prefix
+        backend:
+          service:
+            name: vzconn-service
+            port:
+              number: 51600
+      - path: /px.cloudapi
+        pathType: Prefix
+        backend:
+          service:
+            name: api-service
+            port:
+              number: 51200
+      - path: /px.api
+        pathType: Prefix
+        backend:
+          service:
+            name: cloud-proxy-service
+            port:
+              number: 4444
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: cloud-proxy-service
+            port:
+              number: 443
+  - host: work.test.austrianopencloudcommunity.org
+    http:
+      paths:
+      - path: /px.services
+        pathType: Prefix
+        backend:
+          service:
+            name: vzconn-service
+            port:
+              number: 51600
+      - path: /px.cloudapi
+        pathType: Prefix
+        backend:
+          service:
+            name: api-service
+            port:
+              number: 51200
+      - path: /px.api
+        pathType: Prefix
+        backend:
+          service:
+            name: cloud-proxy-service
+            port:
+              number: 4444
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: cloud-proxy-service
+            port:
+              number: 443

private/cockpit/domain_config.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pl-domain-config
+data:
+  PL_DOMAIN_NAME: test.austrianopencloudcommunity.org
+  PASSTHROUGH_PROXY_PORT: ""

private/cockpit/kustomization.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: plc
+replicas:
+- name: api-server
+  count: 1
+- name: auth-server
+  count: 1
+- name: profile-server
+  count: 1
+- name: cloud-proxy
+  count: 1
+- name: project-manager-server
+  count: 1
+- name: vzmgr-server
+  count: 1
+- name: scriptmgr-server
+  count: 1
+resources:
+- ../../k8s/cloud/base
+- plugin_db_updater_job.yaml
+- oauth_config.yaml
+components:
+- ../../k8s/cloud/overlays/exposed_services_traefik
+labels:
+- includeSelectors: true
+  pairs:
+    app: pl-cloud
+patches:
+- path: artifact_config.yaml
+- path: auth_deployment.yaml
+- path: domain_config.yaml
+- path: script_bundles_config.yaml
+- path: proxy_envoy.yaml
+- path: cloud_ingress.yaml
+  target:
+    kind: Ingress
+    name: cloud-ingress
+- path: servers_transport.yaml
+  target:
+    group: traefik.io
+    version: v1alpha1
+    kind: ServersTransport
+    name: cloud-backend-transport

private/cockpit/oauth_config.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pl-oauth-config
+data:
+  PL_OAUTH_PROVIDER: auth0
+  PL_AUTH_URI: dev-ylinrwd1tckmhm03.us.auth0.com
+  PL_AUTH_CLIENT_ID: rDOrqnuIaTsXiya5I41b1FkNzXOQfWbW
+  PL_AUTH_EMAIL_PASSWORD_CONN: Username-Password-Authentication