From 9ca91be0d31fda2aa9024c28fc30206a03dfba37 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 25 Mar 2026 16:58:03 +0100 Subject: [PATCH 1/9] feat(ledger): add v3 reconciler with Raft StatefulSet support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ledger v3 uses Raft consensus with Pebble embedded storage instead of PostgreSQL. This adds a version-branched reconciler that creates a StatefulSet (with headless service for peer discovery) when the ledger version is >= v3.0.0-alpha. Key changes: - Version gate in Reconcile(): v3+ skips Database/migrations entirely - StatefulSet with OrderedReady policy, 3 PVCs (wal, data, cold-cache) - Headless service (ledger-raft) for Raft peer DNS discovery - ClusterIP service mapping port 8080→9000 for gateway compatibility - Pod entrypoint script: computes node-id from ordinal, bootstrap/join - Settings-driven: replicas, PVC sizes, storage classes, Pebble/Raft tunables - RBAC marker for apps/statefulsets Co-Authored-By: Claude Opus 4.6 --- internal/resources/ledgers/init.go | 7 + internal/resources/ledgers/v3.go | 394 +++++++++++++++++++++++++++++ 2 files changed, 401 insertions(+) create mode 100644 internal/resources/ledgers/v3.go diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index 53c6e6619..dc64ec7f2 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -38,8 +38,14 @@ import ( //+kubebuilder:rbac:groups=formance.com,resources=ledgers/status,verbs=get;update;patch //+kubebuilder:rbac:groups=formance.com,resources=ledgers/finalizers,verbs=update //+kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { + isV3 := !semver.IsValid(version) || semver.Compare(version, "v3.0.0-alpha") >= 0 + if isV3 { + return reconcileV3(ctx, stack, ledger, version) + } + database, err := databases.Create(ctx, stack, ledger) if err != nil { return err @@ -111,6 +117,7 @@ func init() { Init( WithModuleReconciler(Reconcile, WithOwn[*v1beta1.Ledger](&appsv1.Deployment{}), + WithOwn[*v1beta1.Ledger](&appsv1.StatefulSet{}), WithOwn[*v1beta1.Ledger](&batchv1.Job{}), WithOwn[*v1beta1.Ledger](&corev1.Service{}), WithOwn[*v1beta1.Ledger](&v1beta1.GatewayHTTPAPI{}), diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go new file mode 100644 index 000000000..254797aa3 --- /dev/null +++ b/internal/resources/ledgers/v3.go @@ -0,0 +1,394 @@ +package ledgers + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" + "github.com/formancehq/operator/v3/internal/resources/registries" + "github.com/formancehq/operator/v3/internal/resources/services" + "github.com/formancehq/operator/v3/internal/resources/settings" +) + +const ( + v3PortHTTP = int32(9000) + v3PortGRPC = int32(8888) + v3PortRaft = int32(7777) +) + +func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { + imageConfiguration, err := registries.GetFormanceImage(ctx, stack, "ledger", version) + if err != nil { + return err + } + + if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("health")); err != nil { + return err + } + + if err := createV3HeadlessService(ctx, stack, ledger); err != nil { + return err + } + + // ClusterIP service: port 8080 → container port 9000 (gateway compatibility) + if _, err := services.Create(ctx, ledger, "ledger", services.WithConfig(services.PortConfig{ + ServiceName: "ledger", + PortName: "http", + Port: 8080, + TargetPort: "http", + })); err != nil { + return err + } + + if err := installV3StatefulSet(ctx, stack, ledger, imageConfiguration); err != nil { + return err + } + + return nil +} + +func createV3HeadlessService(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger) error { + headlessSvcName := "ledger-raft" + + _, _, err := core.CreateOrUpdate[*corev1.Service](ctx, types.NamespacedName{ + Name: headlessSvcName, + Namespace: stack.Name, + }, + func(t *corev1.Service) error { + t.Spec = corev1.ServiceSpec{ + ClusterIP: "None", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "raft", + Port: v3PortRaft, + Protocol: "TCP", + TargetPort: intstr.FromString("raft"), + }, + { + Name: "grpc", + Port: v3PortGRPC, + Protocol: "TCP", + TargetPort: intstr.FromString("grpc"), + }, + }, + Selector: map[string]string{ + "app.kubernetes.io/name": "ledger", + }, + } + return nil + }, + core.WithController[*corev1.Service](ctx.GetScheme(), ledger), + ) + return err +} + +func installV3StatefulSet(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, image *registries.ImageConfiguration) error { + stackName := stack.Name + + replicas, err := settings.GetInt32OrDefault(ctx, stackName, 3, "ledger", "v3", "replicas") + if err != nil { + return err + } + if replicas%2 == 0 { + return fmt.Errorf("ledger.v3.replicas must be odd, got %d", replicas) + } + + volumeClaims, err := buildV3VolumeClaimTemplates(ctx, stackName) + if err != nil { + return err + } + + podTemplate, err := buildV3PodTemplate(ctx, stack, ledger, image) + if err != nil { + return err + } + + headlessSvcName := "ledger-raft" + stsName := "ledger" + + _, _, err = core.CreateOrUpdate[*appsv1.StatefulSet](ctx, types.NamespacedName{ + Name: stsName, + Namespace: stackName, + }, + func(t *appsv1.StatefulSet) error { + t.Spec = appsv1.StatefulSetSpec{ + Replicas: &replicas, + ServiceName: headlessSvcName, + PodManagementPolicy: appsv1.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "ledger", + }, + }, + Template: *podTemplate, + VolumeClaimTemplates: volumeClaims, + } + return nil + }, + core.WithController[*appsv1.StatefulSet](ctx.GetScheme(), ledger), + ) + return err +} + +func buildV3PodTemplate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, image *registries.ImageConfiguration) (*corev1.PodTemplateSpec, error) { + stackName := stack.Name + + otlpEnv, err := settings.GetOTELEnvVars(ctx, stackName, core.LowerCamelCaseKind(ctx, ledger), " ") + if err != nil { + return nil, err + } + + clusterID, err := settings.GetStringOrDefault(ctx, stackName, "default", "ledger", "v3", "cluster-id") + if err != nil { + return nil, err + } + + dataDir := "/data/app" + walDir := "/data/raft" + + env := []corev1.EnvVar{ + core.Env("BIND_ADDR", fmt.Sprintf("0.0.0.0:%d", v3PortRaft)), + core.Env("CLUSTER_ID", clusterID), + core.Env("GRPC_PORT", fmt.Sprint(v3PortGRPC)), + core.Env("HTTP_PORT", fmt.Sprint(v3PortHTTP)), + core.Env("WAL_DIR", walDir), + core.Env("DATA_DIR", dataDir), + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + } + env = append(env, otlpEnv...) + env = append(env, core.GetDevEnvVars(stack, ledger)...) + + // Add pebble settings + pebbleEnv, err := buildV3PebbleEnvVars(ctx, stackName) + if err != nil { + return nil, err + } + env = append(env, pebbleEnv...) + + // Add raft settings + raftEnv, err := buildV3RaftEnvVars(ctx, stackName) + if err != nil { + return nil, err + } + env = append(env, raftEnv...) + + headlessSvcName := "ledger-raft" + command := buildV3Command(headlessSvcName, dataDir) + + container := corev1.Container{ + Name: "ledger", + Image: image.GetFullImageName(), + Command: []string{"/bin/sh", "-c"}, + Args: []string{command}, + Env: env, + Ports: []corev1.ContainerPort{ + {Name: "http", ContainerPort: v3PortHTTP}, + {Name: "grpc", ContainerPort: v3PortGRPC}, + {Name: "raft", ContainerPort: v3PortRaft}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "wal", MountPath: walDir}, + {Name: "data", MountPath: dataDir}, + {Name: "cold-cache", MountPath: "/data/cold-cache"}, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/livez", + Port: intstr.FromString("http"), + }, + }, + FailureThreshold: 20, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/livez", + Port: intstr.FromString("http"), + }, + }, + FailureThreshold: 30, + PeriodSeconds: 10, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromString("http"), + }, + }, + }, + } + + return &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/name": "ledger", + }, + }, + Spec: corev1.PodSpec{ + ImagePullSecrets: image.PullSecrets, + Containers: []corev1.Container{container}, + }, + }, nil +} + +func buildV3Command(headlessSvc, dataDir string) string { + // Shell script that computes node-id from the StatefulSet ordinal index, + // builds the advertise-addr from the pod's DNS name within the headless service, + // and decides whether to --bootstrap or --join depending on ordinal. + // + // POD_NAME is like "ledger-0", POD_INDEX is extracted from the suffix. + // pod-0 bootstraps (if no existing state), other pods join pod-0. + lines := []string{ + // Extract the ordinal index from the pod name (e.g. "ledger-2" → "2") + `POD_INDEX=${POD_NAME##*-}`, + // Raft node IDs must be >= 1 + `NODE_ID=$((POD_INDEX + 1))`, + // FQDN within the headless service + fmt.Sprintf(`ADVERTISE_ADDR="${POD_NAME}.%s.${POD_NAMESPACE}.svc.cluster.local:%d"`, headlessSvc, v3PortRaft), + // First pod (ordinal 0) bootstraps if no checkpoint exists yet, otherwise normal start. + // Other pods join pod-0. + fmt.Sprintf(`BOOTSTRAP_ADDR="ledger-0.%s.${POD_NAMESPACE}.svc.cluster.local:%d"`, headlessSvc, v3PortGRPC), + `CLUSTER_FLAG=""`, + fmt.Sprintf(`if [ "$POD_INDEX" = "0" ]; then + if [ ! -d "%s/pebble" ]; then + CLUSTER_FLAG="--bootstrap" + fi +else + CLUSTER_FLAG="--join $BOOTSTRAP_ADDR" +fi`, dataDir), + // Exec into the ledger binary + `exec ./ledger run \`, + ` --node-id "$NODE_ID" \`, + ` --advertise-addr "$ADVERTISE_ADDR" \`, + ` $CLUSTER_FLAG`, + } + + return strings.Join(lines, "\n") +} + +func buildV3VolumeClaimTemplates(ctx core.Context, stackName string) ([]corev1.PersistentVolumeClaim, error) { + type volumeSpec struct { + name string + sizeKey string + defaultSize string + storageClassKey string + } + + specs := []volumeSpec{ + {"wal", "ledger.v3.persistence.wal.size", "5Gi", "ledger.v3.persistence.wal.storage-class"}, + {"data", "ledger.v3.persistence.data.size", "10Gi", "ledger.v3.persistence.data.storage-class"}, + {"cold-cache", "ledger.v3.persistence.cold-cache.size", "10Gi", "ledger.v3.persistence.cold-cache.storage-class"}, + } + + var claims []corev1.PersistentVolumeClaim + for _, s := range specs { + sizeStr, err := settings.GetStringOrDefault(ctx, stackName, s.defaultSize, strings.Split(s.sizeKey, ".")...) + if err != nil { + return nil, err + } + + storageClass, err := settings.GetStringOrEmpty(ctx, stackName, strings.Split(s.storageClassKey, ".")...) + if err != nil { + return nil, err + } + + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(sizeStr), + }, + }, + }, + } + if storageClass != "" { + pvc.Spec.StorageClassName = &storageClass + } + claims = append(claims, pvc) + } + + return claims, nil +} + +// pebble setting key → env var name +var v3PebbleSettings = []struct { + key string + envVar string +}{ + {"cache-size", "PEBBLE_CACHE_SIZE"}, + {"memtable-size", "PEBBLE_MEMTABLE_SIZE"}, + {"memtable-stop-writes-threshold", "PEBBLE_MEMTABLE_STOP_WRITES_THRESHOLD"}, + {"l0-compaction-threshold", "PEBBLE_L0_COMPACTION_THRESHOLD"}, + {"l0-stop-writes-threshold", "PEBBLE_L0_STOP_WRITES_THRESHOLD"}, + {"lbase-max-bytes", "PEBBLE_LBASE_MAX_BYTES"}, + {"target-file-size", "PEBBLE_TARGET_FILE_SIZE"}, + {"max-concurrent-compactions", "PEBBLE_MAX_CONCURRENT_COMPACTIONS"}, +} + +func buildV3PebbleEnvVars(ctx core.Context, stackName string) ([]corev1.EnvVar, error) { + var envVars []corev1.EnvVar + for _, s := range v3PebbleSettings { + val, err := settings.GetStringOrEmpty(ctx, stackName, "ledger", "v3", "pebble", s.key) + if err != nil { + return nil, err + } + if val != "" { + envVars = append(envVars, core.Env(s.envVar, val)) + } + } + return envVars, nil +} + +var v3RaftSettings = []struct { + key string + envVar string +}{ + {"snapshot-threshold", "RAFT_SNAPSHOT_THRESHOLD"}, + {"election-tick", "RAFT_ELECTION_TICK"}, + {"heartbeat-tick", "RAFT_HEARTBEAT_TICK"}, + {"tick-interval", "RAFT_TICK_INTERVAL"}, + {"max-size-per-msg", "RAFT_MAX_SIZE_PER_MSG"}, + {"max-inflight-msgs", "RAFT_MAX_INFLIGHT_MSGS"}, + {"compaction-margin", "RAFT_COMPACTION_MARGIN"}, +} + +func buildV3RaftEnvVars(ctx core.Context, stackName string) ([]corev1.EnvVar, error) { + var envVars []corev1.EnvVar + for _, s := range v3RaftSettings { + val, err := settings.GetStringOrEmpty(ctx, stackName, "ledger", "v3", "raft", s.key) + if err != nil { + return nil, err + } + if val != "" { + envVars = append(envVars, core.Env(s.envVar, val)) + } + } + return envVars, nil +} From dd0027882d965d0813a3b3bd3c3a58ceefc3dd40 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 25 Mar 2026 17:02:44 +0100 Subject: [PATCH 2/9] docs: add ledger v3 settings documentation Document all v3-specific Settings keys in both: - Settings reference table (01-Settings.md) - Ledger module page (03-Ledger.md) with architecture overview, YAML examples, and tables for persistence/Pebble/Raft tunables Co-Authored-By: Claude Opus 4.6 --- docs/04-Modules/03-Ledger.md | 115 ++++++++++++++++++ .../09-Configuration reference/01-Settings.md | 23 ++++ 2 files changed, 138 insertions(+) diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index 9bbaea5bd..146439a80 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -100,3 +100,118 @@ Available fields: - `push-retry-period`: Retry period for failed pushes - `sync-period`: Synchronization period - `logs-page-size`: Number of logs per page + +## Ledger v3 + +Ledger v3 is an architecturally different version that uses Raft consensus with Pebble embedded storage instead of PostgreSQL. When the version is `>= v3.0.0-alpha`, the operator deploys a **StatefulSet** instead of a Deployment. + +### Requirements + +Ledger v3 does **not** require PostgreSQL or a message broker. Storage is fully embedded (Pebble LSM). + +### Architecture + +The operator creates the following resources for v3: + +| Resource | Purpose | +|----------|---------| +| `StatefulSet/ledger` | Raft cluster nodes with `OrderedReady` pod management | +| `Service/ledger-raft` (headless) | DNS-based peer discovery for Raft consensus | +| `Service/ledger` (ClusterIP) | Gateway-facing service, maps port 8080 to container port 9000 | +| 3 PVCs per pod | `wal`, `data`, `cold-cache` | + +### Cluster Settings + +```yaml +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-replicas +spec: + stacks: ["*"] + key: ledger.v3.replicas + value: "3" +--- +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-cluster-id +spec: + stacks: ["*"] + key: ledger.v3.cluster-id + value: default +``` + +- `ledger.v3.replicas`: Number of Raft nodes. **Must be odd** for quorum (default: 3). +- `ledger.v3.cluster-id`: Raft cluster identifier (default: "default"). + +### Persistence Settings + +Each pod gets three PVCs. Size and storage class are configurable: + +```yaml +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-persistence +spec: + stacks: ["*"] + key: ledger.v3.persistence.wal.size + value: "5Gi" +--- +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-data-size +spec: + stacks: ["*"] + key: ledger.v3.persistence.data.size + value: "10Gi" +--- +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-cold-cache-size +spec: + stacks: ["*"] + key: ledger.v3.persistence.cold-cache.size + value: "10Gi" +``` + +| Key | Default | Description | +|-----|---------|-------------| +| `ledger.v3.persistence.wal.size` | 5Gi | WAL PVC size | +| `ledger.v3.persistence.wal.storage-class` | (cluster default) | WAL storage class | +| `ledger.v3.persistence.data.size` | 10Gi | Pebble data PVC size | +| `ledger.v3.persistence.data.storage-class` | (cluster default) | Data storage class | +| `ledger.v3.persistence.cold-cache.size` | 10Gi | Cold cache PVC size | +| `ledger.v3.persistence.cold-cache.storage-class` | (cluster default) | Cold cache storage class | + +### Pebble Tunables + +All Pebble settings are optional. When unset, the ledger binary defaults apply. + +| Key | Example | Description | +|-----|---------|-------------| +| `ledger.v3.pebble.cache-size` | 1073741824 | Block cache size in bytes | +| `ledger.v3.pebble.memtable-size` | 268435456 | Memtable size in bytes | +| `ledger.v3.pebble.memtable-stop-writes-threshold` | 2 | Memtable count before stopping writes | +| `ledger.v3.pebble.l0-compaction-threshold` | 4 | L0 files to trigger compaction | +| `ledger.v3.pebble.l0-stop-writes-threshold` | 12 | L0 files before stopping writes | +| `ledger.v3.pebble.lbase-max-bytes` | 67108864 | L1 max size in bytes | +| `ledger.v3.pebble.target-file-size` | 67108864 | SST file target size | +| `ledger.v3.pebble.max-concurrent-compactions` | 2 | Compaction parallelism | + +### Raft Tunables + +All Raft settings are optional. When unset, the ledger binary defaults apply. + +| Key | Example | Description | +|-----|---------|-------------| +| `ledger.v3.raft.snapshot-threshold` | 5000 | Log entries before snapshot | +| `ledger.v3.raft.election-tick` | 10 | Election timeout in ticks | +| `ledger.v3.raft.heartbeat-tick` | 1 | Heartbeat interval in ticks | +| `ledger.v3.raft.tick-interval` | 100ms | Duration of one tick | +| `ledger.v3.raft.max-size-per-msg` | 1048576 | Max message size in bytes | +| `ledger.v3.raft.max-inflight-msgs` | 256 | Max in-flight messages | +| `ledger.v3.raft.compaction-margin` | 1000 | Log retention after snapshot | diff --git a/docs/09-Configuration reference/01-Settings.md b/docs/09-Configuration reference/01-Settings.md index 6d56972a4..a55947335 100644 --- a/docs/09-Configuration reference/01-Settings.md +++ b/docs/09-Configuration reference/01-Settings.md @@ -32,6 +32,29 @@ While we have some basic types (string, number, bool ...), we also have some com | ledger.worker.async-block-hasher | Map | max-block-size=1000, schedule="0 * * * * *" | Configure async block hasher for the Ledger worker (v2.3+). Fields: `max-block-size`, `schedule` | | ledger.worker.bucket-cleanup | Map | retention-period=720h, schedule="0 0 * * *" | Configure bucket cleanup for the Ledger worker (v2.4+). Fields: `retention-period`, `schedule` | | ledger.worker.pipelines | Map | pull-interval=5s, push-retry-period=10s, sync-period=1m, logs-page-size=100 | Configure pipelines for the Ledger worker (v2.3+). Fields: `pull-interval`, `push-retry-period`, `sync-period`, `logs-page-size` | +| ledger.v3.replicas | Int | 3 | Raft cluster node count (v3+). Must be odd for quorum. Default: 3 | +| ledger.v3.cluster-id | String | default | Raft cluster ID (v3+). Default: "default" | +| ledger.v3.persistence.wal.size | String | 5Gi | PVC size for the Raft write-ahead log (v3+). Default: 5Gi | +| ledger.v3.persistence.wal.storage-class | String | | Storage class for the WAL PVC (v3+). Empty = cluster default | +| ledger.v3.persistence.data.size | String | 10Gi | PVC size for the Pebble data directory (v3+). Default: 10Gi | +| ledger.v3.persistence.data.storage-class | String | | Storage class for the data PVC (v3+). Empty = cluster default | +| ledger.v3.persistence.cold-cache.size | String | 10Gi | PVC size for the cold storage cache (v3+). Default: 10Gi | +| ledger.v3.persistence.cold-cache.storage-class | String | | Storage class for the cold cache PVC (v3+). Empty = cluster default | +| ledger.v3.pebble.cache-size | String | 1073741824 | Pebble block cache size in bytes (v3+) | +| ledger.v3.pebble.memtable-size | String | 268435456 | Pebble memtable size in bytes (v3+) | +| ledger.v3.pebble.memtable-stop-writes-threshold | String | 2 | Pebble memtable count before stopping writes (v3+) | +| ledger.v3.pebble.l0-compaction-threshold | String | 4 | L0 file count to trigger compaction (v3+) | +| ledger.v3.pebble.l0-stop-writes-threshold | String | 12 | L0 file count before stopping writes (v3+) | +| ledger.v3.pebble.lbase-max-bytes | String | 67108864 | L1 max size in bytes (v3+) | +| ledger.v3.pebble.target-file-size | String | 67108864 | SST file target size in bytes (v3+) | +| ledger.v3.pebble.max-concurrent-compactions | String | 2 | Compaction parallelism (v3+) | +| ledger.v3.raft.snapshot-threshold | String | 5000 | Log entries before taking a snapshot (v3+) | +| ledger.v3.raft.election-tick | String | 10 | Election timeout in ticks (v3+) | +| ledger.v3.raft.heartbeat-tick | String | 1 | Heartbeat interval in ticks (v3+) | +| ledger.v3.raft.tick-interval | String | 100ms | Duration of one Raft tick (v3+) | +| ledger.v3.raft.max-size-per-msg | String | 1048576 | Max Raft message size in bytes (v3+) | +| ledger.v3.raft.max-inflight-msgs | String | 256 | Max in-flight Raft messages (v3+) | +| ledger.v3.raft.compaction-margin | String | 1000 | Raft log retention after snapshot (v3+) | | payments.encryption-key | string | | Payments data encryption key | | payments.worker.temporal-max-concurrent-workflow-task-pollers | Int | | Payments worker max concurrent workflow task pollers configuration | | payments.worker.temporal-max-concurrent-activity-task-pollers | Int | | Payments worker max concurrent activity task pollers configuration | From dd2cca4080c042c4c0c8d901952e9adede9f184e Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 25 Mar 2026 17:57:24 +0100 Subject: [PATCH 3/9] feat(ledger): add v3 integration tests and fix settings prefix - Add 20 Ginkgo integration tests for v3 StatefulSet reconciler - Prefix all v3 settings with "module." for consistency - Fix version gate to use semver.Major == "v3" (avoids breaking v2 tests) - Remove redundant services.Create (GatewayHTTPAPI handles ClusterIP service) - Update docs with correct settings key paths Co-Authored-By: Claude Opus 4.6 --- config/rbac/role.yaml | 12 + docs/04-Modules/03-Ledger.md | 56 ++-- .../09-Configuration reference/01-Settings.md | 46 +-- internal/resources/ledgers/init.go | 2 +- internal/resources/ledgers/v3.go | 28 +- internal/tests/ledger_v3_controller_test.go | 306 ++++++++++++++++++ 6 files changed, 380 insertions(+), 70 deletions(-) create mode 100644 internal/tests/ledger_v3_controller_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index fcd0e7ca7..b7b295bc3 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -42,6 +42,18 @@ rules: - patch - update - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - batch resources: diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index 146439a80..383c1d002 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -129,7 +129,7 @@ metadata: name: ledger-v3-replicas spec: stacks: ["*"] - key: ledger.v3.replicas + key: module.ledger.v3.replicas value: "3" --- apiVersion: formance.com/v1beta1 @@ -138,12 +138,12 @@ metadata: name: ledger-v3-cluster-id spec: stacks: ["*"] - key: ledger.v3.cluster-id + key: module.ledger.v3.cluster-id value: default ``` -- `ledger.v3.replicas`: Number of Raft nodes. **Must be odd** for quorum (default: 3). -- `ledger.v3.cluster-id`: Raft cluster identifier (default: "default"). +- `module.ledger.v3.replicas`: Number of Raft nodes. **Must be odd** for quorum (default: 3). +- `module.ledger.v3.cluster-id`: Raft cluster identifier (default: "default"). ### Persistence Settings @@ -156,7 +156,7 @@ metadata: name: ledger-v3-persistence spec: stacks: ["*"] - key: ledger.v3.persistence.wal.size + key: module.ledger.v3.persistence.wal.size value: "5Gi" --- apiVersion: formance.com/v1beta1 @@ -165,7 +165,7 @@ metadata: name: ledger-v3-data-size spec: stacks: ["*"] - key: ledger.v3.persistence.data.size + key: module.ledger.v3.persistence.data.size value: "10Gi" --- apiVersion: formance.com/v1beta1 @@ -174,18 +174,18 @@ metadata: name: ledger-v3-cold-cache-size spec: stacks: ["*"] - key: ledger.v3.persistence.cold-cache.size + key: module.ledger.v3.persistence.cold-cache.size value: "10Gi" ``` | Key | Default | Description | |-----|---------|-------------| -| `ledger.v3.persistence.wal.size` | 5Gi | WAL PVC size | -| `ledger.v3.persistence.wal.storage-class` | (cluster default) | WAL storage class | -| `ledger.v3.persistence.data.size` | 10Gi | Pebble data PVC size | -| `ledger.v3.persistence.data.storage-class` | (cluster default) | Data storage class | -| `ledger.v3.persistence.cold-cache.size` | 10Gi | Cold cache PVC size | -| `ledger.v3.persistence.cold-cache.storage-class` | (cluster default) | Cold cache storage class | +| `module.ledger.v3.persistence.wal.size` | 5Gi | WAL PVC size | +| `module.ledger.v3.persistence.wal.storage-class` | (cluster default) | WAL storage class | +| `module.ledger.v3.persistence.data.size` | 10Gi | Pebble data PVC size | +| `module.ledger.v3.persistence.data.storage-class` | (cluster default) | Data storage class | +| `module.ledger.v3.persistence.cold-cache.size` | 10Gi | Cold cache PVC size | +| `module.ledger.v3.persistence.cold-cache.storage-class` | (cluster default) | Cold cache storage class | ### Pebble Tunables @@ -193,14 +193,14 @@ All Pebble settings are optional. When unset, the ledger binary defaults apply. | Key | Example | Description | |-----|---------|-------------| -| `ledger.v3.pebble.cache-size` | 1073741824 | Block cache size in bytes | -| `ledger.v3.pebble.memtable-size` | 268435456 | Memtable size in bytes | -| `ledger.v3.pebble.memtable-stop-writes-threshold` | 2 | Memtable count before stopping writes | -| `ledger.v3.pebble.l0-compaction-threshold` | 4 | L0 files to trigger compaction | -| `ledger.v3.pebble.l0-stop-writes-threshold` | 12 | L0 files before stopping writes | -| `ledger.v3.pebble.lbase-max-bytes` | 67108864 | L1 max size in bytes | -| `ledger.v3.pebble.target-file-size` | 67108864 | SST file target size | -| `ledger.v3.pebble.max-concurrent-compactions` | 2 | Compaction parallelism | +| `module.ledger.v3.pebble.cache-size` | 1073741824 | Block cache size in bytes | +| `module.ledger.v3.pebble.memtable-size` | 268435456 | Memtable size in bytes | +| `module.ledger.v3.pebble.memtable-stop-writes-threshold` | 2 | Memtable count before stopping writes | +| `module.ledger.v3.pebble.l0-compaction-threshold` | 4 | L0 files to trigger compaction | +| `module.ledger.v3.pebble.l0-stop-writes-threshold` | 12 | L0 files before stopping writes | +| `module.ledger.v3.pebble.lbase-max-bytes` | 67108864 | L1 max size in bytes | +| `module.ledger.v3.pebble.target-file-size` | 67108864 | SST file target size | +| `module.ledger.v3.pebble.max-concurrent-compactions` | 2 | Compaction parallelism | ### Raft Tunables @@ -208,10 +208,10 @@ All Raft settings are optional. When unset, the ledger binary defaults apply. | Key | Example | Description | |-----|---------|-------------| -| `ledger.v3.raft.snapshot-threshold` | 5000 | Log entries before snapshot | -| `ledger.v3.raft.election-tick` | 10 | Election timeout in ticks | -| `ledger.v3.raft.heartbeat-tick` | 1 | Heartbeat interval in ticks | -| `ledger.v3.raft.tick-interval` | 100ms | Duration of one tick | -| `ledger.v3.raft.max-size-per-msg` | 1048576 | Max message size in bytes | -| `ledger.v3.raft.max-inflight-msgs` | 256 | Max in-flight messages | -| `ledger.v3.raft.compaction-margin` | 1000 | Log retention after snapshot | +| `module.ledger.v3.raft.snapshot-threshold` | 5000 | Log entries before snapshot | +| `module.ledger.v3.raft.election-tick` | 10 | Election timeout in ticks | +| `module.ledger.v3.raft.heartbeat-tick` | 1 | Heartbeat interval in ticks | +| `module.ledger.v3.raft.tick-interval` | 100ms | Duration of one tick | +| `module.ledger.v3.raft.max-size-per-msg` | 1048576 | Max message size in bytes | +| `module.ledger.v3.raft.max-inflight-msgs` | 256 | Max in-flight messages | +| `module.ledger.v3.raft.compaction-margin` | 1000 | Log retention after snapshot | diff --git a/docs/09-Configuration reference/01-Settings.md b/docs/09-Configuration reference/01-Settings.md index a55947335..15c998964 100644 --- a/docs/09-Configuration reference/01-Settings.md +++ b/docs/09-Configuration reference/01-Settings.md @@ -32,29 +32,29 @@ While we have some basic types (string, number, bool ...), we also have some com | ledger.worker.async-block-hasher | Map | max-block-size=1000, schedule="0 * * * * *" | Configure async block hasher for the Ledger worker (v2.3+). Fields: `max-block-size`, `schedule` | | ledger.worker.bucket-cleanup | Map | retention-period=720h, schedule="0 0 * * *" | Configure bucket cleanup for the Ledger worker (v2.4+). Fields: `retention-period`, `schedule` | | ledger.worker.pipelines | Map | pull-interval=5s, push-retry-period=10s, sync-period=1m, logs-page-size=100 | Configure pipelines for the Ledger worker (v2.3+). Fields: `pull-interval`, `push-retry-period`, `sync-period`, `logs-page-size` | -| ledger.v3.replicas | Int | 3 | Raft cluster node count (v3+). Must be odd for quorum. Default: 3 | -| ledger.v3.cluster-id | String | default | Raft cluster ID (v3+). Default: "default" | -| ledger.v3.persistence.wal.size | String | 5Gi | PVC size for the Raft write-ahead log (v3+). Default: 5Gi | -| ledger.v3.persistence.wal.storage-class | String | | Storage class for the WAL PVC (v3+). Empty = cluster default | -| ledger.v3.persistence.data.size | String | 10Gi | PVC size for the Pebble data directory (v3+). Default: 10Gi | -| ledger.v3.persistence.data.storage-class | String | | Storage class for the data PVC (v3+). Empty = cluster default | -| ledger.v3.persistence.cold-cache.size | String | 10Gi | PVC size for the cold storage cache (v3+). Default: 10Gi | -| ledger.v3.persistence.cold-cache.storage-class | String | | Storage class for the cold cache PVC (v3+). Empty = cluster default | -| ledger.v3.pebble.cache-size | String | 1073741824 | Pebble block cache size in bytes (v3+) | -| ledger.v3.pebble.memtable-size | String | 268435456 | Pebble memtable size in bytes (v3+) | -| ledger.v3.pebble.memtable-stop-writes-threshold | String | 2 | Pebble memtable count before stopping writes (v3+) | -| ledger.v3.pebble.l0-compaction-threshold | String | 4 | L0 file count to trigger compaction (v3+) | -| ledger.v3.pebble.l0-stop-writes-threshold | String | 12 | L0 file count before stopping writes (v3+) | -| ledger.v3.pebble.lbase-max-bytes | String | 67108864 | L1 max size in bytes (v3+) | -| ledger.v3.pebble.target-file-size | String | 67108864 | SST file target size in bytes (v3+) | -| ledger.v3.pebble.max-concurrent-compactions | String | 2 | Compaction parallelism (v3+) | -| ledger.v3.raft.snapshot-threshold | String | 5000 | Log entries before taking a snapshot (v3+) | -| ledger.v3.raft.election-tick | String | 10 | Election timeout in ticks (v3+) | -| ledger.v3.raft.heartbeat-tick | String | 1 | Heartbeat interval in ticks (v3+) | -| ledger.v3.raft.tick-interval | String | 100ms | Duration of one Raft tick (v3+) | -| ledger.v3.raft.max-size-per-msg | String | 1048576 | Max Raft message size in bytes (v3+) | -| ledger.v3.raft.max-inflight-msgs | String | 256 | Max in-flight Raft messages (v3+) | -| ledger.v3.raft.compaction-margin | String | 1000 | Raft log retention after snapshot (v3+) | +| module.ledger.v3.replicas | Int | 3 | Raft cluster node count (v3+). Must be odd for quorum. Default: 3 | +| module.ledger.v3.cluster-id | String | default | Raft cluster ID (v3+). Default: "default" | +| module.ledger.v3.persistence.wal.size | String | 5Gi | PVC size for the Raft write-ahead log (v3+). Default: 5Gi | +| module.ledger.v3.persistence.wal.storage-class | String | | Storage class for the WAL PVC (v3+). Empty = cluster default | +| module.ledger.v3.persistence.data.size | String | 10Gi | PVC size for the Pebble data directory (v3+). Default: 10Gi | +| module.ledger.v3.persistence.data.storage-class | String | | Storage class for the data PVC (v3+). Empty = cluster default | +| module.ledger.v3.persistence.cold-cache.size | String | 10Gi | PVC size for the cold storage cache (v3+). Default: 10Gi | +| module.ledger.v3.persistence.cold-cache.storage-class | String | | Storage class for the cold cache PVC (v3+). Empty = cluster default | +| module.ledger.v3.pebble.cache-size | String | 1073741824 | Pebble block cache size in bytes (v3+) | +| module.ledger.v3.pebble.memtable-size | String | 268435456 | Pebble memtable size in bytes (v3+) | +| module.ledger.v3.pebble.memtable-stop-writes-threshold | String | 2 | Pebble memtable count before stopping writes (v3+) | +| module.ledger.v3.pebble.l0-compaction-threshold | String | 4 | L0 file count to trigger compaction (v3+) | +| module.ledger.v3.pebble.l0-stop-writes-threshold | String | 12 | L0 file count before stopping writes (v3+) | +| module.ledger.v3.pebble.lbase-max-bytes | String | 67108864 | L1 max size in bytes (v3+) | +| module.ledger.v3.pebble.target-file-size | String | 67108864 | SST file target size in bytes (v3+) | +| module.ledger.v3.pebble.max-concurrent-compactions | String | 2 | Compaction parallelism (v3+) | +| module.ledger.v3.raft.snapshot-threshold | String | 5000 | Log entries before taking a snapshot (v3+) | +| module.ledger.v3.raft.election-tick | String | 10 | Election timeout in ticks (v3+) | +| module.ledger.v3.raft.heartbeat-tick | String | 1 | Heartbeat interval in ticks (v3+) | +| module.ledger.v3.raft.tick-interval | String | 100ms | Duration of one Raft tick (v3+) | +| module.ledger.v3.raft.max-size-per-msg | String | 1048576 | Max Raft message size in bytes (v3+) | +| module.ledger.v3.raft.max-inflight-msgs | String | 256 | Max in-flight Raft messages (v3+) | +| module.ledger.v3.raft.compaction-margin | String | 1000 | Raft log retention after snapshot (v3+) | | payments.encryption-key | string | | Payments data encryption key | | payments.worker.temporal-max-concurrent-workflow-task-pollers | Int | | Payments worker max concurrent workflow task pollers configuration | | payments.worker.temporal-max-concurrent-activity-task-pollers | Int | | Payments worker max concurrent activity task pollers configuration | diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index dc64ec7f2..871b172da 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -41,7 +41,7 @@ import ( //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { - isV3 := !semver.IsValid(version) || semver.Compare(version, "v3.0.0-alpha") >= 0 + isV3 := semver.IsValid(version) && semver.Major(version) == "v3" if isV3 { return reconcileV3(ctx, stack, ledger, version) } diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 254797aa3..e13e34c22 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -15,7 +15,6 @@ import ( "github.com/formancehq/operator/v3/internal/core" "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" "github.com/formancehq/operator/v3/internal/resources/registries" - "github.com/formancehq/operator/v3/internal/resources/services" "github.com/formancehq/operator/v3/internal/resources/settings" ) @@ -39,15 +38,8 @@ func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, return err } - // ClusterIP service: port 8080 → container port 9000 (gateway compatibility) - if _, err := services.Create(ctx, ledger, "ledger", services.WithConfig(services.PortConfig{ - ServiceName: "ledger", - PortName: "http", - Port: 8080, - TargetPort: "http", - })); err != nil { - return err - } + // The GatewayHTTPAPI reconciler creates a ClusterIP service "ledger" with port 8080→"http". + // Since our container port named "http" is 9000, the service routes 8080→9000 automatically. if err := installV3StatefulSet(ctx, stack, ledger, imageConfiguration); err != nil { return err @@ -95,12 +87,12 @@ func createV3HeadlessService(ctx core.Context, stack *v1beta1.Stack, ledger *v1b func installV3StatefulSet(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, image *registries.ImageConfiguration) error { stackName := stack.Name - replicas, err := settings.GetInt32OrDefault(ctx, stackName, 3, "ledger", "v3", "replicas") + replicas, err := settings.GetInt32OrDefault(ctx, stackName, 3, "module", "ledger", "v3", "replicas") if err != nil { return err } if replicas%2 == 0 { - return fmt.Errorf("ledger.v3.replicas must be odd, got %d", replicas) + return fmt.Errorf("module.ledger.v3.replicas must be odd, got %d", replicas) } volumeClaims, err := buildV3VolumeClaimTemplates(ctx, stackName) @@ -148,7 +140,7 @@ func buildV3PodTemplate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. return nil, err } - clusterID, err := settings.GetStringOrDefault(ctx, stackName, "default", "ledger", "v3", "cluster-id") + clusterID, err := settings.GetStringOrDefault(ctx, stackName, "default", "module", "ledger", "v3", "cluster-id") if err != nil { return nil, err } @@ -298,9 +290,9 @@ func buildV3VolumeClaimTemplates(ctx core.Context, stackName string) ([]corev1.P } specs := []volumeSpec{ - {"wal", "ledger.v3.persistence.wal.size", "5Gi", "ledger.v3.persistence.wal.storage-class"}, - {"data", "ledger.v3.persistence.data.size", "10Gi", "ledger.v3.persistence.data.storage-class"}, - {"cold-cache", "ledger.v3.persistence.cold-cache.size", "10Gi", "ledger.v3.persistence.cold-cache.storage-class"}, + {"wal", "module.ledger.v3.persistence.wal.size", "5Gi", "module.ledger.v3.persistence.wal.storage-class"}, + {"data", "module.ledger.v3.persistence.data.size", "10Gi", "module.ledger.v3.persistence.data.storage-class"}, + {"cold-cache", "module.ledger.v3.persistence.cold-cache.size", "10Gi", "module.ledger.v3.persistence.cold-cache.storage-class"}, } var claims []corev1.PersistentVolumeClaim @@ -355,7 +347,7 @@ var v3PebbleSettings = []struct { func buildV3PebbleEnvVars(ctx core.Context, stackName string) ([]corev1.EnvVar, error) { var envVars []corev1.EnvVar for _, s := range v3PebbleSettings { - val, err := settings.GetStringOrEmpty(ctx, stackName, "ledger", "v3", "pebble", s.key) + val, err := settings.GetStringOrEmpty(ctx, stackName, "module", "ledger", "v3", "pebble", s.key) if err != nil { return nil, err } @@ -382,7 +374,7 @@ var v3RaftSettings = []struct { func buildV3RaftEnvVars(ctx core.Context, stackName string) ([]corev1.EnvVar, error) { var envVars []corev1.EnvVar for _, s := range v3RaftSettings { - val, err := settings.GetStringOrEmpty(ctx, stackName, "ledger", "v3", "raft", s.key) + val, err := settings.GetStringOrEmpty(ctx, stackName, "module", "ledger", "v3", "raft", s.key) if err != nil { return nil, err } diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go new file mode 100644 index 000000000..651658f98 --- /dev/null +++ b/internal/tests/ledger_v3_controller_test.go @@ -0,0 +1,306 @@ +package tests_test + +import ( + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/formancehq/operator/v3/internal/core" + "github.com/formancehq/operator/v3/internal/resources/settings" + . "github.com/formancehq/operator/v3/internal/tests/internal" +) + +var _ = Describe("LedgerV3Controller", func() { + Context("When creating a Ledger with v3 version", func() { + var ( + stack *v1beta1.Stack + ledger *v1beta1.Ledger + ) + BeforeEach(func() { + stack = &v1beta1.Stack{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.StackSpec{Version: "v3.0.0"}, + } + ledger = &v1beta1.Ledger{ + ObjectMeta: RandObjectMeta(), + Spec: v1beta1.LedgerSpec{ + StackDependency: v1beta1.StackDependency{ + Stack: stack.Name, + }, + }, + } + }) + JustBeforeEach(func() { + Expect(Create(stack)).To(Succeed()) + Expect(Create(ledger)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(ledger)).To(Succeed()) + Expect(Delete(stack)).To(Succeed()) + }) + + It("Should create a StatefulSet", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(sts).To(BeControlledBy(ledger)) + }) + + It("Should create a StatefulSet with 3 replicas by default", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(*sts.Spec.Replicas).To(Equal(int32(3))) + }) + + It("Should create a StatefulSet with OrderedReady pod management", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(sts.Spec.PodManagementPolicy).To(Equal(appsv1.OrderedReadyPodManagement)) + }) + + It("Should create a StatefulSet using the headless service", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(sts.Spec.ServiceName).To(Equal("ledger-raft")) + }) + + It("Should create 3 volume claim templates (wal, data, cold-cache)", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(sts.Spec.VolumeClaimTemplates).To(HaveLen(3)) + Expect(sts.Spec.VolumeClaimTemplates[0].Name).To(Equal("wal")) + Expect(sts.Spec.VolumeClaimTemplates[1].Name).To(Equal("data")) + Expect(sts.Spec.VolumeClaimTemplates[2].Name).To(Equal("cold-cache")) + }) + + It("Should configure the container with 3 ports (http, grpc, raft)", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + container := sts.Spec.Template.Spec.Containers[0] + Expect(container.Ports).To(HaveLen(3)) + Expect(container.Ports).To(ContainElements( + HaveField("Name", "http"), + HaveField("Name", "grpc"), + HaveField("Name", "raft"), + )) + }) + + It("Should configure 3 volume mounts", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + container := sts.Spec.Template.Spec.Containers[0] + Expect(container.VolumeMounts).To(ConsistOf( + corev1.VolumeMount{Name: "wal", MountPath: "/data/raft"}, + corev1.VolumeMount{Name: "data", MountPath: "/data/app"}, + corev1.VolumeMount{Name: "cold-cache", MountPath: "/data/cold-cache"}, + )) + }) + + It("Should configure liveness, readiness, and startup probes", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + container := sts.Spec.Template.Spec.Containers[0] + Expect(container.LivenessProbe).NotTo(BeNil()) + Expect(container.LivenessProbe.HTTPGet.Path).To(Equal("/livez")) + Expect(container.ReadinessProbe).NotTo(BeNil()) + Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) + Expect(container.StartupProbe).NotTo(BeNil()) + Expect(container.StartupProbe.HTTPGet.Path).To(Equal("/livez")) + }) + + It("Should set CLUSTER_ID env var with default value", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(sts.Spec.Template.Spec.Containers[0].Env).To( + ContainElement(core.Env("CLUSTER_ID", "default")), + ) + }) + + It("Should set downward API env vars (POD_NAME, POD_NAMESPACE)", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + env := sts.Spec.Template.Spec.Containers[0].Env + Expect(env).To(ContainElement(HaveField("Name", "POD_NAME"))) + Expect(env).To(ContainElement(HaveField("Name", "POD_NAMESPACE"))) + }) + + It("Should create a headless service for Raft peer discovery", func() { + svc := &corev1.Service{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger-raft", svc) + }).Should(Succeed()) + Expect(svc).To(BeControlledBy(ledger)) + Expect(svc.Spec.ClusterIP).To(Equal("None")) + Expect(svc.Spec.PublishNotReadyAddresses).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElements( + HaveField("Name", "raft"), + HaveField("Name", "grpc"), + )) + }) + + It("Should create a GatewayHTTPAPI with health endpoint", func() { + httpAPI := &v1beta1.GatewayHTTPAPI{} + Eventually(func() error { + return LoadResource("", core.GetObjectName(stack.Name, "ledger"), httpAPI) + }).Should(Succeed()) + Expect(httpAPI.Spec.HealthCheckEndpoint).To(Equal("health")) + }) + + It("Should NOT create a Database object", func() { + Consistently(func() error { + return LoadResource("", core.GetObjectName(stack.Name, "ledger"), &v1beta1.Database{}) + }).ShouldNot(Succeed()) + }) + + It("Should use the correct image", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + Expect(sts.Spec.Template.Spec.Containers[0].Image).To(ContainSubstring("ledger")) + Expect(sts.Spec.Template.Spec.Containers[0].Image).To(ContainSubstring("v3.0.0")) + }) + + Context("with custom replicas setting", func() { + var replicasSetting *v1beta1.Settings + BeforeEach(func() { + replicasSetting = settings.New(uuid.NewString(), "module.ledger.v3.replicas", "5", stack.Name) + }) + JustBeforeEach(func() { + Expect(Create(replicasSetting)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(replicasSetting)).To(Succeed()) + }) + It("Should create a StatefulSet with 5 replicas", func() { + sts := &appsv1.StatefulSet{} + Eventually(func(g Gomega) int32 { + g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) + return *sts.Spec.Replicas + }).Should(Equal(int32(5))) + }) + }) + + Context("with custom cluster-id setting", func() { + var clusterIDSetting *v1beta1.Settings + BeforeEach(func() { + clusterIDSetting = settings.New(uuid.NewString(), "module.ledger.v3.cluster-id", "my-cluster", stack.Name) + }) + JustBeforeEach(func() { + Expect(Create(clusterIDSetting)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(clusterIDSetting)).To(Succeed()) + }) + It("Should set CLUSTER_ID env var to custom value", func() { + sts := &appsv1.StatefulSet{} + Eventually(func(g Gomega) []corev1.EnvVar { + g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) + return sts.Spec.Template.Spec.Containers[0].Env + }).Should(ContainElement(core.Env("CLUSTER_ID", "my-cluster"))) + }) + }) + + Context("with custom persistence sizes", func() { + var walSizeSetting *v1beta1.Settings + BeforeEach(func() { + walSizeSetting = settings.New(uuid.NewString(), "module.ledger.v3.persistence.wal.size", "20Gi", stack.Name) + }) + JustBeforeEach(func() { + Expect(Create(walSizeSetting)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(walSizeSetting)).To(Succeed()) + }) + It("Should create WAL PVC with custom size", func() { + sts := &appsv1.StatefulSet{} + Eventually(func(g Gomega) string { + g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) + return sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().String() + }).Should(Equal("20Gi")) + }) + }) + + Context("with pebble settings", func() { + var cacheSizeSetting *v1beta1.Settings + BeforeEach(func() { + cacheSizeSetting = settings.New(uuid.NewString(), "module.ledger.v3.pebble.cache-size", "2147483648", stack.Name) + }) + JustBeforeEach(func() { + Expect(Create(cacheSizeSetting)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(cacheSizeSetting)).To(Succeed()) + }) + It("Should set PEBBLE_CACHE_SIZE env var", func() { + sts := &appsv1.StatefulSet{} + Eventually(func(g Gomega) []corev1.EnvVar { + g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) + return sts.Spec.Template.Spec.Containers[0].Env + }).Should(ContainElement(core.Env("PEBBLE_CACHE_SIZE", "2147483648"))) + }) + }) + + Context("with raft settings", func() { + var snapshotSetting *v1beta1.Settings + BeforeEach(func() { + snapshotSetting = settings.New(uuid.NewString(), "module.ledger.v3.raft.snapshot-threshold", "10000", stack.Name) + }) + JustBeforeEach(func() { + Expect(Create(snapshotSetting)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(snapshotSetting)).To(Succeed()) + }) + It("Should set RAFT_SNAPSHOT_THRESHOLD env var", func() { + sts := &appsv1.StatefulSet{} + Eventually(func(g Gomega) []corev1.EnvVar { + g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) + return sts.Spec.Template.Spec.Containers[0].Env + }).Should(ContainElement(core.Env("RAFT_SNAPSHOT_THRESHOLD", "10000"))) + }) + }) + + Context("with monitoring enabled", func() { + var otelTracesDSNSetting *v1beta1.Settings + BeforeEach(func() { + otelTracesDSNSetting = settings.New(uuid.NewString(), "opentelemetry.traces.dsn", "grpc://collector", stack.Name) + }) + JustBeforeEach(func() { + Expect(Create(otelTracesDSNSetting)).To(Succeed()) + }) + AfterEach(func() { + Expect(Delete(otelTracesDSNSetting)).To(Succeed()) + }) + It("Should add OTEL env vars to the StatefulSet", func() { + sts := &appsv1.StatefulSet{} + Eventually(func(g Gomega) []corev1.EnvVar { + g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) + return sts.Spec.Template.Spec.Containers[0].Env + }).Should(ContainElement(HaveField("Name", "OTEL_SERVICE_NAME"))) + }) + }) + }) +}) From 24a53f1b007c852a938c72074f2fe529836de176 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 25 Mar 2026 18:15:45 +0100 Subject: [PATCH 4/9] feat(ledger): add preStop hook for proper Raft scale-down On scale-down, the preStop lifecycle hook: 1. Calls POST /_admin/deregister to remove the node from the Raft cluster 2. Cleans the WAL directory so future re-joins start as fresh learners This ensures clean Raft membership management during StatefulSet scaling. Co-Authored-By: Claude Opus 4.6 --- internal/resources/ledgers/v3.go | 23 +++++++++++++++++++++ internal/tests/ledger_v3_controller_test.go | 14 +++++++++++++ 2 files changed, 37 insertions(+) diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index e13e34c22..c7e018a89 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -231,6 +231,13 @@ func buildV3PodTemplate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. }, }, }, + Lifecycle: &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"/bin/sh", "-c", buildV3PreStopScript(walDir)}, + }, + }, + }, } return &corev1.PodTemplateSpec{ @@ -281,6 +288,22 @@ fi`, dataDir), return strings.Join(lines, "\n") } +// buildV3PreStopScript returns a shell script executed by the Kubernetes preStop +// lifecycle hook before a pod is terminated. It deregisters the local node from +// the Raft cluster and cleans the WAL directory so that a future re-join (after +// scale-up) starts as a fresh learner. +func buildV3PreStopScript(walDir string) string { + lines := []string{ + // Best-effort deregister: call the admin endpoint to remove this node + // from the Raft cluster. Ignore errors (e.g., last node, or already removed). + fmt.Sprintf(`wget --post-data='' -q -O- http://localhost:%d/_admin/deregister || true`, v3PortHTTP), + // Clean WAL so that if this pod restarts (scale-up), it joins as a fresh learner. + fmt.Sprintf(`rm -rf %s/* || true`, walDir), + } + + return strings.Join(lines, "\n") +} + func buildV3VolumeClaimTemplates(ctx core.Context, stackName string) ([]corev1.PersistentVolumeClaim, error) { type volumeSpec struct { name string diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 651658f98..50e2fa234 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -126,6 +126,20 @@ var _ = Describe("LedgerV3Controller", func() { Expect(container.StartupProbe.HTTPGet.Path).To(Equal("/livez")) }) + It("Should configure a preStop lifecycle hook for Raft deregistration", func() { + sts := &appsv1.StatefulSet{} + Eventually(func() error { + return LoadResource(stack.Name, "ledger", sts) + }).Should(Succeed()) + container := sts.Spec.Template.Spec.Containers[0] + Expect(container.Lifecycle).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop.Exec).NotTo(BeNil()) + Expect(container.Lifecycle.PreStop.Exec.Command).To(HaveLen(3)) + Expect(container.Lifecycle.PreStop.Exec.Command[2]).To(ContainSubstring("/_admin/deregister")) + Expect(container.Lifecycle.PreStop.Exec.Command[2]).To(ContainSubstring("rm -rf")) + }) + It("Should set CLUSTER_ID env var with default value", func() { sts := &appsv1.StatefulSet{} Eventually(func() error { From a9652ecd14e334ce7cc00fddb4ec902b5d0ace0e Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 25 Mar 2026 18:49:55 +0100 Subject: [PATCH 5/9] chore: regenerate helm RBAC template with StatefulSet permissions Co-Authored-By: Claude Opus 4.6 --- ....k8s.io_v1_clusterrole_formance-manager-role.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml index 8bb7f718b..df20410fc 100644 --- a/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml +++ b/helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml @@ -41,6 +41,18 @@ rules: - patch - update - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - batch resources: From ab59f2bca64e18d6785169c0e7dcdf20dfd32d09 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 31 Mar 2026 12:01:56 +0200 Subject: [PATCH 6/9] feat(kubectl-stacks): add create command and install recipe Add a `create ` command to kubectl-stacks and a `just install-kubectl-stacks` recipe to build and install the plugin into $GOPATH/bin. Co-Authored-By: Claude Opus 4.6 --- Justfile | 3 +++ tools/kubectl-stacks/create.go | 38 ++++++++++++++++++++++++++++++++++ tools/kubectl-stacks/main.go | 1 + 3 files changed, 42 insertions(+) create mode 100644 tools/kubectl-stacks/create.go diff --git a/Justfile b/Justfile index 4f6519a98..857034df1 100644 --- a/Justfile +++ b/Justfile @@ -99,5 +99,8 @@ generate-docs: --templates-dir=./crd-doc-templates \ --config=./docs.config.yaml +install-kubectl-stacks: + cd tools/kubectl-stacks && go build -o {{env('GOPATH', `go env GOPATH`)}}/bin/kubectl-stacks . + deploy: helm-update earthly +deploy diff --git a/tools/kubectl-stacks/create.go b/tools/kubectl-stacks/create.go new file mode 100644 index 000000000..a93917c24 --- /dev/null +++ b/tools/kubectl-stacks/create.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/rest" +) + +func NewCreateCommand(configFlags *genericclioptions.ConfigFlags) *cobra.Command { + return &cobra.Command{ + Use: "create ", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := getRestClient(configFlags) + if err != nil { + return err + } + + return create(cmd, client, args[0]) + }, + } +} + +func create(cmd *cobra.Command, client *rest.RESTClient, name string) error { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Creating stack '%s'...\r\n", name) + + stack := &v1beta1.Stack{} + stack.SetName(name) + + return client.Post(). + Resource("Stacks"). + Body(stack). + Do(cmd.Context()). + Error() +} diff --git a/tools/kubectl-stacks/main.go b/tools/kubectl-stacks/main.go index a078af91f..568a5e412 100644 --- a/tools/kubectl-stacks/main.go +++ b/tools/kubectl-stacks/main.go @@ -22,6 +22,7 @@ func NewRootCommand() *cobra.Command { configFlags := genericclioptions.NewConfigFlags(true) configFlags.AddFlags(cmd.PersistentFlags()) cmd.AddCommand( + NewCreateCommand(configFlags), NewLockCommand(configFlags), NewUnlockCommand(configFlags), NewListCommand(configFlags), From 8a63607e72ff6b393af8bdab0ba27996014afe20 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 31 Mar 2026 15:57:34 +0200 Subject: [PATCH 7/9] chore: Add setting --- .gitignore | 3 + docs/04-Modules/03-Ledger.md | 60 +++++++--- helm/crds/.helmignore | 1 + helm/operator/.helmignore | 1 + internal/resources/ledgers/init.go | 55 +++++++++- internal/resources/ledgers/v3.go | 115 ++++++++++++++++---- tools/kubectl-stacks/apiextensions.go | 30 +++++ tools/kubectl-stacks/enable_module.go | 151 ++++++++++++++++++++++++++ tools/kubectl-stacks/main.go | 1 + 9 files changed, 372 insertions(+), 45 deletions(-) create mode 100644 tools/kubectl-stacks/apiextensions.go create mode 100644 tools/kubectl-stacks/enable_module.go diff --git a/.gitignore b/.gitignore index ab67db318..99f6e1280 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ Dockerfile.cross charts *.tgz +*.tar.gz + +dist \ No newline at end of file diff --git a/docs/04-Modules/03-Ledger.md b/docs/04-Modules/03-Ledger.md index 383c1d002..9accfcab4 100644 --- a/docs/04-Modules/03-Ledger.md +++ b/docs/04-Modules/03-Ledger.md @@ -101,13 +101,47 @@ Available fields: - `sync-period`: Synchronization period - `logs-page-size`: Number of logs per page -## Ledger v3 +## Ledger v3 Mirror -Ledger v3 is an architecturally different version that uses Raft consensus with Pebble embedded storage instead of PostgreSQL. When the version is `>= v3.0.0-alpha`, the operator deploys a **StatefulSet** instead of a Deployment. +Ledger v3 can run alongside an existing v2 deployment as a **mirror**: it continuously replicates v2 ledger data into its own Raft-based storage. This allows gradual migration or read-offloading without disrupting v2. -### Requirements +When the `modules.ledger.v3-mirror` setting is present, the operator: +1. Deploys v2 normally (database, migrations, Deployment) +2. Deploys a v3 Raft StatefulSet in parallel +3. Runs a provisioning Job that creates mirror ledgers in v3, each sourcing data from the v2 PostgreSQL database + +### Enabling v3 Mirror + +Create a Settings resource with the key `modules.ledger.v3-mirror`. The value format is: + +``` +:,,... +``` + +- **v3-image-tag**: The container image tag of the ledger v3 binary (e.g. `v3.0.0-alpha.1`) +- **ledger names**: Comma-separated list of v2 ledger names to mirror + +```yaml +apiVersion: formance.com/v1beta1 +kind: Settings +metadata: + name: ledger-v3-mirror +spec: + stacks: ["my-stack"] + key: modules.ledger.v3-mirror + value: "v3.0.0-alpha.1:default,payments" +``` -Ledger v3 does **not** require PostgreSQL or a message broker. Storage is fully embedded (Pebble LSM). +This example deploys a v3 cluster using image tag `v3.0.0-alpha.1` and creates two mirror ledgers (`default` and `payments`) that replicate from the v2 PostgreSQL database. + +### How It Works + +The provisioning Job connects to the v3 cluster's gRPC endpoint and calls `ledgerctl ledgers create` for each listed ledger with: +- `--mode mirror` — marks the ledger as a mirror (read-only, no direct writes) +- `--mirror-source-type postgres` — uses direct PostgreSQL access for replication +- `--mirror-dsn` — the PostgreSQL DSN of the v2 database (derived automatically from the Database resource) + +The Job is idempotent: if a mirror ledger already exists, the error is ignored. It retries on failure (e.g. if the v3 cluster is not yet ready). ### Architecture @@ -117,9 +151,13 @@ The operator creates the following resources for v3: |----------|---------| | `StatefulSet/ledger` | Raft cluster nodes with `OrderedReady` pod management | | `Service/ledger-raft` (headless) | DNS-based peer discovery for Raft consensus | -| `Service/ledger` (ClusterIP) | Gateway-facing service, maps port 8080 to container port 9000 | +| `Job/v3-mirror-provision` | Creates mirror ledgers in the v3 cluster | | 3 PVCs per pod | `wal`, `data`, `cold-cache` | +### Requirements + +Ledger v3 does **not** require its own PostgreSQL or message broker. Storage is fully embedded (Pebble LSM). However, the v3 pods need network access to the v2 PostgreSQL database for mirror replication. + ### Cluster Settings ```yaml @@ -131,19 +169,11 @@ spec: stacks: ["*"] key: module.ledger.v3.replicas value: "3" ---- -apiVersion: formance.com/v1beta1 -kind: Settings -metadata: - name: ledger-v3-cluster-id -spec: - stacks: ["*"] - key: module.ledger.v3.cluster-id - value: default ``` - `module.ledger.v3.replicas`: Number of Raft nodes. **Must be odd** for quorum (default: 3). -- `module.ledger.v3.cluster-id`: Raft cluster identifier (default: "default"). + +The Raft cluster ID is automatically set to the stack name. ### Persistence Settings diff --git a/helm/crds/.helmignore b/helm/crds/.helmignore index 9e8e34ffa..eb5621967 100644 --- a/helm/crds/.helmignore +++ b/helm/crds/.helmignore @@ -21,6 +21,7 @@ .idea/ *.tmproj .vscode/ +./*.tgz # ignore kustomization.yaml files templates/rbac/kustomization.yaml diff --git a/helm/operator/.helmignore b/helm/operator/.helmignore index 9e8e34ffa..eb5621967 100644 --- a/helm/operator/.helmignore +++ b/helm/operator/.helmignore @@ -21,6 +21,7 @@ .idea/ *.tmproj .vscode/ +./*.tgz # ignore kustomization.yaml files templates/rbac/kustomization.yaml diff --git a/internal/resources/ledgers/init.go b/internal/resources/ledgers/init.go index 871b172da..7f57f99cb 100644 --- a/internal/resources/ledgers/init.go +++ b/internal/resources/ledgers/init.go @@ -18,6 +18,8 @@ package ledgers import ( _ "embed" + "fmt" + "strings" "github.com/pkg/errors" "golang.org/x/mod/semver" @@ -32,6 +34,7 @@ import ( "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" "github.com/formancehq/operator/v3/internal/resources/jobs" "github.com/formancehq/operator/v3/internal/resources/registries" + "github.com/formancehq/operator/v3/internal/resources/settings" ) //+kubebuilder:rbac:groups=formance.com,resources=ledgers,verbs=get;list;watch;create;update;patch;delete @@ -41,11 +44,6 @@ import ( //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { - isV3 := semver.IsValid(version) && semver.Major(version) == "v3" - if isV3 { - return reconcileV3(ctx, stack, ledger, version) - } - database, err := databases.Create(ctx, stack, ledger) if err != nil { return err @@ -110,7 +108,52 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, versio } } - return installLedger(ctx, stack, ledger, database, imageConfiguration, version) + if err := installLedger(ctx, stack, ledger, database, imageConfiguration, version); err != nil { + return err + } + + // If v3 mirror is configured, deploy the v3 StatefulSet and provision mirror ledgers. + // The setting value is "image:ledger1,ledger2,..." (e.g. "v3.0.0-alpha.1:default,payments"). + v3MirrorSetting, err := settings.GetString(ctx, stack.Name, "modules", "ledger", "v3-mirror") + if err != nil { + return err + } + if v3MirrorSetting != nil { + v3Image, mirrorLedgers, err := parseV3MirrorSetting(*v3MirrorSetting) + if err != nil { + return err + } + if err := reconcileV3(ctx, stack, ledger, database, v3Image, mirrorLedgers); err != nil { + return err + } + } + + return nil +} + +// parseV3MirrorSetting parses the setting value "image:ledger1,ledger2,...". +// Returns the image tag and the list of ledger names to mirror. +func parseV3MirrorSetting(value string) (string, []string, error) { + parts := strings.SplitN(value, ":", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", nil, fmt.Errorf("invalid v3-mirror setting %q: expected format \"tag:ledger1,ledger2,...\"", value) + } + + image := parts[0] + + var ledgers []string + for _, l := range strings.Split(parts[1], ",") { + l = strings.TrimSpace(l) + if l != "" { + ledgers = append(ledgers, l) + } + } + + if len(ledgers) == 0 { + return "", nil, fmt.Errorf("invalid v3-mirror setting %q: no ledger names specified", value) + } + + return image, ledgers, nil } func init() { diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index c7e018a89..810c7c6ad 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -13,7 +13,8 @@ import ( "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/formancehq/operator/v3/internal/core" - "github.com/formancehq/operator/v3/internal/resources/gatewayhttpapis" + "github.com/formancehq/operator/v3/internal/resources/databases" + "github.com/formancehq/operator/v3/internal/resources/jobs" "github.com/formancehq/operator/v3/internal/resources/registries" "github.com/formancehq/operator/v3/internal/resources/settings" ) @@ -24,24 +25,27 @@ const ( v3PortRaft = int32(7777) ) -func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, version string) error { - imageConfiguration, err := registries.GetFormanceImage(ctx, stack, "ledger", version) +func reconcileV3(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, database *v1beta1.Database, version string, mirrorLedgers []string) error { + imageConfiguration, err := registries.GetImageConfiguration(ctx, stack.Name, fmt.Sprintf("ghcr.io/formancehq/ledger-v3:%s", version)) if err != nil { return err } - if err := gatewayhttpapis.Create(ctx, ledger, gatewayhttpapis.WithHealthCheckEndpoint("health")); err != nil { + if err := createV3HeadlessService(ctx, stack, ledger); err != nil { return err } - if err := createV3HeadlessService(ctx, stack, ledger); err != nil { + if err := installV3StatefulSet(ctx, stack, ledger, imageConfiguration); err != nil { return err } - // The GatewayHTTPAPI reconciler creates a ClusterIP service "ledger" with port 8080→"http". - // Since our container port named "http" is 9000, the service routes 8080→9000 automatically. + // Build postgres env vars from the v2 database for the mirror source. + postgresEnvVars, err := buildV2PostgresEnvVars(ctx, stack, database) + if err != nil { + return err + } - if err := installV3StatefulSet(ctx, stack, ledger, imageConfiguration); err != nil { + if err := createV3MirrorProvisioningJob(ctx, stack, ledger, imageConfiguration, postgresEnvVars, mirrorLedgers); err != nil { return err } @@ -74,7 +78,7 @@ func createV3HeadlessService(ctx core.Context, stack *v1beta1.Stack, ledger *v1b }, }, Selector: map[string]string{ - "app.kubernetes.io/name": "ledger", + "app.kubernetes.io/name": "ledger-v3", }, } return nil @@ -113,18 +117,21 @@ func installV3StatefulSet(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta Namespace: stackName, }, func(t *appsv1.StatefulSet) error { - t.Spec = appsv1.StatefulSetSpec{ - Replicas: &replicas, - ServiceName: headlessSvcName, - PodManagementPolicy: appsv1.OrderedReadyPodManagement, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/name": "ledger", - }, + // VolumeClaimTemplates are immutable after creation. + // Only set them when the StatefulSet is new (no UID yet). + if t.UID == "" { + t.Spec.VolumeClaimTemplates = volumeClaims + } + + t.Spec.Replicas = &replicas + t.Spec.ServiceName = headlessSvcName + t.Spec.PodManagementPolicy = appsv1.OrderedReadyPodManagement + t.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "ledger-v3", }, - Template: *podTemplate, - VolumeClaimTemplates: volumeClaims, } + t.Spec.Template = *podTemplate return nil }, core.WithController[*appsv1.StatefulSet](ctx.GetScheme(), ledger), @@ -140,10 +147,7 @@ func buildV3PodTemplate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. return nil, err } - clusterID, err := settings.GetStringOrDefault(ctx, stackName, "default", "module", "ledger", "v3", "cluster-id") - if err != nil { - return nil, err - } + clusterID := stackName dataDir := "/data/app" walDir := "/data/raft" @@ -243,7 +247,7 @@ func buildV3PodTemplate(ctx core.Context, stack *v1beta1.Stack, ledger *v1beta1. return &corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app.kubernetes.io/name": "ledger", + "app.kubernetes.io/name": "ledger-v3", }, }, Spec: corev1.PodSpec{ @@ -407,3 +411,66 @@ func buildV3RaftEnvVars(ctx core.Context, stackName string) ([]corev1.EnvVar, er } return envVars, nil } + +// buildV2PostgresEnvVars builds env vars that resolve to the v2 postgres DSN at runtime, +// using the same secret-based credential resolution as GetPostgresEnvVars. +func buildV2PostgresEnvVars(ctx core.Context, stack *v1beta1.Stack, database *v1beta1.Database) ([]corev1.EnvVar, error) { + pgEnvVars, err := databases.GetPostgresEnvVars(ctx, stack, database) + if err != nil { + return nil, err + } + + // POSTGRES_URI is already computed by GetPostgresEnvVars. + // Alias it as MIRROR_POSTGRES_DSN for the provisioning job. + pgEnvVars = append(pgEnvVars, core.Env("MIRROR_POSTGRES_DSN", core.EnvVarPlaceholder("POSTGRES_URI"))) + + return pgEnvVars, nil +} + +// createV3MirrorProvisioningJob creates a Job that provisions mirror ledgers in the v3 cluster. +// It uses ledgerctl to create each mirror ledger with the postgres source pointing at the v2 database. +func createV3MirrorProvisioningJob( + ctx core.Context, + stack *v1beta1.Stack, + ledger *v1beta1.Ledger, + image *registries.ImageConfiguration, + postgresEnvVars []corev1.EnvVar, + mirrorLedgers []string, +) error { + headlessSvcName := "ledger-raft" + grpcAddr := fmt.Sprintf("ledger-0.%s.%s.svc.cluster.local:%d", headlessSvcName, stack.Name, v3PortGRPC) + + // Build a shell script that creates each mirror ledger. + // "already exists" errors are ignored (idempotent), all others are fatal. + var scriptLines []string + scriptLines = append(scriptLines, `set -e`) + for _, name := range mirrorLedgers { + scriptLines = append(scriptLines, fmt.Sprintf( + `echo "Creating mirror ledger %s..." +OUT=$(./ledgerctl ledgers create --name %s --mode mirror --mirror-source-type postgres --mirror-dsn "$MIRROR_POSTGRES_DSN" --server %s --insecure 2>&1) || { + if echo "$OUT" | grep -qi "already exists"; then + echo "Ledger %s already exists, skipping." + else + echo "$OUT" >&2 + exit 1 + fi +}`, + name, name, grpcAddr, name, + )) + } + scriptLines = append(scriptLines, `echo "All mirror ledgers provisioned."`) + + script := strings.Join(scriptLines, "\n") + + container := corev1.Container{ + Name: "provision-mirrors", + Image: image.GetFullImageName(), + Command: []string{"/bin/sh", "-c"}, + Args: []string{script}, + Env: postgresEnvVars, + } + + return jobs.Handle(ctx, ledger, "v3-mirror-provision", container, + jobs.WithImagePullSecrets(image.PullSecrets), + ) +} diff --git a/tools/kubectl-stacks/apiextensions.go b/tools/kubectl-stacks/apiextensions.go new file mode 100644 index 000000000..4854f3a83 --- /dev/null +++ b/tools/kubectl-stacks/apiextensions.go @@ -0,0 +1,30 @@ +package main + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" +) + +var apiExtensionsGV = schema.GroupVersion{ + Group: "apiextensions.k8s.io", + Version: "v1", +} + +// unstructuredNegotiator implements runtime.NegotiatedSerializer for raw JSON +// responses (used to query CRDs without importing apiextensions types). +type unstructuredNegotiator struct{} + +func (unstructuredNegotiator) EncoderForVersion(e runtime.Encoder, _ runtime.GroupVersioner) runtime.Encoder { + return e +} + +func (unstructuredNegotiator) DecoderToVersion(d runtime.Decoder, _ runtime.GroupVersioner) runtime.Decoder { + return d +} + +func (u unstructuredNegotiator) SupportedMediaTypes() []runtime.SerializerInfo { + scheme := runtime.NewScheme() + codecs := serializer.NewCodecFactory(scheme) + return codecs.SupportedMediaTypes() +} diff --git a/tools/kubectl-stacks/enable_module.go b/tools/kubectl-stacks/enable_module.go new file mode 100644 index 000000000..79b437414 --- /dev/null +++ b/tools/kubectl-stacks/enable_module.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/rest" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" +) + +// moduleCRD holds the kind and plural resource name extracted from a CRD. +type moduleCRD struct { + Kind string + Plural string +} + +// discoverModules lists CRDs with label formance.com/kind=module and returns +// the available module kinds with their plural resource names. +func discoverModules(ctx context.Context, configFlags *genericclioptions.ConfigFlags) ([]moduleCRD, error) { + restConfig, err := configFlags.ToRESTConfig() + if err != nil { + return nil, err + } + + restConfig.APIPath = "/apis" + restConfig.GroupVersion = &apiExtensionsGV + restConfig.NegotiatedSerializer = unstructuredNegotiator{} + + client, err := rest.RESTClientFor(restConfig) + if err != nil { + return nil, err + } + + raw, err := client.Get(). + Resource("customresourcedefinitions"). + Param("labelSelector", "formance.com/kind=module"). + Do(ctx). + Raw() + if err != nil { + return nil, fmt.Errorf("failed to list CRDs: %w", err) + } + + var crdList struct { + Items []struct { + Spec struct { + Names struct { + Kind string `json:"kind"` + Plural string `json:"plural"` + } `json:"names"` + } `json:"spec"` + } `json:"items"` + } + if err := json.Unmarshal(raw, &crdList); err != nil { + return nil, fmt.Errorf("failed to parse CRD list: %w", err) + } + + modules := make([]moduleCRD, 0, len(crdList.Items)) + for _, item := range crdList.Items { + modules = append(modules, moduleCRD{ + Kind: item.Spec.Names.Kind, + Plural: item.Spec.Names.Plural, + }) + } + + return modules, nil +} + +// resolveModule does a case-insensitive lookup of the input against discovered modules. +func resolveModule(input string, modules []moduleCRD) (moduleCRD, error) { + lower := strings.ToLower(input) + for _, m := range modules { + if strings.ToLower(m.Kind) == lower { + return m, nil + } + } + kinds := make([]string, len(modules)) + for i, m := range modules { + kinds[i] = m.Kind + } + return moduleCRD{}, fmt.Errorf("unknown module %q, available modules: %v", input, kinds) +} + +func NewEnableModuleCommand(configFlags *genericclioptions.ConfigFlags) *cobra.Command { + return &cobra.Command{ + Use: "enable-module ", + Short: "Enable a module on a stack", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := getRestClient(configFlags) + if err != nil { + return err + } + + modules, err := discoverModules(cmd.Context(), configFlags) + if err != nil { + return err + } + + return enableModule(cmd, client, modules, args[0], args[1]) + }, + } +} + +func enableModule(cmd *cobra.Command, client *rest.RESTClient, modules []moduleCRD, stackName, moduleInput string) error { + mod, err := resolveModule(moduleInput, modules) + if err != nil { + return err + } + + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Enabling module '%s' on stack '%s'...\r\n", mod.Kind, stackName) + + obj := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": v1beta1.GroupVersion.String(), + "kind": mod.Kind, + "metadata": map[string]any{ + "name": stackName + "-" + toLowerKebab(mod.Kind), + }, + "spec": map[string]any{ + "stack": stackName, + }, + }, + } + + return client.Post(). + Resource(mod.Plural). + Body(obj). + Do(cmd.Context()). + Error() +} + +func toLowerKebab(s string) string { + var result []byte + for i, c := range s { + if c >= 'A' && c <= 'Z' { + if i > 0 { + result = append(result, '-') + } + result = append(result, byte(c)+32) + } else { + result = append(result, byte(c)) + } + } + return string(result) +} diff --git a/tools/kubectl-stacks/main.go b/tools/kubectl-stacks/main.go index 568a5e412..4bbcfffcc 100644 --- a/tools/kubectl-stacks/main.go +++ b/tools/kubectl-stacks/main.go @@ -29,6 +29,7 @@ func NewRootCommand() *cobra.Command { NewSetDebugCommand(configFlags), NewDisableCommand(configFlags), NewEnableCommand(configFlags), + NewEnableModuleCommand(configFlags), NewUpgradeCommand(configFlags), NewSettingsCommand(configFlags), ) From ce73ffee76fd2a8fa4e81274ab9cc9b661be8026 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 1 Apr 2026 10:32:25 +0200 Subject: [PATCH 8/9] fix(ledger): align v3 tests with actual v3-mirror reconciliation path Tests were written for a standalone v3 mode that doesn't exist. The v3 deployment is triggered via the modules.ledger.v3-mirror setting as a mirror sidecar, not as a version-based dispatch. Fix CLUSTER_ID expectation, health endpoint, database creation, image name, and add mirror provisioning job test. Co-Authored-By: Claude Opus 4.6 --- internal/resources/ledgers/v3.go | 2 +- internal/tests/ledger_v3_controller_test.go | 79 ++++++++++++--------- tools/kubectl-stacks/create.go | 3 +- tools/kubectl-stacks/enable_module.go | 4 +- 4 files changed, 50 insertions(+), 38 deletions(-) diff --git a/internal/resources/ledgers/v3.go b/internal/resources/ledgers/v3.go index 810c7c6ad..21815deef 100644 --- a/internal/resources/ledgers/v3.go +++ b/internal/resources/ledgers/v3.go @@ -467,7 +467,7 @@ OUT=$(./ledgerctl ledgers create --name %s --mode mirror --mirror-source-type po Image: image.GetFullImageName(), Command: []string{"/bin/sh", "-c"}, Args: []string{script}, - Env: postgresEnvVars, + Env: postgresEnvVars, } return jobs.Handle(ctx, ledger, "v3-mirror-provision", container, diff --git a/internal/tests/ledger_v3_controller_test.go b/internal/tests/ledger_v3_controller_test.go index 50e2fa234..169232445 100644 --- a/internal/tests/ledger_v3_controller_test.go +++ b/internal/tests/ledger_v3_controller_test.go @@ -5,6 +5,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "github.com/formancehq/operator/v3/api/formance.com/v1beta1" @@ -14,16 +15,20 @@ import ( ) var _ = Describe("LedgerV3Controller", func() { - Context("When creating a Ledger with v3 version", func() { + Context("When creating a Ledger with v3-mirror setting", func() { var ( - stack *v1beta1.Stack - ledger *v1beta1.Ledger + stack *v1beta1.Stack + ledger *v1beta1.Ledger + databaseSettings *v1beta1.Settings + v3MirrorSetting *v1beta1.Settings ) BeforeEach(func() { stack = &v1beta1.Stack{ ObjectMeta: RandObjectMeta(), - Spec: v1beta1.StackSpec{Version: "v3.0.0"}, + Spec: v1beta1.StackSpec{Version: "v99.0.0"}, } + databaseSettings = settings.New(uuid.NewString(), "postgres.*.uri", "postgresql://localhost", stack.Name) + v3MirrorSetting = settings.New(uuid.NewString(), "modules.ledger.v3-mirror", "v3.0.0:default,payments", stack.Name) ledger = &v1beta1.Ledger{ ObjectMeta: RandObjectMeta(), Spec: v1beta1.LedgerSpec{ @@ -35,10 +40,14 @@ var _ = Describe("LedgerV3Controller", func() { }) JustBeforeEach(func() { Expect(Create(stack)).To(Succeed()) + Expect(Create(databaseSettings)).To(Succeed()) + Expect(Create(v3MirrorSetting)).To(Succeed()) Expect(Create(ledger)).To(Succeed()) }) AfterEach(func() { Expect(Delete(ledger)).To(Succeed()) + Expect(Delete(v3MirrorSetting)).To(Succeed()) + Expect(Delete(databaseSettings)).To(Succeed()) Expect(Delete(stack)).To(Succeed()) }) @@ -140,13 +149,13 @@ var _ = Describe("LedgerV3Controller", func() { Expect(container.Lifecycle.PreStop.Exec.Command[2]).To(ContainSubstring("rm -rf")) }) - It("Should set CLUSTER_ID env var with default value", func() { + It("Should set CLUSTER_ID env var to the stack name", func() { sts := &appsv1.StatefulSet{} Eventually(func() error { return LoadResource(stack.Name, "ledger", sts) }).Should(Succeed()) Expect(sts.Spec.Template.Spec.Containers[0].Env).To( - ContainElement(core.Env("CLUSTER_ID", "default")), + ContainElement(core.Env("CLUSTER_ID", stack.Name)), ) }) @@ -174,29 +183,51 @@ var _ = Describe("LedgerV3Controller", func() { )) }) - It("Should create a GatewayHTTPAPI with health endpoint", func() { + It("Should also create a v2 GatewayHTTPAPI with _healthcheck endpoint", func() { httpAPI := &v1beta1.GatewayHTTPAPI{} Eventually(func() error { return LoadResource("", core.GetObjectName(stack.Name, "ledger"), httpAPI) }).Should(Succeed()) - Expect(httpAPI.Spec.HealthCheckEndpoint).To(Equal("health")) + Expect(httpAPI.Spec.HealthCheckEndpoint).To(Equal("_healthcheck")) }) - It("Should NOT create a Database object", func() { - Consistently(func() error { - return LoadResource("", core.GetObjectName(stack.Name, "ledger"), &v1beta1.Database{}) - }).ShouldNot(Succeed()) + It("Should also create a Database object for the v2 path", func() { + database := &v1beta1.Database{} + Eventually(func() error { + return LoadResource("", core.GetObjectName(stack.Name, "ledger"), database) + }).Should(Succeed()) }) - It("Should use the correct image", func() { + It("Should use the correct v3 image", func() { sts := &appsv1.StatefulSet{} Eventually(func() error { return LoadResource(stack.Name, "ledger", sts) }).Should(Succeed()) - Expect(sts.Spec.Template.Spec.Containers[0].Image).To(ContainSubstring("ledger")) + Expect(sts.Spec.Template.Spec.Containers[0].Image).To(ContainSubstring("ledger-v3")) Expect(sts.Spec.Template.Spec.Containers[0].Image).To(ContainSubstring("v3.0.0")) }) + It("Should create a mirror provisioning job", func() { + jobList := &batchv1.JobList{} + Eventually(func(g Gomega) { + g.Expect(List(jobList)).To(Succeed()) + found := false + for _, j := range jobList.Items { + if j.Namespace == stack.Name { + for _, c := range j.Spec.Template.Spec.Containers { + if c.Name == "provision-mirrors" { + found = true + g.Expect(c.Image).To(ContainSubstring("ledger-v3")) + g.Expect(c.Args[0]).To(ContainSubstring("default")) + g.Expect(c.Args[0]).To(ContainSubstring("payments")) + } + } + } + } + g.Expect(found).To(BeTrue()) + }).Should(Succeed()) + }) + Context("with custom replicas setting", func() { var replicasSetting *v1beta1.Settings BeforeEach(func() { @@ -217,26 +248,6 @@ var _ = Describe("LedgerV3Controller", func() { }) }) - Context("with custom cluster-id setting", func() { - var clusterIDSetting *v1beta1.Settings - BeforeEach(func() { - clusterIDSetting = settings.New(uuid.NewString(), "module.ledger.v3.cluster-id", "my-cluster", stack.Name) - }) - JustBeforeEach(func() { - Expect(Create(clusterIDSetting)).To(Succeed()) - }) - AfterEach(func() { - Expect(Delete(clusterIDSetting)).To(Succeed()) - }) - It("Should set CLUSTER_ID env var to custom value", func() { - sts := &appsv1.StatefulSet{} - Eventually(func(g Gomega) []corev1.EnvVar { - g.Expect(LoadResource(stack.Name, "ledger", sts)).To(Succeed()) - return sts.Spec.Template.Spec.Containers[0].Env - }).Should(ContainElement(core.Env("CLUSTER_ID", "my-cluster"))) - }) - }) - Context("with custom persistence sizes", func() { var walSizeSetting *v1beta1.Settings BeforeEach(func() { diff --git a/tools/kubectl-stacks/create.go b/tools/kubectl-stacks/create.go index a93917c24..3e3e49d4c 100644 --- a/tools/kubectl-stacks/create.go +++ b/tools/kubectl-stacks/create.go @@ -3,10 +3,11 @@ package main import ( "fmt" - "github.com/formancehq/operator/v3/api/formance.com/v1beta1" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/rest" + + "github.com/formancehq/operator/v3/api/formance.com/v1beta1" ) func NewCreateCommand(configFlags *genericclioptions.ConfigFlags) *cobra.Command { diff --git a/tools/kubectl-stacks/enable_module.go b/tools/kubectl-stacks/enable_module.go index 79b437414..4f613925a 100644 --- a/tools/kubectl-stacks/enable_module.go +++ b/tools/kubectl-stacks/enable_module.go @@ -16,8 +16,8 @@ import ( // moduleCRD holds the kind and plural resource name extracted from a CRD. type moduleCRD struct { - Kind string - Plural string + Kind string + Plural string } // discoverModules lists CRDs with label formance.com/kind=module and returns From 99ad9c2758474d7fba89d89881bb674a39bb7fa4 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Wed, 1 Apr 2026 11:25:47 +0200 Subject: [PATCH 9/9] chore: add settings doc --- docs/09-Configuration reference/01-Settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/09-Configuration reference/01-Settings.md b/docs/09-Configuration reference/01-Settings.md index 15c998964..be2e14fea 100644 --- a/docs/09-Configuration reference/01-Settings.md +++ b/docs/09-Configuration reference/01-Settings.md @@ -32,6 +32,7 @@ While we have some basic types (string, number, bool ...), we also have some com | ledger.worker.async-block-hasher | Map | max-block-size=1000, schedule="0 * * * * *" | Configure async block hasher for the Ledger worker (v2.3+). Fields: `max-block-size`, `schedule` | | ledger.worker.bucket-cleanup | Map | retention-period=720h, schedule="0 0 * * *" | Configure bucket cleanup for the Ledger worker (v2.4+). Fields: `retention-period`, `schedule` | | ledger.worker.pipelines | Map | pull-interval=5s, push-retry-period=10s, sync-period=1m, logs-page-size=100 | Configure pipelines for the Ledger worker (v2.3+). Fields: `pull-interval`, `push-retry-period`, `sync-period`, `logs-page-size` | +| module.ledger.v3-mirror | String | :ledger1,ledger2 | Configure mirror mode for the ledger | | module.ledger.v3.replicas | Int | 3 | Raft cluster node count (v3+). Must be odd for quorum. Default: 3 | | module.ledger.v3.cluster-id | String | default | Raft cluster ID (v3+). Default: "default" | | module.ledger.v3.persistence.wal.size | String | 5Gi | PVC size for the Raft write-ahead log (v3+). Default: 5Gi |