diff --git a/core/cmd/initialize/initialize.go b/core/cmd/initialize/initialize.go index 1afd2ee48..c791d68c7 100644 --- a/core/cmd/initialize/initialize.go +++ b/core/cmd/initialize/initialize.go @@ -61,15 +61,15 @@ type Options struct { } // NewTier1Options creates Options for Tier1 initialization. -func NewTier1Options(cfg *config.Tier1Config) Options { +func NewTier1Options(fs afero.Fs, cfg *config.Tier1Config) Options { walDirectory := cfg.Wal.WALPath kopiaDirectory := cfg.Base.RepositoryDirectory return Options{ - WalFS: afero.NewBasePathFs(afero.NewOsFs(), walDirectory), + WalFS: afero.NewBasePathFs(fs, walDirectory), WalEncryptionPassword: cfg.EncryptionKey, Kopia: &KopiaOptions{ - FS: afero.NewBasePathFs(afero.NewOsFs(), kopiaDirectory), + FS: afero.NewBasePathFs(fs, kopiaDirectory), EncryptionPassword: cfg.EncryptionKey, InitializeRepo: func(ctx context.Context) error { return kopiaserver.InitializeTier1(ctx, cfg) diff --git a/core/cmd/server/directories_test.go b/core/cmd/server/directories_test.go new file mode 100644 index 000000000..024e5484b --- /dev/null +++ b/core/cmd/server/directories_test.go @@ -0,0 +1,124 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package server + +import ( + "context" + "testing" + + "github.com/spf13/afero" + + "github.com/cloudnative-pg/klio/core/pkg/config" +) + +func assertDirExists(t *testing.T, fs afero.Fs, dir string) { + t.Helper() + + ok, err := afero.DirExists(fs, dir) + if err != nil { + t.Fatalf("while checking if directory %q exists: %v", dir, err) + } + if !ok { + t.Fatalf("expected directory %q to exist", dir) + } +} + +func assertDirNotExists(t *testing.T, fs afero.Fs, dir string) { + t.Helper() + + ok, err := afero.DirExists(fs, dir) + if err != nil { + t.Fatalf("while checking if directory %q exists: %v", dir, err) + } + if ok { + t.Fatalf("expected directory %q not to exist", dir) + } +} + +func newTestServerConfig() *config.ServerConfig { + return &config.ServerConfig{ + QueueDirectory: "/queue", + Tier1: config.Tier1Config{ + Base: config.BaseServerConfig{ + CacheDirectory: "/cache_tier1/kopia-cache", + RepositoryDirectory: "/data/base", + }, + Wal: config.WalServerConfig{ + WALPath: "/data/wal", + }, + }, + Tier2: config.Tier2Config{ + CacheDirectory: "/cache_tier2/kopia-cache", + }, + } +} + +// TestInitializeTier1CreatesDirectories verifies that initializeTier1 +// creates the queue, WAL, Kopia repository, and cache directories, and does +// not touch the tier2 one, even though it later fails (the kopia binary is +// absent in unit tests). This test only asserts directory creation. +func TestInitializeTier1CreatesDirectories(t *testing.T) { + cfg := newTestServerConfig() + fs := afero.NewMemMapFs() + + _ = initializeTier1(context.Background(), fs, cfg) + + assertDirExists(t, fs, cfg.QueueDirectory) + assertDirExists(t, fs, cfg.Tier1.Wal.WALPath) + assertDirExists(t, fs, cfg.Tier1.Base.RepositoryDirectory) + assertDirExists(t, fs, cfg.Tier1.Base.CacheDirectory) + assertDirNotExists(t, fs, cfg.Tier2.CacheDirectory) +} + +// TestInitializeTier1RequiresQueueDirectory verifies that initializeTier1 +// fails, without creating any directory, when the queue directory is not +// configured. +func TestInitializeTier1RequiresQueueDirectory(t *testing.T) { + cfg := newTestServerConfig() + cfg.QueueDirectory = "" + fs := afero.NewMemMapFs() + + err := initializeTier1(context.Background(), fs, cfg) + if err == nil { + t.Fatal("expected an error, got nil") + } + + assertDirNotExists(t, fs, cfg.Tier1.Wal.WALPath) + assertDirNotExists(t, fs, cfg.Tier1.Base.RepositoryDirectory) + assertDirNotExists(t, fs, cfg.Tier1.Base.CacheDirectory) +} + +// TestInitializeTier2CreatesDirectories verifies that initializeTier2 +// creates only the tier2 cache directory, and none of the tier1 ones (nor +// the queue directory, which is only needed when tier1 is enabled), even +// though it later fails (the kopia binary is absent in unit tests). This +// test only asserts directory creation. +func TestInitializeTier2CreatesDirectories(t *testing.T) { + cfg := newTestServerConfig() + fs := afero.NewMemMapFs() + + _ = initializeTier2(context.Background(), fs, cfg) + + assertDirExists(t, fs, cfg.Tier2.CacheDirectory) + assertDirNotExists(t, fs, cfg.QueueDirectory) + assertDirNotExists(t, fs, cfg.Tier1.Wal.WALPath) + assertDirNotExists(t, fs, cfg.Tier1.Base.RepositoryDirectory) + assertDirNotExists(t, fs, cfg.Tier1.Base.CacheDirectory) +} diff --git a/core/cmd/server/initialize.go b/core/cmd/server/initialize.go index 6cf006914..5dbba7ceb 100644 --- a/core/cmd/server/initialize.go +++ b/core/cmd/server/initialize.go @@ -21,9 +21,11 @@ package server import ( "context" + "errors" "fmt" "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/spf13/afero" "github.com/cloudnative-pg/klio/core/cmd/initialize" "github.com/cloudnative-pg/klio/core/internal/tier2" @@ -32,13 +34,13 @@ import ( func initializeRepository(ctx context.Context, opts serverOpts) error { if opts.tier1 { - if err := initializeTier1(ctx, opts.cfg); err != nil { + if err := initializeTier1(ctx, opts.fs, opts.cfg); err != nil { return err } } if opts.tier2 { - if err := initializeTier2(ctx, opts.cfg); err != nil { + if err := initializeTier2(ctx, opts.fs, opts.cfg); err != nil { return err } } @@ -46,20 +48,47 @@ func initializeRepository(ctx context.Context, opts serverOpts) error { return nil } -func initializeTier1(ctx context.Context, cfg *config.ServerConfig) error { - walDirectory := cfg.Tier1.Wal.WALPath - kopiaDirectory := cfg.Tier1.Base.RepositoryDirectory - +func initializeTier1(ctx context.Context, fs afero.Fs, cfg *config.ServerConfig) error { log.FromContext(ctx).Info( "Ensuring tier1 repository is initialized.", - "walDirectory", walDirectory, - "kopiaDirectory", kopiaDirectory, + "walDirectory", cfg.Tier1.Wal.WALPath, + "kopiaDirectory", cfg.Tier1.Base.RepositoryDirectory, + "cacheDirectory", cfg.Tier1.Base.CacheDirectory, + "queueDirectory", cfg.QueueDirectory, ) - return initialize.Run(ctx, initialize.NewTier1Options(&cfg.Tier1)) + // The queue is always required when tier1 is enabled to support retention + // policy enforcement. Retention needs to check for pending tier2 transfers + // even when tier2 is not configured, to allow consistent behavior across + // configurations. + if cfg.QueueDirectory == "" { + return errors.New("queue is required when tier1 is enabled") + } + + if err := fs.MkdirAll(cfg.QueueDirectory, 0o750); err != nil { + return fmt.Errorf("while ensuring that the queue directory exists: %w", err) + } + + if err := fs.MkdirAll(cfg.Tier1.Wal.WALPath, 0o750); err != nil { + return fmt.Errorf("while ensuring that the tier1 WAL directory exists: %w", err) + } + + if err := fs.MkdirAll(cfg.Tier1.Base.RepositoryDirectory, 0o750); err != nil { + return fmt.Errorf("while ensuring that the tier1 repository directory exists: %w", err) + } + + if err := fs.MkdirAll(cfg.Tier1.Base.CacheDirectory, 0o750); err != nil { + return fmt.Errorf("while ensuring that the tier1 cache directory exists: %w", err) + } + + return initialize.Run(ctx, initialize.NewTier1Options(fs, &cfg.Tier1)) } -func initializeTier2(ctx context.Context, cfg *config.ServerConfig) error { +func initializeTier2(ctx context.Context, fs afero.Fs, cfg *config.ServerConfig) error { + if err := fs.MkdirAll(cfg.Tier2.CacheDirectory, 0o750); err != nil { + return fmt.Errorf("while ensuring that the tier2 cache directory exists: %w", err) + } + tier2BaseFS, err := tier2.ConnectBase(ctx, &cfg.Tier2) if err != nil { return fmt.Errorf("error while connecting to tier2 (base): %w", err) diff --git a/core/cmd/server/server.go b/core/cmd/server/server.go index df709554e..76e2cdc0e 100644 --- a/core/cmd/server/server.go +++ b/core/cmd/server/server.go @@ -21,11 +21,11 @@ package server import ( "context" - "errors" "fmt" "os" "github.com/cloudnative-pg/machinery/pkg/log" + "github.com/spf13/afero" "github.com/thejerf/suture/v4" "github.com/cloudnative-pg/klio/core/internal/kopia" @@ -81,6 +81,7 @@ type serverOpts struct { tier1 bool tier2 bool + fs afero.Fs cfg *config.ServerConfig adminSocketPath string runID string @@ -193,15 +194,8 @@ func runServer(ctx context.Context, opts serverOpts) error { }) // Configure NATS - // The queue is always required when tier1 is enabled to support retention policy - // enforcement. Retention needs to check for pending tier2 transfers even when - // tier2 is not configured, to allow consistent behavior across configurations. var queueURL string if opts.tier1 { - if opts.cfg.QueueDirectory == "" { - return errors.New("queue is required when tier1 is enabled") - } - nats, err := server.NewNatsService(opts.cfg.QueueDirectory) if err != nil { return err diff --git a/core/cmd/server/start.go b/core/cmd/server/start.go index 8bd272f20..051b657d1 100644 --- a/core/cmd/server/start.go +++ b/core/cmd/server/start.go @@ -27,6 +27,7 @@ import ( "github.com/cloudnative-pg/machinery/pkg/log" "github.com/google/uuid" + "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -92,6 +93,7 @@ var startCmd = &cobra.Command{ opts := serverOpts{ tier1: tier1Enabled, tier2: tier2Enabled, + fs: afero.NewOsFs(), cfg: &configuration, runID: runID.String(), runSecret: runSecret.String(), diff --git a/documentation/.wordlist.txt b/documentation/.wordlist.txt index 187b52eb6..1d0b9d914 100644 --- a/documentation/.wordlist.txt +++ b/documentation/.wordlist.txt @@ -252,6 +252,7 @@ github goroutines grpc gzip +hardcodes hostPath http https @@ -322,6 +323,8 @@ sfixed sint snapshotted str +subdirectories +subdirectory subprocess tablespaces teardown diff --git a/documentation/web/docs/developer/running-e2e-tests.md b/documentation/web/docs/developer/running-e2e-tests.md index a9cbbaff7..b63f1565a 100644 --- a/documentation/web/docs/developer/running-e2e-tests.md +++ b/documentation/web/docs/developer/running-e2e-tests.md @@ -121,6 +121,10 @@ The E2E tests are located in `operator/test/e2e/` and include: (`RecoverClusterFromTier2`) - **`tier2_pitr_test.go`** - Point-in-time recovery from tier2 storage (`RecoverClusterFromTier2Pitr`) +- **`tier2_recovery_common_test.go`** - Shared tier2 recovery helpers used + by both of the above; also asserts that the read-only (tier2-only) + recovery Server gets the unified `klio` PVC/mount, same as a tier1 + server - **`tier2_retention_test.go`** - Backup and WAL retention policy enforcement in tier2 storage (`Tier2Retention`) - **`compression_test.go`** - Kopia compression policies: verifies the @@ -132,12 +136,14 @@ The E2E tests are located in `operator/test/e2e/` and include: server-side tier1 retention prunes old WALs only after they reach tier2, driven by backup completion rather than a client command (`WALRetentionQueueAwareness`) -- **`server_reconfig_test.go`** - Adding tier2 storage to an existing - tier1+queue server (`ServerTierReconfiguration`) +- **`server_reconfig_test.go`** - Adding tier2 to an existing tier1-only + server: verifies the single unified `klio` PVC is retained (same UID, + no new PVC created) since the StatefulSet's VolumeClaimTemplates are + unaffected by tier1/tier2 changes (`ServerTierReconfiguration`) - **`pluginconfiguration_update_test.go`** - PluginConfiguration updates and sidecar restart behavior (`PluginConfigurationUpdate`) -- **`pvc_resize_test.go`** - PVC resize for data, cache, and queue - volumes (`PVCResize`) +- **`pvc_resize_test.go`** - Resize of the single unified `klio` PVC + backing `/klio` (`PVCResize`) - **`otel_test.go`** - OpenTelemetry metrics and traces export: deploys an OTEL Collector and verifies that backup lifecycle metrics and traces are correctly exported via OTLP. After the success-path diff --git a/documentation/web/docs/user/api/_klio_api.md b/documentation/web/docs/user/api/_klio_api.md index 968a146d9..2d32d506d 100644 --- a/documentation/web/docs/user/api/_klio_api.md +++ b/documentation/web/docs/user/api/_klio_api.md @@ -12,23 +12,6 @@ Package v1alpha1 contains API Schema definitions for the klio v1alpha1 API group -#### Cache - - - -Cache defines the configuration for the cache directory. - - - -_Appears in:_ -- [Tier1Configuration](#tier1configuration) -- [Tier2Configuration](#tier2configuration) - -| Field | Description | Required | Default | Validation | -| --- | --- | --- | --- | --- | -| `pvcTemplate` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#persistentvolumeclaimspec-v1-core)_ | | True | | | - - #### CompressionAlgorithm _Underlying type:_ _string_ @@ -69,22 +52,6 @@ _Appears in:_ | `maxSize` _integer_ | MaxSize is the maximum file size, in bytes, to attempt compression for.
Files larger than this are stored uncompressed. Zero means no maximum. | | | Minimum: 0
Optional: \{\}
| -#### Data - - - -Data defines the configuration for the data directory. - - - -_Appears in:_ -- [Tier1Configuration](#tier1configuration) - -| Field | Description | Required | Default | Validation | -| --- | --- | --- | --- | --- | -| `pvcTemplate` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#persistentvolumeclaimspec-v1-core)_ | Template to be used to generate the Persistent Volume Claim needed for the data folder,
containing base backups and WAL files. | True | | | - - #### EmbeddedObjectMeta @@ -234,23 +201,6 @@ _Appears in:_ | `spec` _[PodSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#podspec-v1-core)_ | | | | Optional: \{\}
| -#### Queue - - - -Queue defines the configuration for the directory hosting the -task queue. - - - -_Appears in:_ -- [ServerSpec](#serverspec) - -| Field | Description | Required | Default | Validation | -| --- | --- | --- | --- | --- | -| `pvcTemplate` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#persistentvolumeclaimspec-v1-core)_ | PersistentVolumeClaimTemplate is used to generate the configuration for
the PVC hosting the work queue. | True | | | - - #### RetentionPolicy @@ -354,7 +304,7 @@ _Appears in:_ | `mode` _[ServerMode](#servermode)_ | Mode selects the operation mode of the server. | True | standard | Enum: [standard read-only]
| | `tier1` _[Tier1Configuration](#tier1configuration)_ | Tier1 is the Tier 1 configuration | True | | | | `tier2` _[Tier2Configuration](#tier2configuration)_ | Tier2 is the Tier 2 configuration | True | | | -| `queue` _[Queue](#queue)_ | Queue is the configuration of the PVC that should host
the task queue. | | | Optional: \{\}
| +| `storage` _[Storage](#storage)_ | Storage is the configuration of the single PersistentVolumeClaim
mounted at /klio, hosting base backups, WAL, the work queue, and
the Tier 1/Tier 2 caches as fixed subdirectories (data, queue,
cache_tier1, cache_tier2). | True | | | | `template` _[PodTemplateSpec](#podtemplatespec)_ | Template to override the default StatefulSet of the Klio server.
WARNING: Modifying this template may break the server functionality if not done carefully.
This field is primarily intended for advanced configuration such as telemetry setup.
Use at your own risk and ensure thorough testing before applying changes. | | | Optional: \{\}
| @@ -371,6 +321,23 @@ _Appears in:_ +#### Storage + + + +Storage defines the configuration for the Klio server's +PersistentVolumeClaim. + + + +_Appears in:_ +- [ServerSpec](#serverspec) + +| Field | Description | Required | Default | Validation | +| --- | --- | --- | --- | --- | +| `pvcTemplate` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.37/#persistentvolumeclaimspec-v1-core)_ | PersistentVolumeClaimTemplate is used to generate the PVC that
backs the /klio directory tree for this server. | True | | | + + #### TLSConfiguration @@ -402,8 +369,6 @@ _Appears in:_ | Field | Description | Required | Default | Validation | | --- | --- | --- | --- | --- | -| `cache` _[Cache](#cache)_ | Cache is the configuration of the PVC that should be
used for the cache. | True | | | -| `data` _[Data](#data)_ | Data is the configuration of the PVC that should be used
for the base backups. | True | | | | `encryptionKeyFile` _[FileSource](#filesource)_ | EncryptionKeyFile specifies the Age-encrypted encryption key file. | True | | ExactlyOneOf: [fileReference]
| | `identityFile` _[FileSource](#filesource)_ | IdentityFile specifies the Age identity (private key) file used to
decrypt the encryption key. | True | | ExactlyOneOf: [fileReference]
| | `compression` _[CompressionPolicy](#compressionpolicy)_ | Compression defines the repository-wide (global) compression policy
applied to base backups stored on tier1. Individual clusters can
override it through their PluginConfiguration. | | | Optional: \{\}
| @@ -439,7 +404,6 @@ _Appears in:_ | Field | Description | Required | Default | Validation | | --- | --- | --- | --- | --- | -| `cache` _[Cache](#cache)_ | Cache is the configuration of the PVC that should be
used for the cache. | True | | | | `s3` _[S3Configuration](#s3configuration)_ | S3 contains the configuration parameters for an S3-based tier 2. | True | | | | `encryptionKeyFile` _[FileSource](#filesource)_ | EncryptionKeyFile specifies the Age-encrypted encryption key file. | True | | ExactlyOneOf: [fileReference]
| | `identityFile` _[FileSource](#filesource)_ | IdentityFile specifies the Age identity (private key) file used to
decrypt the encryption key. | True | | ExactlyOneOf: [fileReference]
| diff --git a/documentation/web/docs/user/klio_server.md b/documentation/web/docs/user/klio_server.md index e43acc781..bcf1b6e10 100644 --- a/documentation/web/docs/user/klio_server.md +++ b/documentation/web/docs/user/klio_server.md @@ -12,9 +12,11 @@ The Klio server runs as a single `server` container. On startup, it first initializes the Kopia repository, then starts serving both base backups (using Kopia) and the incoming stream of PostgreSQL Write-Ahead Logs (WAL). -The base backups and WAL files are stored on a single PersistentVolume attached -to the Klio server pod, in the `/data/base` and `/data/wal` directories, -respectively. +All of the server's persistent state — base backups, WAL archive, work +queue, and Kopia caches — lives on a single PersistentVolumeClaim (PVC), +mounted at `/klio` inside the pod. Base backups and WAL files are stored +in the `/klio/data/base` and `/klio/data/wal` directories, respectively. +See [Storage Requirements](#storage-requirements) for the full layout. ## Storage Tiers @@ -42,8 +44,8 @@ See the [Object Store](#object-store) section for configuration details. ### The Work Queue When Tier 1 is configured, the Klio Server pods will use a work queue. -The work queue is backed by NATS JetStream with file storage on a separate -`PersistentVolume` mounted at `/queue`. +The work queue is backed by NATS JetStream with file storage in the +`/klio/queue` directory, on the same PVC as everything else. The queue serves two purposes: - **Retention policy enforcement**: Tracks which WAL files are in use before @@ -53,28 +55,41 @@ The queue serves two purposes: ## Storage Requirements -The Klio Server uses multiple PersistentVolumeClaims (PVCs), each -serving a different purpose. Understanding what each PVC contains helps you -size them appropriately for your environment. For guidance on managing -storage capacity and resizing PVCs, see -[Managing Storage](managing_storage.md). +The Klio Server uses a single PersistentVolumeClaim (PVC), mounted at +`/klio`, for all of its persistent state. There is one size and one +`storageClassName` for the whole volume, set via +`spec.storage.pvcTemplate` on the `Server` resource; backups, caches, and +the queue share that single size. For guidance on resizing this PVC and +recovering from a full disk, see [Managing Storage](managing_storage.md). -### Data PVC +On startup, the server creates the subdirectories it needs under +`/klio`: -The data PVC stores all backup data and WAL archives for Tier 1 storage. +- **`data`** — base backups and the WAL archive for Tier 1 storage +- **`queue`** — the NATS JetStream work queue (Tier 1 only) +- **`cache_tier1`** — the Kopia cache for Tier 1 +- **`cache_tier2`** — the Kopia cache for Tier 2 -It holds the base backups and the WAL archive of all the servers that are backed -up. +A read-only (Tier 2-only) server still gets the PVC and the `/klio` +mount; it only ever creates `cache_tier2`, since it has no Tier 1 data +or queue to manage. -The following factors should be considered when defining the PVC size: +The sections below cover what drives the size of each subdirectory, so +you can size the single PVC to hold all of them. + +### Base Backups and WAL Archive (`data`) + +The `data` subdirectory holds the base backups and the WAL archive of +all the servers that are backed up by this Klio server. The following +factors should be considered when sizing for it: 1. WAL file production rate 1. Base backup size 1. Retention policies -### Cache PVCs +### Kopia Caches (`cache_tier1`, `cache_tier2`) -The cache PVCs (one for Tier 1 and Tier 2 each) are used by Kopia for its +The Tier 1 and Tier 2 caches are used by Kopia for its [caching operations](https://kopia.io/docs/advanced/caching/). They are used to speed up snapshot operations. @@ -83,20 +98,19 @@ Klio is currently limited to use the default cache size when creating a Kopia repository, 5GB for content and 5GB for metadata. The cache sizes are not hard limits, as the cache is swept periodically, so users should have a space buffer to account for this additional space. -This limitation will be removed in a future version. ::: -### Queue PVC +### The Work Queue (`queue`) -The queue PVC is required when Tier 1 is configured. It stores the NATS -JetStream work queue used for retention policy enforcement and asynchronous -Tier 2 replication. +The `queue` subdirectory is created when Tier 1 is configured. It holds +the NATS JetStream work queue used for retention policy enforcement and +asynchronous Tier 2 replication. #### Queue Sizing Guidelines The queue stores only task metadata (cluster name and WAL filename), not the -actual WAL content. This means queue size depends on the **number of WAL -segments** generated, not the size of your database. +actual WAL content. This means the space it needs depends on the **number of +WAL segments** generated, not the size of your database. **Sizing formula:** @@ -115,9 +129,9 @@ Where: - **300 bytes**: Approximate storage per WAL task (message + JetStream overhead) - **2**: Safety factor -**Recommended sizes:** +**Recommended headroom:** -| Workload | WAL Rate | Recommended Size | +| Workload | WAL Rate | Recommended Headroom | |----------|----------|------------------| | Low write (OLTP) | ~60 segments/hour | **10 MiB** | | Medium write | ~120 segments/hour | **25 MiB** | @@ -133,11 +147,12 @@ for: - **Low cost of headroom**: Storage is cheap relative to the risk of queue overflow, which causes WAL loss -For shorter tolerance windows, you can reduce the queue size proportionally, but +For shorter tolerance windows, you can reduce this headroom proportionally, but keep the safety margin. :::tip -Start with 50 MiB as a conservative default. Monitor queue usage with the +Budget at least 50 MiB of headroom in the PVC for the queue as a +conservative default. Monitor queue usage with the `klio admin queue status` command and adjust based on actual WAL production rates in your environment. ::: @@ -183,7 +198,6 @@ A read-only server requires: - `mode: read-only` field in the spec - `tier2` configuration (S3 object storage) - **No** `tier1` configuration -- **No** `queue` configuration :::note The `mode` field is immutable. Once a Server is created, its mode @@ -212,18 +226,20 @@ spec: # Client authentication configuration caSecretName: klio-server-ca + # The single PVC, mounted at /klio. In read-only mode the + # server only ever populates the cache_tier2 subdirectory, but the + # PVC and mount are the same for every server regardless of mode. + storage: + pvcTemplate: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + # Tier 2 configuration (required for read-only mode) tier2: - # Cache storage configuration - cache: - pvcTemplate: - storageClassName: standard - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi # Only cache needed, no data storage - # Age-encrypted encryption key file encryptionKeyFile: fileReference: diff --git a/documentation/web/docs/user/managing_storage.md b/documentation/web/docs/user/managing_storage.md index 66f3cf3b1..ab551307d 100644 --- a/documentation/web/docs/user/managing_storage.md +++ b/documentation/web/docs/user/managing_storage.md @@ -7,9 +7,12 @@ sidebar_position: 9 This guide explains how to manage storage on your Klio server, prevent disk full scenarios, and recover when storage is exhausted. -The Klio server uses persistent storage for backup data, WAL archives, cache, -and the work queue. When the data PVC approaches capacity, backup and WAL -archival operations may fail. +The Klio server uses a single PersistentVolumeClaim (PVC), mounted at +`/klio`, for backup data, WAL archives, Kopia caches, and the work +queue — see [Storage Requirements](klio_server.md#storage-requirements) +for the full layout. Because everything shares one volume, growth in +any of these areas eats into the space the others need. When the PVC +approaches capacity, backup and WAL archival operations may fail. ## How Disk Space Is Freed @@ -40,7 +43,7 @@ kopia maintenance run \ ## When the Disk Is Full -When the Klio data PVC is completely full: +When the Klio server's `/klio` PVC is completely full: - All backup operations block (new backups, deletions, maintenance) - WAL streaming to Klio stops @@ -63,8 +66,7 @@ failures. ### Expand the PVC The simplest option is to expand the PVC. The Klio operator supports -expansion of PersistentVolumeClaims (PVCs) for all storage components: -data, cache (Tier 1 and Tier 2), and queue. +expansion of its PVC. #### Prerequisites @@ -92,9 +94,9 @@ If the output is not `true`, you need to either: #### Expanding PVC Size -To expand a PVC, update the corresponding -`pvcTemplate.resources.requests.storage` field in the Server spec with -a larger value: +To expand the PVC, update the +`spec.storage.pvcTemplate.resources.requests.storage` field in the +Server spec with a larger value: ```yaml apiVersion: klio.cnpg.io/v1alpha1 @@ -102,22 +104,11 @@ kind: Server metadata: name: klio-server spec: - tier1: - data: - pvcTemplate: - resources: - requests: - storage: 200Gi # Increased from 100Gi - cache: - pvcTemplate: - resources: - requests: - storage: 20Gi # Increased from 10Gi - queue: + storage: pvcTemplate: resources: requests: - storage: 20Gi # Increased from 10Gi + storage: 220Gi # Increased from 120Gi ``` Apply the updated Server resource: @@ -128,27 +119,27 @@ kubectl apply -f klio-server.yaml #### What Happens During Resize -When you update the Server spec with larger PVC sizes, the following +When you update the Server spec with a larger PVC size, the following occurs: -1. **PVC expansion**: The operator patches PVCs directly to the new - size. This modifies the PVC resources but does **not** update the - StatefulSet—the StatefulSet's VolumeClaimTemplates remain unchanged - at this point. +1. **PVC expansion**: The operator patches the `klio` PVC directly to + the new size. This modifies the PVC resources but does **not** + update the StatefulSet—the StatefulSet's VolumeClaimTemplates remain + unchanged at this point. 1. **Temporary misalignment**: After the PVC patch, there is a brief - period where the PVCs have the new size but the StatefulSet + period where the PVC has the new size but the StatefulSet VolumeClaimTemplates still reflect the old size. 1. **StatefulSet recreation**: The operator detects that the expected StatefulSet (with new VolumeClaimTemplates) differs from the current one. Since VolumeClaimTemplates are immutable in Kubernetes, the StatefulSet is deleted and recreated to align with the new spec. 1. **Pod restart**: The Klio server pod restarts and mounts the - already-expanded PVCs. + already-expanded PVC. :::note Why explicit PVC patching is necessary VolumeClaimTemplates only define specs for *new* PVCs—they do not resize existing ones. Without explicit PVC patching by the operator, the -StatefulSet would be recreated but the PVCs would remain at their +StatefulSet would be recreated but the PVC would remain at its original size, creating a permanent mismatch between the Server spec and actual storage. ::: @@ -157,7 +148,7 @@ and actual storage. After the full resize operation completes: -- The **PVCs** have the new expanded size +- The **PVC** has the new expanded size - The **StatefulSet VolumeClaimTemplates** match the new size (after recreation) - The **Server spec** is consistent with both @@ -167,14 +158,16 @@ This ensures no drift between the desired state and actual resources. #### Monitoring Resize Progress The operator emits a `PVCExpanded` Kubernetes event on the Server -resource when a PVC is successfully expanded. You can view these events +resource when the PVC is successfully expanded. You can view these events with: ```bash kubectl describe server klio-server ``` -Check the PVC status to monitor the resize operation: +Check the PVC status to monitor the resize operation. Since there is +only one PVC per server, the label selector returns exactly one +result: ```bash kubectl get pvc -l klio.cnpg.io/klio-server=klio-server @@ -187,14 +180,24 @@ The PVC will show the new requested size in For detailed status, including any resize conditions: ```bash -kubectl describe pvc data-klio-server-klio-0 +kubectl describe pvc klio-klio-server-klio-0 ``` +:::note PVC naming +The PVC name follows Kubernetes' StatefulSet convention +`--`. The +volume claim template is itself named `klio`, and the StatefulSet is +named `-klio`, so for a server named `klio-server` the PVC +is `klio-klio-server-klio-0` — the doubled `klio` is expected, not a +typo. +::: + #### Limitations - **Expansion only**: PVC shrinking is not supported by Kubernetes. - Attempting to reduce the storage size will be ignored and logged as - a warning. + Attempting to decrease `spec.storage.pvcTemplate.resources.requests.storage` + is rejected by the API server at admission time, with the error + `storage PVC size cannot be decreased`. - **StorageClass support**: The StorageClass must have `allowVolumeExpansion: true`. If the StorageClass does not support expansion, the resize will fail and an error will be logged. @@ -211,8 +214,12 @@ achieve a larger size, as this would result in **permanent data loss**. The only options in this case are: 1. Migrate to a StorageClass that supports volume expansion -1. Create a new Klio server with larger PVCs and restore from backup -1. Manually migrate data (requires downtime and careful planning) +1. Create a new Klio server with a larger PVC and restore from backup +1. Manually migrate data to a new PVC on the same StorageClass (requires + downtime and careful planning — see [Migrating from the Multi-PVC + Model](upgrade_notes.md#migrating-from-the-multi-pvc-model) for the + closely related procedure used when moving off an old, per-component + PVC layout) ::: @@ -276,11 +283,13 @@ and running maintenance manually. alerts. Set up monitoring and alerting on your PVC usage to detect capacity issues before they cause failures. -1. **Size Tier 1 storage appropriately**: Account for your backup - frequency, database size, change rate, and retention requirements when - provisioning the data PVC. Include buffer for the 24-hour window - during which deleted backup data is not yet eligible for garbage - collection. +1. **Size the PVC appropriately**: Account for your backup + frequency, database size, change rate, and retention requirements + when provisioning it, and add headroom for the Kopia caches and the + work queue on top of that — see + [Storage Requirements](klio_server.md#storage-requirements). Include + buffer for the 24-hour window during which deleted backup data is + not yet eligible for garbage collection. 1. **Use Tier 2 for long-term retention**: Object storage (S3, etc.) is more cost-effective and scales easily for long-term backup retention. diff --git a/documentation/web/docs/user/quickstart.md b/documentation/web/docs/user/quickstart.md index a8e7eaa39..dcead3096 100644 --- a/documentation/web/docs/user/quickstart.md +++ b/documentation/web/docs/user/quickstart.md @@ -236,7 +236,7 @@ client certificates must satisfy. ## Step 4: Create the Klio server The `Server` resource creates a StatefulSet running the Klio server, -along with the persistent volumes holding your backups. +along with the persistent volume holding your backups. Save the following as `klio-server.yaml`: @@ -255,24 +255,18 @@ spec: # CA used to verify client certificates caSecretName: klio-server-ca + # Single PVC backing base backups, WAL, the work queue, and the + # Kopia cache. The default Kopia cache is 5 GB of content plus 5 GB + # of metadata, so leave headroom beyond your base backups and WAL. + storage: + pvcTemplate: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + tier1: - # Kopia cache. The default Kopia cache is 5 GB of content plus - # 5 GB of metadata, so leave some headroom. - cache: - pvcTemplate: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - # Base backups and the WAL archive - data: - pvcTemplate: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi encryptionKeyFile: fileReference: volume: @@ -285,15 +279,6 @@ spec: secret: secretName: klio-age-identity path: identity.txt - - # Work queue, required whenever tier1 is configured - queue: - pvcTemplate: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi ``` @@ -303,10 +288,10 @@ Apply it: kubectl apply -f klio-server.yaml ``` -The volumes above use the default storage class. Set -`storageClassName` in each `pvcTemplate` to choose a different one, +The volume above uses the default storage class. Set +`storageClassName` in the `pvcTemplate` to choose a different one, and see [Storage Requirements](klio_server.md#storage-requirements) -for how to size them for real workloads. +for how to size it for real workloads. Wait for the server pod to come up: diff --git a/documentation/web/docs/user/upgrade_notes.md b/documentation/web/docs/user/upgrade_notes.md index aaf16d01b..90f3f6e1f 100644 --- a/documentation/web/docs/user/upgrade_notes.md +++ b/documentation/web/docs/user/upgrade_notes.md @@ -7,3 +7,147 @@ sidebar_position: 91 This page lists version-specific changes that may require manual action when upgrading Klio. For the upgrade procedure, see the [Helm chart page](helm_chart.mdx#upgrades). + +## 0.0.20 to 0.0.21 + +### Migrating from the Multi-PVC Model + +Klio servers created until v0.0.20 used four separate +PersistentVolumeClaims per server: `data`, `cachetier1`, `cachetier2`, +and `queue`. The current `Server` CRD no longer accepts that shape — +`spec.storage.pvcTemplate` is mandatory, and the old `tier1.data`, +`tier1.cache`, `tier2.cache`, and top-level `queue` fields no longer +validate. There is no automatic conversion: migrating an existing +Server to the unified PVC is a manual procedure. + +Only `data` (the actual backups and WAL) and `queue` +(pending, not-yet-processed tasks) contain information that should be +moved to the new PVC. `cachetier1` and `cachetier2` will be automatically +recreated and populated on demand. + +:::warning Plan for immediate, cluster-wide downtime on upgrade +Upgrading the operator to 0.0.21 deletes the StatefulSet for +**every** Server still on the old PVC layout, the +moment the new operator starts reconciling them — not when you get +around to migrating a given server. There is no way to defer or stage +this per server. Work out the PVC name and StorageClass for each +server (step 2 below) and schedule a maintenance window *before* +upgrading the operator, not after. + +Between the operator upgrade and the Server rewrite (step 4), the +`Server` object shows no status and emits no event: the only signal +that a given server is affected is an operator log line such as +`failed to reconcile statefulset ... +spec.volumeClaimTemplates[0].spec.accessModes: Required value`. +::: + +1. **Installing the operator 0.0.21 will delete the StatefulSets and + the pods associated with the `Server` resources**. + The old PVCs survive this because the StatefulSet's + `persistentVolumeClaimRetentionPolicy` is `Retain`. + +1. **Create a PVC named to match what the StatefulSet will + adopt.** Following the StatefulSet PVC naming convention + `--`, where the + unified volume claim template is named `klio` and the StatefulSet is + named `-klio`, the PVC to create is + `klio--klio-0` (for example, `klio-klio-server-klio-0` + for a server named `klio-server`). Give it + a size at least as large as the old total across `data`, both + caches, and `queue` combined. + + The operator hardcodes this name: the StatefulSet always looks for + `klio--klio-0`. + + :::warning Getting this name wrong orphans the copied data + The StatefulSet provisions a brand new, empty PVC instead of + adopting the one you pre-created, and the copied data is silently + orphaned on a PVC nothing mounts. After migrating, confirm the PVC + is the one adopted by running: + + ```shell + kubectl get pvc -l klio.cnpg.io/klio-server= + ``` + ::: + + For a server named ``, the PVC manifest looks like: + + ```yaml + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: klio--klio-0 + labels: + klio.cnpg.io/klio-server: + spec: + accessModes: + - ReadWriteOnce + storageClassName: + resources: + requests: + storage: + ``` + +1. **Run a one-off Job to copy the data.** Mount the old `data` PVC, + and the old `queue` PVC if present, read-only, alongside the new PVC + (read-write), and copy: + - the old `data` PVC's contents into the new PVC's `data/` + - the old `queue` PVC's contents into the new PVC's `queue/` + + For a server named ``, following the old PVC naming + convention `--` + (`data--klio-0`, `queue--klio-0`), and the new + PVC pre-created in the previous step (`klio--klio-0`): + + ```yaml + apiVersion: batch/v1 + kind: Job + metadata: + name: -pvc-migration + spec: + template: + spec: + restartPolicy: Never + containers: + - name: migrate + image: busybox + command: + - sh + - -c + - | + set -e + mkdir -p /new/data /new/queue + cp -a /old-data/. /new/data/ + if [ -d /old-queue ] && [ -n "$(ls -A /old-queue)" ]; then + cp -a /old-queue/. /new/queue/ + fi + volumeMounts: + - name: old-data + mountPath: /old-data + readOnly: true + - name: old-queue + mountPath: /old-queue + readOnly: true + - name: new-klio + mountPath: /new + volumes: + - name: old-data + persistentVolumeClaim: + claimName: data--klio-0 + - name: old-queue + persistentVolumeClaim: + claimName: queue--klio-0 + - name: new-klio + persistentVolumeClaim: + claimName: klio--klio-0 + ``` + +1. **Apply the new Server CR**: same name, updated image, new CRD shape, with + `spec.storage` sized to match the old total size and StorageClass + used across `data`, both caches, and `queue`. The StatefulSet adopts + the PVC you created in step 2 by name, instead of provisioning + an empty one. + +1. **Confirm the Server is healthy** against the migrated data before removing + the old PVCs. The old PVCs can be deleted after the new Server is + running and healthy. diff --git a/operator/api/v1alpha1/server_types.go b/operator/api/v1alpha1/server_types.go index 7452fca29..636edc0df 100644 --- a/operator/api/v1alpha1/server_types.go +++ b/operator/api/v1alpha1/server_types.go @@ -39,9 +39,8 @@ const ( // +kubebuilder:validation:XValidation:rule="self.mode == 'read-only' || has(self.tier1)",message="tier1 is required" // +kubebuilder:validation:XValidation:rule="self.mode != 'read-only' || has(self.tier2)",message="tier2 is required when mode is read-only" // +kubebuilder:validation:XValidation:rule="!(self.mode == 'read-only' && has(self.tier1))",message="tier1 cannot be set when mode is read-only" -// +kubebuilder:validation:XValidation:rule="!(self.mode == 'read-only' && has(self.queue))",message="queue cannot be set when mode is read-only" // +kubebuilder:validation:XValidation:rule="!(self.mode == 'read-only' && has(self.tier2) && has(self.tier2.compression))",message="tier2.compression cannot be set when mode is read-only" -// +kubebuilder:validation:XValidation:rule="self.mode == 'read-only' || has(self.queue)",message="queue is required when tier1 is configured" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.storage) || !has(oldSelf.storage.pvcTemplate.resources) || !has(oldSelf.storage.pvcTemplate.resources.requests) || !('storage' in oldSelf.storage.pvcTemplate.resources.requests) || !('storage' in self.storage.pvcTemplate.resources.requests) || !quantity(self.storage.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.storage.pvcTemplate.resources.requests['storage']))",message="storage PVC size cannot be decreased" type ServerSpec struct { // ImageConfiguration tells how to download the Klio // image. @@ -63,10 +62,11 @@ type ServerSpec struct { // Tier2 is the Tier 2 configuration Tier2 *Tier2Configuration `json:"tier2,omitempty"` - // Queue is the configuration of the PVC that should host - // the task queue. - // +optional - Queue *Queue `json:"queue,omitempty"` + // Storage is the configuration of the single PersistentVolumeClaim + // mounted at /klio, hosting base backups, WAL, the work queue, and + // the Tier 1/Tier 2 caches as fixed subdirectories (data, queue, + // cache_tier1, cache_tier2). + Storage Storage `json:"storage"` // Template to override the default StatefulSet of the Klio server. // WARNING: Modifying this template may break the server functionality if not done carefully. @@ -138,23 +138,11 @@ type TLSConfiguration struct { ClientCASecretName string `json:"caSecretName"` } -// Data defines the configuration for the data directory. -type Data struct { - // Template to be used to generate the Persistent Volume Claim needed for the data folder, - // containing base backups and WAL files. - PersistentVolumeClaimTemplate corev1.PersistentVolumeClaimSpec `json:"pvcTemplate"` -} - -// Cache defines the configuration for the cache directory. -type Cache struct { - PersistentVolumeClaimTemplate corev1.PersistentVolumeClaimSpec `json:"pvcTemplate"` -} - -// Queue defines the configuration for the directory hosting the -// task queue. -type Queue struct { - // PersistentVolumeClaimTemplate is used to generate the configuration for - // the PVC hosting the work queue. +// Storage defines the configuration for the Klio server's +// PersistentVolumeClaim. +type Storage struct { + // PersistentVolumeClaimTemplate is used to generate the PVC that + // backs the /klio directory tree for this server. PersistentVolumeClaimTemplate corev1.PersistentVolumeClaimSpec `json:"pvcTemplate"` } @@ -178,14 +166,6 @@ type FileSource struct { // Tier1Configuration is the tier 1 configuration. type Tier1Configuration struct { - // Cache is the configuration of the PVC that should be - // used for the cache. - Cache Cache `json:"cache"` - - // Data is the configuration of the PVC that should be used - // for the base backups. - Data Data `json:"data"` - // EncryptionKeyFile specifies the Age-encrypted encryption key file. EncryptionKeyFile FileSource `json:"encryptionKeyFile"` @@ -202,10 +182,6 @@ type Tier1Configuration struct { // Tier2Configuration is the tier 2 configuration. type Tier2Configuration struct { - // Cache is the configuration of the PVC that should be - // used for the cache. - Cache Cache `json:"cache"` - // S3 contains the configuration parameters for an S3-based tier 2. S3 *S3Configuration `json:"s3"` @@ -266,10 +242,6 @@ type ServerStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !('storage' in oldSelf.spec.tier1.data.pvcTemplate.resources.requests) || !('storage' in self.spec.tier1.data.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.data.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier1.data.pvcTemplate.resources.requests['storage']))",message="tier1.data PVC size cannot be decreased" -// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !('storage' in oldSelf.spec.tier1.cache.pvcTemplate.resources.requests) || !('storage' in self.spec.tier1.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.cache.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier1.cache.pvcTemplate.resources.requests['storage']))",message="tier1.cache PVC size cannot be decreased" -// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.tier2) || !has(self.spec.tier2) || !('storage' in oldSelf.spec.tier2.cache.pvcTemplate.resources.requests) || !('storage' in self.spec.tier2.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier2.cache.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.tier2.cache.pvcTemplate.resources.requests['storage']))",message="tier2.cache PVC size cannot be decreased" -// +kubebuilder:validation:XValidation:rule="!has(oldSelf.spec.queue) || !has(self.spec.queue) || !('storage' in oldSelf.spec.queue.pvcTemplate.resources.requests) || !('storage' in self.spec.queue.pvcTemplate.resources.requests) || !quantity(self.spec.queue.pvcTemplate.resources.requests['storage']).isLessThan(quantity(oldSelf.spec.queue.pvcTemplate.resources.requests['storage']))",message="queue PVC size cannot be decreased" // Server is the Schema for the servers API. type Server struct { @@ -291,6 +263,11 @@ type ServerList struct { Items []Server `json:"items"` } +// GetStatefulSetName returns the name of the StatefulSet running the Klio server. +func (s *Server) GetStatefulSetName() string { + return s.Name + "-klio" +} + // GetServiceName returns the name of the service associated with the Klio server. func (s *Server) GetServiceName() string { return s.Name diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index ea0d72789..9ece6c0d7 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -30,22 +30,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Cache) DeepCopyInto(out *Cache) { - *out = *in - in.PersistentVolumeClaimTemplate.DeepCopyInto(&out.PersistentVolumeClaimTemplate) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cache. -func (in *Cache) DeepCopy() *Cache { - if in == nil { - return nil - } - out := new(Cache) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CompressionPolicy) DeepCopyInto(out *CompressionPolicy) { *out = *in @@ -61,22 +45,6 @@ func (in *CompressionPolicy) DeepCopy() *CompressionPolicy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Data) DeepCopyInto(out *Data) { - *out = *in - in.PersistentVolumeClaimTemplate.DeepCopyInto(&out.PersistentVolumeClaimTemplate) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Data. -func (in *Data) DeepCopy() *Data { - if in == nil { - return nil - } - out := new(Data) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EmbeddedObjectMeta) DeepCopyInto(out *EmbeddedObjectMeta) { *out = *in @@ -297,22 +265,6 @@ func (in *PodTemplateSpec) DeepCopy() *PodTemplateSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Queue) DeepCopyInto(out *Queue) { - *out = *in - in.PersistentVolumeClaimTemplate.DeepCopyInto(&out.PersistentVolumeClaimTemplate) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Queue. -func (in *Queue) DeepCopy() *Queue { - if in == nil { - return nil - } - out := new(Queue) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RetentionPolicy) DeepCopyInto(out *RetentionPolicy) { *out = *in @@ -467,11 +419,7 @@ func (in *ServerSpec) DeepCopyInto(out *ServerSpec) { *out = new(Tier2Configuration) (*in).DeepCopyInto(*out) } - if in.Queue != nil { - in, out := &in.Queue, &out.Queue - *out = new(Queue) - (*in).DeepCopyInto(*out) - } + in.Storage.DeepCopyInto(&out.Storage) if in.Template != nil { in, out := &in.Template, &out.Template *out = new(PodTemplateSpec) @@ -504,6 +452,22 @@ func (in *ServerStatus) DeepCopy() *ServerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Storage) DeepCopyInto(out *Storage) { + *out = *in + in.PersistentVolumeClaimTemplate.DeepCopyInto(&out.PersistentVolumeClaimTemplate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Storage. +func (in *Storage) DeepCopy() *Storage { + if in == nil { + return nil + } + out := new(Storage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSConfiguration) DeepCopyInto(out *TLSConfiguration) { *out = *in @@ -522,8 +486,6 @@ func (in *TLSConfiguration) DeepCopy() *TLSConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tier1Configuration) DeepCopyInto(out *Tier1Configuration) { *out = *in - in.Cache.DeepCopyInto(&out.Cache) - in.Data.DeepCopyInto(&out.Data) in.EncryptionKeyFile.DeepCopyInto(&out.EncryptionKeyFile) in.IdentityFile.DeepCopyInto(&out.IdentityFile) if in.Compression != nil { @@ -571,7 +533,6 @@ func (in *Tier1PluginConfiguration) DeepCopy() *Tier1PluginConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tier2Configuration) DeepCopyInto(out *Tier2Configuration) { *out = *in - in.Cache.DeepCopyInto(&out.Cache) if in.S3 != nil { in, out := &in.S3, &out.S3 *out = new(S3Configuration) diff --git a/operator/config/crd/bases/klio.cnpg.io_servers.yaml b/operator/config/crd/bases/klio.cnpg.io_servers.yaml index 9fe98a45b..1d66bb85b 100644 --- a/operator/config/crd/bases/klio.cnpg.io_servers.yaml +++ b/operator/config/crd/bases/klio.cnpg.io_servers.yaml @@ -82,15 +82,17 @@ spec: x-kubernetes-validations: - message: mode is immutable rule: self == oldSelf - queue: + storage: description: |- - Queue is the configuration of the PVC that should host - the task queue. + Storage is the configuration of the single PersistentVolumeClaim + mounted at /klio, hosting base backups, WAL, the work queue, and + the Tier 1/Tier 2 caches as fixed subdirectories (data, queue, + cache_tier1, cache_tier2). properties: pvcTemplate: description: |- - PersistentVolumeClaimTemplate is used to generate the configuration for - the PVC hosting the work queue. + PersistentVolumeClaimTemplate is used to generate the PVC that + backs the /klio directory tree for this server. properties: accessModes: description: |- @@ -9104,210 +9106,6 @@ spec: tier1: description: Tier1 is the Tier 1 configuration properties: - cache: - description: |- - Cache is the configuration of the PVC that should be - used for the cache. - properties: - pvcTemplate: - description: |- - PersistentVolumeClaimSpec describes the common attributes of storage devices - and allows a Source for provider-specific attributes - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to - consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the - PersistentVolume backing this claim. - type: string - type: object - required: - - pvcTemplate - type: object compression: description: |- Compression defines the repository-wide (global) compression policy @@ -9357,210 +9155,6 @@ spec: - message: minSize must not be greater than maxSize rule: '!has(self.maxSize) || self.maxSize == 0 || !has(self.minSize) || self.minSize <= self.maxSize' - data: - description: |- - Data is the configuration of the PVC that should be used - for the base backups. - properties: - pvcTemplate: - description: |- - Template to be used to generate the Persistent Volume Claim needed for the data folder, - containing base backups and WAL files. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to - consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the - PersistentVolume backing this claim. - type: string - type: object - required: - - pvcTemplate - type: object encryptionKeyFile: description: EncryptionKeyFile specifies the Age-encrypted encryption key file. @@ -13647,218 +13241,12 @@ spec: rule: '[has(self.fileReference)].filter(x,x==true).size() == 1' required: - - cache - - data - encryptionKeyFile - identityFile type: object tier2: description: Tier2 is the Tier 2 configuration properties: - cache: - description: |- - Cache is the configuration of the PVC that should be - used for the cache. - properties: - pvcTemplate: - description: |- - PersistentVolumeClaimSpec describes the common attributes of storage devices - and allows a Source for provider-specific attributes - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to - consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the - PersistentVolume backing this claim. - type: string - type: object - required: - - pvcTemplate - type: object compression: description: |- Compression defines the repository-wide (global) compression policy @@ -18067,7 +17455,6 @@ spec: - bucketName type: object required: - - cache - encryptionKeyFile - identityFile - s3 @@ -18081,6 +17468,7 @@ spec: - caSecretName - image - mode + - storage - tlsSecretName type: object x-kubernetes-validations: @@ -18090,12 +17478,13 @@ spec: rule: self.mode != 'read-only' || has(self.tier2) - message: tier1 cannot be set when mode is read-only rule: '!(self.mode == ''read-only'' && has(self.tier1))' - - message: queue cannot be set when mode is read-only - rule: '!(self.mode == ''read-only'' && has(self.queue))' - message: tier2.compression cannot be set when mode is read-only rule: '!(self.mode == ''read-only'' && has(self.tier2) && has(self.tier2.compression))' - - message: queue is required when tier1 is configured - rule: self.mode == 'read-only' || has(self.queue) + - message: storage PVC size cannot be decreased + rule: '!has(oldSelf.storage) || !has(oldSelf.storage.pvcTemplate.resources) + || !has(oldSelf.storage.pvcTemplate.resources.requests) || !(''storage'' + in oldSelf.storage.pvcTemplate.resources.requests) || !(''storage'' + in self.storage.pvcTemplate.resources.requests) || !quantity(self.storage.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.storage.pvcTemplate.resources.requests[''storage'']))' status: description: ServerStatus defines the observed state of Server. type: object @@ -18103,23 +17492,6 @@ spec: - metadata - spec type: object - x-kubernetes-validations: - - message: tier1.data PVC size cannot be decreased - rule: '!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !(''storage'' - in oldSelf.spec.tier1.data.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.tier1.data.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.data.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.tier1.data.pvcTemplate.resources.requests[''storage'']))' - - message: tier1.cache PVC size cannot be decreased - rule: '!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !(''storage'' - in oldSelf.spec.tier1.cache.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.tier1.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.cache.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.tier1.cache.pvcTemplate.resources.requests[''storage'']))' - - message: tier2.cache PVC size cannot be decreased - rule: '!has(oldSelf.spec.tier2) || !has(self.spec.tier2) || !(''storage'' - in oldSelf.spec.tier2.cache.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.tier2.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier2.cache.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.tier2.cache.pvcTemplate.resources.requests[''storage'']))' - - message: queue PVC size cannot be decreased - rule: '!has(oldSelf.spec.queue) || !has(self.spec.queue) || !(''storage'' - in oldSelf.spec.queue.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.queue.pvcTemplate.resources.requests) || !quantity(self.spec.queue.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.queue.pvcTemplate.resources.requests[''storage'']))' served: true storage: true subresources: diff --git a/operator/config/manifests/bases/klio-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/klio-operator.clusterserviceversion.yaml index fdc95bd65..e975f1d29 100644 --- a/operator/config/manifests/bases/klio-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/klio-operator.clusterserviceversion.yaml @@ -29,9 +29,9 @@ spec: owned: - description: Server is the Schema for the Klio backup server API. It manages a StatefulSet running the Klio server together with the - PersistentVolumeClaims that back the cache, the data catalog, and - the task queue, and wires the server to the TLS material used for - client authentication. + PersistentVolumeClaim that backs its base backups, WAL + archive, Kopia caches, and task queue, and wires the server to + the TLS material used for client authentication. displayName: Klio Server kind: Server name: servers.klio.cnpg.io @@ -80,24 +80,16 @@ spec: path: caSecretName x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret + - description: Configuration for the PersistentVolumeClaim backing + the server's base backups, WAL archive, Kopia caches, and task queue. + displayName: Storage + path: storage + - description: Template used to provision the server's PVC. + displayName: Storage PVC Template + path: storage.pvcTemplate - description: Configuration for the local (Tier 1) storage layer. displayName: Tier 1 Configuration path: tier1 - - description: Local cache layer used by the Klio server when staging Tier 1 - data. - displayName: Tier 1 Cache - path: tier1.cache - - description: Template used to provision the Tier 1 cache PVC. - displayName: Tier 1 Cache PVC Template - path: tier1.cache.pvcTemplate - - description: Persistent data catalog backing the Tier 1 base backups and - WAL files. - displayName: Tier 1 Data - path: tier1.data - - description: Template used to provision the Tier 1 data PVC that hosts base - backups and WAL files. - displayName: Tier 1 Data PVC Template - path: tier1.data.pvcTemplate - description: Age-encrypted encryption key used to encrypt backups at rest. displayName: Tier 1 Encryption Key File path: tier1.encryptionKeyFile @@ -108,13 +100,6 @@ spec: - description: Configuration for the remote (Tier 2) object storage layer. displayName: Tier 2 Configuration path: tier2 - - description: Local cache layer used by the Klio server when staging Tier 2 - data. - displayName: Tier 2 Cache - path: tier2.cache - - description: Template used to provision the Tier 2 cache PVC. - displayName: Tier 2 Cache PVC Template - path: tier2.cache.pvcTemplate - description: S3-compatible object storage settings for the Tier 2 layer. displayName: S3 Configuration path: tier2.s3 @@ -184,13 +169,6 @@ spec: - description: Age identity file used to decrypt the Tier 2 encryption key. displayName: Tier 2 Identity File path: tier2.identityFile - - description: Configuration for the task queue used to schedule backup and - WAL operations. - displayName: Task Queue - path: queue - - description: Template used to provision the PVC that backs the task queue. - displayName: Task Queue PVC Template - path: queue.pvcTemplate - description: Advanced pod/StatefulSet template override. Modifying this field can break server functionality; use at your own risk. displayName: Pod Template Override diff --git a/operator/config/samples/klio_v1alpha1_server.yaml b/operator/config/samples/klio_v1alpha1_server.yaml index 7eb25e0f7..0e691961e 100644 --- a/operator/config/samples/klio_v1alpha1_server.yaml +++ b/operator/config/samples/klio_v1alpha1_server.yaml @@ -6,22 +6,6 @@ metadata: klio.cnpg.io/pprof: "true" spec: tier1: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - - data: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - encryptionKeyFile: fileReference: volume: @@ -36,11 +20,11 @@ spec: secretName: server-sample-encryption path: secret-key - queue: + storage: pvcTemplate: resources: requests: - storage: 100Mi + storage: 3Gi accessModes: - ReadWriteOnce diff --git a/operator/config/samples/klio_v1alpha1_server_readonly.yaml b/operator/config/samples/klio_v1alpha1_server_readonly.yaml index 4f32e0e16..f319b095d 100644 --- a/operator/config/samples/klio_v1alpha1_server_readonly.yaml +++ b/operator/config/samples/klio_v1alpha1_server_readonly.yaml @@ -14,13 +14,6 @@ metadata: spec: mode: read-only tier2: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce encryptionKeyFile: fileReference: volume: @@ -48,6 +41,14 @@ spec: name: rustfs-tls key: tls.crt + storage: + pvcTemplate: + resources: + requests: + storage: 1Gi + accessModes: + - ReadWriteOnce + tlsSecretName: server-ro-tls caSecretName: server-ro-ca diff --git a/operator/config/samples/klio_v1alpha1_server_tier2.yaml b/operator/config/samples/klio_v1alpha1_server_tier2.yaml index 4d38f832c..3db3263c4 100644 --- a/operator/config/samples/klio_v1alpha1_server_tier2.yaml +++ b/operator/config/samples/klio_v1alpha1_server_tier2.yaml @@ -13,22 +13,6 @@ metadata: klio.cnpg.io/pprof: "true" spec: tier1: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - - data: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - encryptionKeyFile: fileReference: volume: @@ -44,13 +28,6 @@ spec: path: secret-key tier2: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce encryptionKeyFile: fileReference: volume: @@ -80,11 +57,11 @@ spec: name: rustfs-tls key: tls.crt - queue: + storage: pvcTemplate: resources: requests: - storage: 100Mi + storage: 3Gi accessModes: - ReadWriteOnce diff --git a/operator/config/samples/opentelemetry/single/server/server.yaml b/operator/config/samples/opentelemetry/single/server/server.yaml index 4c71c07b6..0219b6a1c 100644 --- a/operator/config/samples/opentelemetry/single/server/server.yaml +++ b/operator/config/samples/opentelemetry/single/server/server.yaml @@ -3,30 +3,14 @@ kind: Server metadata: name: klio spec: - queue: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: + storage: + pvcTemplate: + resources: + requests: + storage: 2Gi + accessModes: - ReadWriteOnce tier1: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - - data: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - encryptionKeyFile: fileReference: volume: diff --git a/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_a.yaml b/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_a.yaml index 05fcd9fce..bdcd8cf10 100644 --- a/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_a.yaml +++ b/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_a.yaml @@ -11,22 +11,6 @@ metadata: namespace: dc-a spec: tier1: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - - data: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - encryptionKeyFile: fileReference: volume: @@ -41,11 +25,11 @@ spec: secretName: klioserver-dc-a-encryption path: secret-key - queue: + storage: pvcTemplate: resources: requests: - storage: 100Mi + storage: 2Gi accessModes: - ReadWriteOnce diff --git a/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_b.yaml b/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_b.yaml index 95dffd217..c4cf54a86 100644 --- a/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_b.yaml +++ b/operator/config/samples/replica-clusters/klio_v1alpha1_server_dc_b.yaml @@ -13,22 +13,6 @@ metadata: klio.cnpg.io/pprof: "true" spec: tier1: - cache: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - - data: - pvcTemplate: - resources: - requests: - storage: 1Gi - accessModes: - - ReadWriteOnce - encryptionKeyFile: fileReference: volume: @@ -43,11 +27,11 @@ spec: secretName: klioserver-dc-b-encryption path: secret-key - queue: + storage: pvcTemplate: resources: requests: - storage: 100Mi + storage: 2Gi accessModes: - ReadWriteOnce diff --git a/operator/dist/chart/crds/server-crd.yaml b/operator/dist/chart/crds/server-crd.yaml index 49e3ab966..077665db1 100644 --- a/operator/dist/chart/crds/server-crd.yaml +++ b/operator/dist/chart/crds/server-crd.yaml @@ -81,15 +81,17 @@ spec: x-kubernetes-validations: - message: mode is immutable rule: self == oldSelf - queue: + storage: description: |- - Queue is the configuration of the PVC that should host - the task queue. + Storage is the configuration of the single PersistentVolumeClaim + mounted at /klio, hosting base backups, WAL, the work queue, and + the Tier 1/Tier 2 caches as fixed subdirectories (data, queue, + cache_tier1, cache_tier2). properties: pvcTemplate: description: |- - PersistentVolumeClaimTemplate is used to generate the configuration for - the PVC hosting the work queue. + PersistentVolumeClaimTemplate is used to generate the PVC that + backs the /klio directory tree for this server. properties: accessModes: description: |- @@ -9103,210 +9105,6 @@ spec: tier1: description: Tier1 is the Tier 1 configuration properties: - cache: - description: |- - Cache is the configuration of the PVC that should be - used for the cache. - properties: - pvcTemplate: - description: |- - PersistentVolumeClaimSpec describes the common attributes of storage devices - and allows a Source for provider-specific attributes - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to - consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the - PersistentVolume backing this claim. - type: string - type: object - required: - - pvcTemplate - type: object compression: description: |- Compression defines the repository-wide (global) compression policy @@ -9356,210 +9154,6 @@ spec: - message: minSize must not be greater than maxSize rule: '!has(self.maxSize) || self.maxSize == 0 || !has(self.minSize) || self.minSize <= self.maxSize' - data: - description: |- - Data is the configuration of the PVC that should be used - for the base backups. - properties: - pvcTemplate: - description: |- - Template to be used to generate the Persistent Volume Claim needed for the data folder, - containing base backups and WAL files. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to - consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the - PersistentVolume backing this claim. - type: string - type: object - required: - - pvcTemplate - type: object encryptionKeyFile: description: EncryptionKeyFile specifies the Age-encrypted encryption key file. @@ -13646,218 +13240,12 @@ spec: rule: '[has(self.fileReference)].filter(x,x==true).size() == 1' required: - - cache - - data - encryptionKeyFile - identityFile type: object tier2: description: Tier2 is the Tier 2 configuration properties: - cache: - description: |- - Cache is the configuration of the PVC that should be - used for the cache. - properties: - pvcTemplate: - description: |- - PersistentVolumeClaimSpec describes the common attributes of storage devices - and allows a Source for provider-specific attributes - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - Users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to - consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the - PersistentVolume backing this claim. - type: string - type: object - required: - - pvcTemplate - type: object compression: description: |- Compression defines the repository-wide (global) compression policy @@ -18066,7 +17454,6 @@ spec: - bucketName type: object required: - - cache - encryptionKeyFile - identityFile - s3 @@ -18080,6 +17467,7 @@ spec: - caSecretName - image - mode + - storage - tlsSecretName type: object x-kubernetes-validations: @@ -18089,12 +17477,13 @@ spec: rule: self.mode != 'read-only' || has(self.tier2) - message: tier1 cannot be set when mode is read-only rule: '!(self.mode == ''read-only'' && has(self.tier1))' - - message: queue cannot be set when mode is read-only - rule: '!(self.mode == ''read-only'' && has(self.queue))' - message: tier2.compression cannot be set when mode is read-only rule: '!(self.mode == ''read-only'' && has(self.tier2) && has(self.tier2.compression))' - - message: queue is required when tier1 is configured - rule: self.mode == 'read-only' || has(self.queue) + - message: storage PVC size cannot be decreased + rule: '!has(oldSelf.storage) || !has(oldSelf.storage.pvcTemplate.resources) + || !has(oldSelf.storage.pvcTemplate.resources.requests) || !(''storage'' + in oldSelf.storage.pvcTemplate.resources.requests) || !(''storage'' + in self.storage.pvcTemplate.resources.requests) || !quantity(self.storage.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.storage.pvcTemplate.resources.requests[''storage'']))' status: description: ServerStatus defines the observed state of Server. type: object @@ -18102,23 +17491,6 @@ spec: - metadata - spec type: object - x-kubernetes-validations: - - message: tier1.data PVC size cannot be decreased - rule: '!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !(''storage'' - in oldSelf.spec.tier1.data.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.tier1.data.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.data.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.tier1.data.pvcTemplate.resources.requests[''storage'']))' - - message: tier1.cache PVC size cannot be decreased - rule: '!has(oldSelf.spec.tier1) || !has(self.spec.tier1) || !(''storage'' - in oldSelf.spec.tier1.cache.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.tier1.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier1.cache.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.tier1.cache.pvcTemplate.resources.requests[''storage'']))' - - message: tier2.cache PVC size cannot be decreased - rule: '!has(oldSelf.spec.tier2) || !has(self.spec.tier2) || !(''storage'' - in oldSelf.spec.tier2.cache.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.tier2.cache.pvcTemplate.resources.requests) || !quantity(self.spec.tier2.cache.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.tier2.cache.pvcTemplate.resources.requests[''storage'']))' - - message: queue PVC size cannot be decreased - rule: '!has(oldSelf.spec.queue) || !has(self.spec.queue) || !(''storage'' - in oldSelf.spec.queue.pvcTemplate.resources.requests) || !(''storage'' - in self.spec.queue.pvcTemplate.resources.requests) || !quantity(self.spec.queue.pvcTemplate.resources.requests[''storage'']).isLessThan(quantity(oldSelf.spec.queue.pvcTemplate.resources.requests[''storage'']))' served: true storage: true subresources: diff --git a/operator/internal/controller/server_controller_test.go b/operator/internal/controller/server_controller_test.go index 67f70d061..766f1d0e6 100644 --- a/operator/internal/controller/server_controller_test.go +++ b/operator/internal/controller/server_controller_test.go @@ -23,9 +23,12 @@ import ( "context" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -36,6 +39,8 @@ import ( . "github.com/onsi/gomega" ) +const serverCRDName = "servers.klio.cnpg.io" + var _ = Describe("Server Controller", func() { Context("When reconciling a resource", func() { const resourceName = "test-resource" @@ -74,13 +79,10 @@ var _ = Describe("Server Controller", func() { ClientCASecretName: "ca-secret", }, Mode: kliov1alpha1.ModeStandard, + Storage: kliov1alpha1.Storage{ + PersistentVolumeClaimTemplate: pvcTemplate, + }, Tier1: &kliov1alpha1.Tier1Configuration{ - Cache: kliov1alpha1.Cache{ - PersistentVolumeClaimTemplate: pvcTemplate, - }, - Data: kliov1alpha1.Data{ - PersistentVolumeClaimTemplate: pvcTemplate, - }, EncryptionKeyFile: kliov1alpha1.FileSource{ FileReference: &kliov1alpha1.FileReference{ Volume: corev1.VolumeSource{ @@ -98,9 +100,6 @@ var _ = Describe("Server Controller", func() { }, }, }, - Queue: &kliov1alpha1.Queue{ - PersistentVolumeClaimTemplate: pvcTemplate, - }, }, } Expect(k8sClient.Create(ctx, resource)).To(Succeed()) @@ -128,5 +127,141 @@ var _ = Describe("Server Controller", func() { }) Expect(err).NotTo(HaveOccurred()) }) + + It("should allow growing the storage PVC but reject shrinking it", func() { + resize := func(size string) error { + cur := &kliov1alpha1.Server{} + Expect(k8sClient.Get(ctx, typeNamespacedName, cur)).To(Succeed()) + requests := cur.Spec.Storage.PersistentVolumeClaimTemplate.Resources.Requests + requests[corev1.ResourceStorage] = resource.MustParse(size) + + return k8sClient.Update(ctx, cur) + } + + Expect(resize("2Gi")).To(Succeed(), "growing must be allowed") + Expect(resize("2Gi")).To(Succeed(), "keeping the same size must be allowed") + + err := resize("1Gi") + Expect(err).To(HaveOccurred(), "shrinking must be rejected") + Expect(err.Error()).To(ContainSubstring("storage PVC size cannot be decreased")) + }) + + It("should not error when adding spec.storage to a Server whose stored object lacks it", func() { + // spec.storage is required on write, but the required-field check + // does not apply retroactively: a Server stored while an older + // CRD schema was active can still be read back without it. The + // storage-shrink CEL rule runs on every update and reads + // oldSelf.storage, so it must handle that object shape without + // erroring. + storageName := "server-without-storage" + storageKey := types.NamespacedName{Name: storageName, Namespace: "default"} + + By("relaxing the CRD's required fields so a Server without spec.storage can be created") + crd := &apiextensionsv1.CustomResourceDefinition{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: serverCRDName}, crd)).To(Succeed()) + + versionIdx := -1 + for i := range crd.Spec.Versions { + if crd.Spec.Versions[i].Storage { + versionIdx = i + break + } + } + Expect(versionIdx).To(BeNumerically(">=", 0), "no storage version found for the Server CRD") + + specSchema := crd.Spec.Versions[versionIdx].Schema.OpenAPIV3Schema.Properties["spec"] + originalRequired := specSchema.Required + relaxedRequired := make([]string, 0, len(originalRequired)) + for _, field := range originalRequired { + if field != "storage" { + relaxedRequired = append(relaxedRequired, field) + } + } + specSchema.Required = relaxedRequired + crd.Spec.Versions[versionIdx].Schema.OpenAPIV3Schema.Properties["spec"] = specSchema + Expect(k8sClient.Update(ctx, crd)).To(Succeed()) + + DeferCleanup(func() { + By("restoring the storage requirement on the CRD") + restored := &apiextensionsv1.CustomResourceDefinition{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: serverCRDName}, restored)).To(Succeed()) + restoredSpecSchema := restored.Spec.Versions[versionIdx].Schema.OpenAPIV3Schema.Properties["spec"] + restoredSpecSchema.Required = originalRequired + restored.Spec.Versions[versionIdx].Schema.OpenAPIV3Schema.Properties["spec"] = restoredSpecSchema + Expect(k8sClient.Update(ctx, restored)).To(Succeed()) + }) + + By("creating a Server with no spec.storage") + typed := &kliov1alpha1.Server{ + ObjectMeta: metav1.ObjectMeta{ + Name: storageName, + Namespace: "default", + }, + Spec: kliov1alpha1.ServerSpec{ + ImageConfiguration: kliov1alpha1.ImageConfiguration{ + Image: "klio:test", + }, + TLSConfiguration: kliov1alpha1.TLSConfiguration{ + TLSSecretName: "tls-secret", + ClientCASecretName: "ca-secret", + }, + Mode: kliov1alpha1.ModeStandard, + Tier1: &kliov1alpha1.Tier1Configuration{ + EncryptionKeyFile: kliov1alpha1.FileSource{ + FileReference: &kliov1alpha1.FileReference{ + Volume: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "enc-secret"}, + }, + Path: "encryption-key.age", + }, + }, + IdentityFile: kliov1alpha1.FileSource{ + FileReference: &kliov1alpha1.FileReference{ + Volume: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "id-secret"}, + }, + Path: "identity.txt", + }, + }, + }, + }, + } + asMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(typed) + Expect(err).NotTo(HaveOccurred()) + unstructured.RemoveNestedField(asMap, "spec", "storage") + + withoutStorage := &unstructured.Unstructured{Object: asMap} + withoutStorage.SetGroupVersionKind(kliov1alpha1.GroupVersion.WithKind("Server")) + Expect(k8sClient.Create(ctx, withoutStorage)).To(Succeed()) + + DeferCleanup(func() { + current := &unstructured.Unstructured{} + current.SetGroupVersionKind(kliov1alpha1.GroupVersion.WithKind("Server")) + Expect(k8sClient.Get(ctx, storageKey, current)).To(Succeed()) + Expect(k8sClient.Delete(ctx, current)).To(Succeed()) + }) + + By("restoring the CRD's required fields") + restored := &apiextensionsv1.CustomResourceDefinition{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: serverCRDName}, restored)).To(Succeed()) + restoredSpecSchema := restored.Spec.Versions[versionIdx].Schema.OpenAPIV3Schema.Properties["spec"] + restoredSpecSchema.Required = originalRequired + restored.Spec.Versions[versionIdx].Schema.OpenAPIV3Schema.Properties["spec"] = restoredSpecSchema + Expect(k8sClient.Update(ctx, restored)).To(Succeed()) + + By("updating the Server to add spec.storage") + current := &unstructured.Unstructured{} + current.SetGroupVersionKind(kliov1alpha1.GroupVersion.WithKind("Server")) + Expect(k8sClient.Get(ctx, storageKey, current)).To(Succeed()) + Expect(unstructured.SetNestedMap(current.Object, map[string]any{ + "accessModes": []any{"ReadWriteOnce"}, + "resources": map[string]any{ + "requests": map[string]any{"storage": "1Gi"}, + }, + }, "spec", "storage", "pvcTemplate")).To(Succeed()) + + err = k8sClient.Update(ctx, current) + Expect(err).NotTo(HaveOccurred(), "the storage-shrink CEL guard must not error when oldSelf has no storage") + }) }) }) diff --git a/operator/internal/controller/server_envbuilder.go b/operator/internal/controller/server_envbuilder.go index 19d3eeca4..fcc0d8550 100644 --- a/operator/internal/controller/server_envbuilder.go +++ b/operator/internal/controller/server_envbuilder.go @@ -92,8 +92,8 @@ func fileSourcePath(volName string, src kliov1alpha1.FileSource) string { } func (e *envBuilder) getCoreEnvVars() []corev1.EnvVar { - basePath := path.Join(kopiaDataMountPath, "base") - walPath := path.Join(kopiaDataMountPath, "wal") + basePath := path.Join(klioMountPath, "data", "base") + walPath := path.Join(klioMountPath, "data", "wal") result := []corev1.EnvVar{ { @@ -115,7 +115,7 @@ func (e *envBuilder) getCoreEnvVars() []corev1.EnvVar { tier1Envs = append(tier1Envs, corev1.EnvVar{ Name: "TIER1_BASE_CACHE", - Value: path.Join(kopiaCacheTier1MountPath, kopiaCacheSubdirectory), + Value: path.Join(klioMountPath, "cache_tier1", kopiaCacheSubdirectory), }, corev1.EnvVar{ Name: "TIER1_BASE_REPOSITORY", @@ -150,7 +150,7 @@ func (e *envBuilder) getCoreEnvVars() []corev1.EnvVar { // retention policy enforcement. tier1Envs = append(tier1Envs, corev1.EnvVar{ Name: "QUEUE_DIRECTORY", - Value: "/queue", + Value: path.Join(klioMountPath, "queue"), }) tier1Envs = appendCompressionEnvs(tier1Envs, "TIER1", e.tier1.Compression) @@ -180,7 +180,7 @@ func (e *envBuilder) getTier2EnvVars() []corev1.EnvVar { }, { Name: "TIER2_CACHE", - Value: path.Join(kopiaCacheTier2MountPath, kopiaCacheSubdirectory), + Value: path.Join(klioMountPath, "cache_tier2", kopiaCacheSubdirectory), }, { Name: "TIER2_BASE_LISTEN_ADDRESS", diff --git a/operator/internal/controller/server_envbuilder_test.go b/operator/internal/controller/server_envbuilder_test.go index 95da8dc70..4a8f925e6 100644 --- a/operator/internal/controller/server_envbuilder_test.go +++ b/operator/internal/controller/server_envbuilder_test.go @@ -63,7 +63,7 @@ func TestGetCoreEnvVarsIncludesQueueWhenTier1Configured(t *testing.T) { envVars := builder.getCoreEnvVars() queueDir := findEnvVar(envVars, "QUEUE_DIRECTORY") require.NotNil(t, queueDir) - assert.Equal(t, "/queue", queueDir.Value) + assert.Equal(t, "/klio/queue", queueDir.Value) } func TestGetCoreEnvVarsExcludesQueueWhenNoTier1(t *testing.T) { @@ -86,11 +86,18 @@ func TestGetCoreEnvVarsIncludesTier1EnvVars(t *testing.T) { envVars := builder.getCoreEnvVars() - assert.NotNil(t, findEnvVar(envVars, "TIER1_BASE_CACHE")) - assert.NotNil(t, findEnvVar(envVars, "TIER1_BASE_REPOSITORY")) + for name, want := range map[string]string{ + "TIER1_BASE_CACHE": "/klio/cache_tier1/kopia-cache", + "TIER1_BASE_REPOSITORY": "/klio/data/base", + "TIER1_WAL_PATH": "/klio/data/wal", + } { + env := findEnvVar(envVars, name) + require.NotNil(t, env, name) + assert.Equal(t, want, env.Value, name) + } + assert.NotNil(t, findEnvVar(envVars, "TIER1_BASE_LISTEN_ADDRESS")) assert.NotNil(t, findEnvVar(envVars, "TIER1_WAL_LISTEN_ADDRESS")) - assert.NotNil(t, findEnvVar(envVars, "TIER1_WAL_PATH")) encKeyFile := findEnvVar(envVars, "TIER1_ENCRYPTION_KEY_FILE") require.NotNil(t, encKeyFile) @@ -284,6 +291,10 @@ func TestGetTier2EnvVars(t *testing.T) { envVars := builder.getTier2EnvVars() + cache := findEnvVar(envVars, "TIER2_CACHE") + require.NotNil(t, cache) + assert.Equal(t, "/klio/cache_tier2/kopia-cache", cache.Value) + encKeyFile := findEnvVar(envVars, "TIER2_ENCRYPTION_KEY_FILE") require.NotNil(t, encKeyFile) assert.Equal(t, "/files/tier2-enc-key-file/encryption-key.age", encKeyFile.Value) @@ -361,6 +372,10 @@ func TestBuildVolumeMounts(t *testing.T) { require.NotNil(t, idMount) assert.Equal(t, "/files/tier1-identity", idMount.MountPath) assert.True(t, idMount.ReadOnly) + + klioMount := findMount("klio") + require.NotNil(t, klioMount) + assert.Equal(t, "/klio", klioMount.MountPath) } func TestBuildIdentityVolumeDefaultMode(t *testing.T) { diff --git a/operator/internal/controller/server_pvc_resize.go b/operator/internal/controller/server_pvc_resize.go index ee68c402b..e9cdae015 100644 --- a/operator/internal/controller/server_pvc_resize.go +++ b/operator/internal/controller/server_pvc_resize.go @@ -34,61 +34,36 @@ import ( kliov1alpha1 "github.com/cloudnative-pg/klio/operator/api/v1alpha1" ) -const ( - pvcTypeData = "data" - pvcTypeQueue = "queue" - pvcTypeCacheTier1 = "cachetier1" - pvcTypeCacheTier2 = "cachetier2" -) +// pvcTypeKlio is the name of the VolumeClaimTemplate backing the Server. +const pvcTypeKlio = "klio" + +// klioPVCName returns the name of the PVC backing a Server. The +// StatefulSet always runs a single replica, so the ordinal is always 0. +func klioPVCName(server *kliov1alpha1.Server) string { + return fmt.Sprintf("%s-%s-0", pvcTypeKlio, server.GetStatefulSetName()) +} -// reconcilePVCResizes handles PVC size expansion for all Server PVCs. +// reconcilePVCResize handles resizing the PVC backing the Server. // StatefulSet VolumeClaimTemplates are immutable, so we must patch PVCs directly. // Note: Only expansion is supported; shrinking PVCs is not possible in Kubernetes. // //nolint:unparam // Result is always zero but signature matches reconciler pattern for consistency. -func (r *ServerReconciler) reconcilePVCResizes(ctx context.Context, server *kliov1alpha1.Server) (ctrl.Result, error) { - desiredSizes := r.buildDesiredPVCSizes(server) - if len(desiredSizes) == 0 { +func (r *ServerReconciler) reconcilePVCResize(ctx context.Context, server *kliov1alpha1.Server) (ctrl.Result, error) { + desiredSize, ok := server.Spec.Storage.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage] + if !ok { return ctrl.Result{}, nil } - var pvcList corev1.PersistentVolumeClaimList - if err := r.List(ctx, &pvcList, - client.InNamespace(server.Namespace), - client.MatchingLabels{klioServerLabel: server.Name}, - ); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to list PVCs: %w", err) - } - - for i := range pvcList.Items { - pvc := &pvcList.Items[i] - if err := r.reconcileSinglePVCResize(ctx, server, pvc, desiredSizes); err != nil { - return ctrl.Result{}, err + var pvc corev1.PersistentVolumeClaim + if err := r.Get(ctx, client.ObjectKey{Namespace: server.Namespace, Name: klioPVCName(server)}, &pvc); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil } - } - return ctrl.Result{}, nil -} - -// reconcileSinglePVCResize handles the resize logic for a single PVC. -func (r *ServerReconciler) reconcileSinglePVCResize( - ctx context.Context, - server *kliov1alpha1.Server, - pvc *corev1.PersistentVolumeClaim, - desiredSizes map[string]resource.Quantity, -) error { - contextLogger := logf.FromContext(ctx) - - pvcType, exists := pvc.Labels[pvcTypeLabel] - if !exists { - return nil - } - - desiredSize, exists := desiredSizes[pvcType] - if !exists { - return nil + return ctrl.Result{}, fmt.Errorf("failed to get PVC: %w", err) } + contextLogger := logf.FromContext(ctx) currentSize := pvc.Spec.Resources.Requests[corev1.ResourceStorage] switch { @@ -98,19 +73,19 @@ func (r *ServerReconciler) reconcileSinglePVCResize( "currentSize", currentSize.String(), "desiredSize", desiredSize.String()) - return nil + return ctrl.Result{}, nil case desiredSize.Cmp(currentSize) == 0: - return nil + return ctrl.Result{}, nil } - if err := r.expandPVC(ctx, pvc, desiredSize, currentSize); err != nil { - return err + if err := r.expandPVC(ctx, &pvc, desiredSize, currentSize); err != nil { + return ctrl.Result{}, err } r.Recorder.Eventf(server, nil, corev1.EventTypeNormal, "PVCExpanded", "ResizePVC", "PVC %s expanded from %s to %s", pvc.Name, currentSize.String(), desiredSize.String()) - return nil + return ctrl.Result{}, nil } // expandPVC patches the PVC to expand its storage size. @@ -150,34 +125,6 @@ func (r *ServerReconciler) expandPVC( return nil } -// buildDesiredPVCSizes returns a map of PVC type labels to their desired sizes. -func (r *ServerReconciler) buildDesiredPVCSizes(server *kliov1alpha1.Server) map[string]resource.Quantity { - sizes := make(map[string]resource.Quantity) - - if server.Spec.Tier1 != nil { - if size, ok := server.Spec.Tier1.Data.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage]; ok { - sizes[pvcTypeData] = size - } - if size, ok := server.Spec.Tier1.Cache.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage]; ok { - sizes[pvcTypeCacheTier1] = size - } - } - - if server.Spec.Tier2 != nil { - if size, ok := server.Spec.Tier2.Cache.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage]; ok { - sizes[pvcTypeCacheTier2] = size - } - } - - if server.Spec.Queue != nil { - if size, ok := server.Spec.Queue.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage]; ok { - sizes[pvcTypeQueue] = size - } - } - - return sizes -} - // isVolumeExpansionError checks if the error indicates the StorageClass doesn't support volume expansion. func isVolumeExpansionError(err error) bool { if !apierrors.IsInvalid(err) && !apierrors.IsForbidden(err) { diff --git a/operator/internal/controller/server_pvc_resize_test.go b/operator/internal/controller/server_pvc_resize_test.go index e9a5429cb..d11be36f5 100644 --- a/operator/internal/controller/server_pvc_resize_test.go +++ b/operator/internal/controller/server_pvc_resize_test.go @@ -59,14 +59,15 @@ func newPVCSpec(size string) corev1.PersistentVolumeClaimSpec { } } -func newTestPVC(name, serverName, pvcType, size string) *corev1.PersistentVolumeClaim { +// newTestPVC builds a PVC as the StatefulSet would name it for the +// "test-server" Server: "-test-server-klio-0". +func newTestPVC(pvcType, size string) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: pvcType + "-test-server-klio-0", Namespace: "default", Labels: map[string]string{ - klioServerLabel: serverName, - pvcTypeLabel: pvcType, + klioServerLabel: "test-server", }, }, Spec: newPVCSpec(size), @@ -88,84 +89,26 @@ func newTestReconciler(objs ...client.Object) (*ServerReconciler, client.Client) }, fakeClient } -func newTestServerTier1(dataSize, cacheSize string) *kliov1alpha1.Server { +func newTestServerTier1(storageSize string) *kliov1alpha1.Server { return &kliov1alpha1.Server{ ObjectMeta: metav1.ObjectMeta{Name: "test-server", Namespace: "default"}, Spec: kliov1alpha1.ServerSpec{ - Tier1: &kliov1alpha1.Tier1Configuration{ - Data: kliov1alpha1.Data{PersistentVolumeClaimTemplate: newPVCSpec(dataSize)}, - Cache: kliov1alpha1.Cache{PersistentVolumeClaimTemplate: newPVCSpec(cacheSize)}, - }, + Storage: kliov1alpha1.Storage{PersistentVolumeClaimTemplate: newPVCSpec(storageSize)}, + Tier1: &kliov1alpha1.Tier1Configuration{}, }, } } -// --- buildDesiredPVCSizes tests --- - -func TestBuildDesiredPVCSizesTier1Only(t *testing.T) { - server := newTestServerTier1("100Gi", "10Gi") - server.Spec.Queue = &kliov1alpha1.Queue{PersistentVolumeClaimTemplate: newPVCSpec("5Gi")} - - sizes := (&ServerReconciler{}).buildDesiredPVCSizes(server) - - require.Len(t, sizes, 3) - assert.Equal(t, resource.MustParse("100Gi"), sizes[pvcTypeData]) - assert.Equal(t, resource.MustParse("10Gi"), sizes[pvcTypeCacheTier1]) - assert.Equal(t, resource.MustParse("5Gi"), sizes[pvcTypeQueue]) -} - -func TestBuildDesiredPVCSizesTier2Only(t *testing.T) { - server := &kliov1alpha1.Server{ - Spec: kliov1alpha1.ServerSpec{ - Mode: kliov1alpha1.ModeReadOnly, - Tier2: &kliov1alpha1.Tier2Configuration{ - Cache: kliov1alpha1.Cache{PersistentVolumeClaimTemplate: newPVCSpec("20Gi")}, - S3: &kliov1alpha1.S3Configuration{BucketName: "test-bucket"}, - }, - }, - } - - sizes := (&ServerReconciler{}).buildDesiredPVCSizes(server) - - require.Len(t, sizes, 1) - assert.Equal(t, resource.MustParse("20Gi"), sizes[pvcTypeCacheTier2]) -} - -func TestBuildDesiredPVCSizesBothTiers(t *testing.T) { - server := newTestServerTier1("100Gi", "10Gi") - server.Spec.Tier2 = &kliov1alpha1.Tier2Configuration{ - Cache: kliov1alpha1.Cache{PersistentVolumeClaimTemplate: newPVCSpec("20Gi")}, - S3: &kliov1alpha1.S3Configuration{BucketName: "test-bucket"}, - } - server.Spec.Queue = &kliov1alpha1.Queue{PersistentVolumeClaimTemplate: newPVCSpec("5Gi")} - - sizes := (&ServerReconciler{}).buildDesiredPVCSizes(server) - - require.Len(t, sizes, 4) - assert.Equal(t, resource.MustParse("100Gi"), sizes[pvcTypeData]) - assert.Equal(t, resource.MustParse("10Gi"), sizes[pvcTypeCacheTier1]) - assert.Equal(t, resource.MustParse("20Gi"), sizes[pvcTypeCacheTier2]) - assert.Equal(t, resource.MustParse("5Gi"), sizes[pvcTypeQueue]) -} - -func TestBuildDesiredPVCSizesEmptyServer(t *testing.T) { - sizes := (&ServerReconciler{}).buildDesiredPVCSizes(&kliov1alpha1.Server{}) - assert.Empty(t, sizes) -} +// --- reconcilePVCResize tests --- -func TestBuildDesiredPVCSizesNoStorageRequests(t *testing.T) { - server := &kliov1alpha1.Server{ - Spec: kliov1alpha1.ServerSpec{ - Tier1: &kliov1alpha1.Tier1Configuration{}, - }, - } +func TestReconcilePVCResizesNoStorageRequests(t *testing.T) { + reconciler, _ := newTestReconciler() - sizes := (&ServerReconciler{}).buildDesiredPVCSizes(server) - assert.Empty(t, sizes) + result, err := reconciler.reconcilePVCResize(context.Background(), &kliov1alpha1.Server{}) + require.NoError(t, err) + assert.True(t, result.IsZero()) } -// --- reconcilePVCResizes tests --- - func TestReconcilePVCResizes(t *testing.T) { testCases := []struct { name string @@ -195,17 +138,17 @@ func TestReconcilePVCResizes(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - pvc := newTestPVC("data-test-server-klio-0", "test-server", pvcTypeData, tc.pvcCurrentSize) + pvc := newTestPVC(pvcTypeKlio, tc.pvcCurrentSize) reconciler, fakeClient := newTestReconciler(pvc) - server := newTestServerTier1(tc.serverDesiredSize, "5Gi") + server := newTestServerTier1(tc.serverDesiredSize) - result, err := reconciler.reconcilePVCResizes(context.Background(), server) + result, err := reconciler.reconcilePVCResize(context.Background(), server) require.NoError(t, err) assert.True(t, result.IsZero(), "should not requeue") var updatedPVC corev1.PersistentVolumeClaim require.NoError(t, fakeClient.Get(context.Background(), client.ObjectKey{ - Name: "data-test-server-klio-0", Namespace: "default", + Name: "klio-test-server-klio-0", Namespace: "default", }, &updatedPVC)) expectedSize := resource.MustParse(tc.expectedPVCSize) @@ -218,31 +161,49 @@ func TestReconcilePVCResizes(t *testing.T) { func TestReconcilePVCResizesNoPVCsExist(t *testing.T) { reconciler, _ := newTestReconciler() - server := newTestServerTier1("20Gi", "5Gi") + server := newTestServerTier1("20Gi") - result, err := reconciler.reconcilePVCResizes(context.Background(), server) + result, err := reconciler.reconcilePVCResize(context.Background(), server) require.NoError(t, err) assert.True(t, result.IsZero()) } -func TestReconcilePVCResizesOrphanedPVCIgnored(t *testing.T) { - // PVC for tier2 cache exists, but server only has tier1 - pvc := newTestPVC("cachetier2-test-server-klio-0", "test-server", pvcTypeCacheTier2, "10Gi") - reconciler, fakeClient := newTestReconciler(pvc) - server := newTestServerTier1("20Gi", "5Gi") - - result, err := reconciler.reconcilePVCResizes(context.Background(), server) +// TestReconcilePVCResizesIgnoresOtherPVCs asserts that reconcilePVCResize +// only ever touches the PVC named after the unified volume claim template. +// A server that is mid-manual-migration off the old per-purpose layout can +// still have those PVCs around, because the StatefulSet's +// PersistentVolumeClaimRetentionPolicy is Retain — reconcilePVCResize gets +// the unified PVC by its exact name, so any differently-named PVC is never +// even looked at. +func TestReconcilePVCResizesIgnoresOtherPVCs(t *testing.T) { + other := newTestPVC("data", "10Gi") + current := newTestPVC(pvcTypeKlio, "10Gi") + + reconciler, fakeClient := newTestReconciler(current, other) + server := newTestServerTier1("20Gi") + + result, err := reconciler.reconcilePVCResize(context.Background(), server) require.NoError(t, err) assert.True(t, result.IsZero()) - var updatedPVC corev1.PersistentVolumeClaim - require.NoError(t, fakeClient.Get(context.Background(), client.ObjectKey{ - Name: "cachetier2-test-server-klio-0", Namespace: "default", - }, &updatedPVC)) - expectedSize := resource.MustParse("10Gi") - actualSize := updatedPVC.Spec.Resources.Requests[corev1.ResourceStorage] - assert.Equal(t, 0, expectedSize.Cmp(actualSize), - "orphaned tier2 PVC should not be modified") + sizeOf := func(name string) resource.Quantity { + var pvc corev1.PersistentVolumeClaim + require.NoError(t, fakeClient.Get(context.Background(), client.ObjectKey{ + Name: name, Namespace: "default", + }, &pvc)) + + return pvc.Spec.Resources.Requests[corev1.ResourceStorage] + } + + expandedSize := resource.MustParse("20Gi") + actualSize := sizeOf(current.Name) + assert.Equal(t, 0, expandedSize.Cmp(actualSize), + "unified PVC should have been expanded, got %s", actualSize.String()) + + untouchedSize := resource.MustParse("10Gi") + actualSize = sizeOf(other.Name) + assert.Equal(t, 0, untouchedSize.Cmp(actualSize), + "other PVC should not be modified, got %s", actualSize.String()) } // --- findServerForPVC tests --- diff --git a/operator/internal/controller/server_reconciler.go b/operator/internal/controller/server_reconciler.go index 49d5731fa..e83675116 100644 --- a/operator/internal/controller/server_reconciler.go +++ b/operator/internal/controller/server_reconciler.go @@ -41,12 +41,7 @@ import ( "github.com/cloudnative-pg/klio/operator/internal/podtemplate" ) -const ( - pvcTypeLabel = "klio.cnpg.io/pvcType" - typeLabel = "klio.cnpg.io/type" - baseTypeLabelValue = "base" - klioServerLabel = "klio.cnpg.io/klio-server" -) +const klioServerLabel = "klio.cnpg.io/klio-server" var errNilFileReference = errors.New("fileReference is not set in FileSource") @@ -75,9 +70,9 @@ func validateFileSources(server *kliov1alpha1.Server) error { } const ( - kopiaDataMountPath = "/data" - kopiaCacheTier1MountPath = "/cache_tier1" - kopiaCacheTier2MountPath = "/cache_tier2" + // klioMountPath is where the PVC is mounted. All the server state + // lives under it as fixed subdirectories. + klioMountPath = "/klio" fileSourceBasePath = "/files" tier1EncKeyFileVolName = "tier1-enc-key-file" @@ -94,9 +89,9 @@ func (r *ServerReconciler) reconcile(ctx context.Context, server *kliov1alpha1.S // Reconcile PVC resizes before StatefulSet to ensure PVCs are expanded // before the StatefulSet is recreated. VolumeClaimTemplates only define // specs for new PVCs, so explicit patching is required to resize existing ones. - if result, err := r.reconcilePVCResizes(ctx, server); err != nil || !result.IsZero() { + if result, err := r.reconcilePVCResize(ctx, server); err != nil || !result.IsZero() { if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to reconcile PVC resizes: %w", err) + return ctrl.Result{}, fmt.Errorf("failed to reconcile PVC resize: %w", err) } return result, nil @@ -110,7 +105,7 @@ func (r *ServerReconciler) reconcileStatefulSet( ctx context.Context, server *kliov1alpha1.Server, ) (ctrl.Result, error) { contextLogger := logf.FromContext(ctx) - klioName := server.Name + "-klio" + klioName := server.GetStatefulSetName() pprof, _ := strconv.ParseBool(server.GetAnnotations()["klio.cnpg.io/pprof"]) @@ -139,7 +134,6 @@ func (r *ServerReconciler) reconcileStatefulSet( Namespace: server.Namespace, Labels: map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, }, Annotations: map[string]string{}, }, @@ -156,14 +150,12 @@ func (r *ServerReconciler) reconcileStatefulSet( Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, }, }, Spec: corev1.PodSpec{ @@ -192,18 +184,7 @@ func (r *ServerReconciler) reconcileStatefulSet( Status: appsv1.StatefulSetStatus{}, } - if server.Spec.Tier1 != nil { - injectTier1VolumeClaimTemplates(expected, *server) - } - - if server.Spec.Queue != nil { - injectQueueConfiguration(expected, *server) - } - - // Add Tier2 containers if the server has Tier 2 configuration - if server.Spec.Tier2 != nil { - injectTier2VolumeClaimTemplates(expected, *server) - } + injectVolumeClaimTemplate(expected, server) if server.Spec.Template != nil { merged, err := podtemplate.Merge(&expected.Spec.Template, server.Spec.Template.ToCoreV1()) @@ -217,7 +198,6 @@ func (r *ServerReconciler) reconcileStatefulSet( expected.Spec.Template.Labels = map[string]string{} } expected.Spec.Template.Labels[klioServerLabel] = server.Name - expected.Spec.Template.Labels[typeLabel] = baseTypeLabelValue } // Append pprof args after merge so overlay replacements of Args cannot drop them @@ -307,16 +287,18 @@ func (r *ServerReconciler) reconcileStatefulSet( return ctrl.Result{}, nil } -func injectQueueConfiguration(expected *appsv1.StatefulSet, server kliov1alpha1.Server) { +// injectVolumeClaimTemplate appends the PVC template backing +// the /klio directory tree. Every server gets it, whatever tiers are enabled: +// the core creates only the subdirectories the active tiers need. +func injectVolumeClaimTemplate(expected *appsv1.StatefulSet, server *kliov1alpha1.Server) { expected.Spec.VolumeClaimTemplates = append(expected.Spec.VolumeClaimTemplates, corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: "queue", + Name: pvcTypeKlio, Labels: map[string]string{ klioServerLabel: server.Name, - pvcTypeLabel: pvcTypeQueue, }, }, - Spec: server.Spec.Queue.PersistentVolumeClaimTemplate, + Spec: server.Spec.Storage.PersistentVolumeClaimTemplate, }) } @@ -352,50 +334,6 @@ func (r *ServerReconciler) serverPodSecurityContext() *corev1.PodSecurityContext } } -func injectTier1VolumeClaimTemplates( - ss *appsv1.StatefulSet, - server kliov1alpha1.Server, -) { - ss.Spec.VolumeClaimTemplates = append(ss.Spec.VolumeClaimTemplates, - corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "data", - Labels: map[string]string{ - klioServerLabel: server.Name, - pvcTypeLabel: pvcTypeData, - }, - }, - Spec: server.Spec.Tier1.Data.PersistentVolumeClaimTemplate, - }, - corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cachetier1", - Labels: map[string]string{ - klioServerLabel: server.Name, - pvcTypeLabel: pvcTypeCacheTier1, - }, - }, - Spec: server.Spec.Tier1.Cache.PersistentVolumeClaimTemplate, - }) -} - -func injectTier2VolumeClaimTemplates( - ss *appsv1.StatefulSet, - server kliov1alpha1.Server, -) { - ss.Spec.VolumeClaimTemplates = append(ss.Spec.VolumeClaimTemplates, - corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cachetier2", - Labels: map[string]string{ - klioServerLabel: server.Name, - pvcTypeLabel: pvcTypeCacheTier2, - }, - }, - Spec: server.Spec.Tier2.Cache.PersistentVolumeClaimTemplate, - }) -} - func (r *ServerReconciler) reconcileService(ctx context.Context, server *kliov1alpha1.Server) error { service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -415,14 +353,12 @@ func (r *ServerReconciler) reconcileService(ctx context.Context, server *kliov1a maps.Copy(service.Labels, map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, }) service.Spec.SessionAffinity = corev1.ServiceAffinityNone service.Spec.Selector = map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, } service.Spec.Ports = []corev1.ServicePort{ { @@ -588,20 +524,13 @@ func (r *ServerReconciler) buildVolumeMounts(server *kliov1alpha1.Server) []core Name: "tmp", MountPath: "/tmp", }, + { + Name: pvcTypeKlio, + MountPath: klioMountPath, + }, } if server.Spec.Tier1 != nil { - volumeMounts = append( - volumeMounts, - corev1.VolumeMount{ - Name: "data", - MountPath: kopiaDataMountPath, - }, - corev1.VolumeMount{ - Name: "cachetier1", - MountPath: kopiaCacheTier1MountPath, - }, - ) _, mount := buildFileSourceVolMount(tier1EncKeyFileVolName, server.Spec.Tier1.EncryptionKeyFile) volumeMounts = append(volumeMounts, mount) @@ -609,16 +538,6 @@ func (r *ServerReconciler) buildVolumeMounts(server *kliov1alpha1.Server) []core volumeMounts = append(volumeMounts, mount) } - if server.Spec.Queue != nil { - volumeMounts = append( - volumeMounts, - corev1.VolumeMount{ - Name: "queue", - MountPath: "/queue", - }, - ) - } - if server.Spec.Tier2 != nil { volumeMounts = append( volumeMounts, @@ -626,10 +545,6 @@ func (r *ServerReconciler) buildVolumeMounts(server *kliov1alpha1.Server) []core Name: "tier2", MountPath: "/tier2", }, - corev1.VolumeMount{ - Name: "cachetier2", - MountPath: kopiaCacheTier2MountPath, - }, ) _, mount := buildFileSourceVolMount(tier2EncKeyFileVolName, server.Spec.Tier2.EncryptionKeyFile) volumeMounts = append(volumeMounts, mount) diff --git a/operator/internal/controller/server_statefulset_test.go b/operator/internal/controller/server_statefulset_test.go index 7856d3058..58b446ee3 100644 --- a/operator/internal/controller/server_statefulset_test.go +++ b/operator/internal/controller/server_statefulset_test.go @@ -56,17 +56,83 @@ func newTestServerForStatefulSet() *kliov1alpha1.Server { TLSSecretName: "tls-secret", ClientCASecretName: "ca-secret", }, - Mode: kliov1alpha1.ModeStandard, + Mode: kliov1alpha1.ModeStandard, + Storage: kliov1alpha1.Storage{PersistentVolumeClaimTemplate: newPVCSpec("10Gi")}, Tier1: &kliov1alpha1.Tier1Configuration{ - Data: kliov1alpha1.Data{PersistentVolumeClaimTemplate: newPVCSpec("10Gi")}, - Cache: kliov1alpha1.Cache{PersistentVolumeClaimTemplate: newPVCSpec("5Gi")}, EncryptionKeyFile: newTestFileSource("enc-secret", "encryption-key.age"), IdentityFile: newTestFileSource("id-secret", "identity.txt"), }, - Queue: &kliov1alpha1.Queue{ - PersistentVolumeClaimTemplate: newPVCSpec("1Gi"), + }, + } +} + +func newTestTier2Configuration() *kliov1alpha1.Tier2Configuration { + return &kliov1alpha1.Tier2Configuration{ + S3: &kliov1alpha1.S3Configuration{BucketName: "test-bucket"}, + EncryptionKeyFile: newTestFileSource("tier2-enc-secret", "encryption-key.age"), + IdentityFile: newTestFileSource("tier2-id-secret", "identity.txt"), + } +} + +// TestReconcileStatefulSetUnifiedPVC asserts that every server, whatever its +// tier configuration, gets exactly one VolumeClaimTemplate mounted at /klio. +func TestReconcileStatefulSetUnifiedPVC(t *testing.T) { + testCases := []struct { + name string + mutate func(*kliov1alpha1.Server) + }{ + { + name: "tier1 only", + mutate: func(_ *kliov1alpha1.Server) {}, + }, + { + name: "both tiers", + mutate: func(server *kliov1alpha1.Server) { + server.Spec.Tier2 = newTestTier2Configuration() }, }, + { + name: "tier2 only, read-only", + mutate: func(server *kliov1alpha1.Server) { + server.Spec.Mode = kliov1alpha1.ModeReadOnly + server.Spec.Tier1 = nil + server.Spec.Tier2 = newTestTier2Configuration() + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + server := newTestServerForStatefulSet() + tc.mutate(server) + + scheme := newTestScheme() + require.NoError(t, appsv1.AddToScheme(scheme)) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(server).Build() + reconciler := &ServerReconciler{ + Client: fakeClient, + Scheme: scheme, + Recorder: &events.FakeRecorder{Events: make(chan string, 10)}, + } + + result, err := reconciler.reconcileStatefulSet(context.Background(), server) + require.NoError(t, err) + assert.True(t, result.IsZero()) + + var statefulSet appsv1.StatefulSet + require.NoError(t, fakeClient.Get(context.Background(), client.ObjectKey{ + Name: "test-server-klio", Namespace: "default", + }, &statefulSet)) + + require.Len(t, statefulSet.Spec.VolumeClaimTemplates, 1) + pvc := statefulSet.Spec.VolumeClaimTemplates[0] + assert.Equal(t, "klio", pvc.Name) + assert.Equal(t, "test-server", pvc.Labels[klioServerLabel]) + assert.Equal(t, newPVCSpec("10Gi"), pvc.Spec) + + mounts := statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts + assert.Contains(t, mounts, corev1.VolumeMount{Name: "klio", MountPath: "/klio"}) + }) } } @@ -145,14 +211,12 @@ func TestReconcileStatefulSetInvalidSpecExistingStatefulSet(t *testing.T) { Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ klioServerLabel: server.Name, - typeLabel: baseTypeLabelValue, }, }, }, diff --git a/operator/internal/controller/suite_test.go b/operator/internal/controller/suite_test.go index 4430f2cf5..21722e0b7 100644 --- a/operator/internal/controller/suite_test.go +++ b/operator/internal/controller/suite_test.go @@ -24,6 +24,7 @@ import ( "path/filepath" "testing" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -60,6 +61,9 @@ var _ = BeforeSuite(func() { err = kliov1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = apiextensionsv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + // +kubebuilder:scaffold:scheme By("bootstrapping test environment") diff --git a/operator/test/e2e/main_test.go b/operator/test/e2e/main_test.go index b8492cac7..b05680ea5 100644 --- a/operator/test/e2e/main_test.go +++ b/operator/test/e2e/main_test.go @@ -27,6 +27,7 @@ import ( certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/cloudnative-pg/cloudnative-pg/tests/utils/sternmultitailer" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/kubernetes" "sigs.k8s.io/e2e-framework/pkg/envconf" @@ -91,18 +92,26 @@ func TestMain(m *testing.M) { // Disable linter and SonarQube: cancel is stored in sternCancel and called in teardown var sternCtx context.Context sternCtx, sternCancel = context.WithCancel(ctx) //nolint:gosec - labelSelectors := []labels.Set{ - {"app.kubernetes.io/name": "klio"}, - {"app.kubernetes.io/name": "cloudnative-pg"}, - {"app.kubernetes.io/name": "postgresql"}, - {"app.kubernetes.io/instance": "rustfs"}, - {"batch.kubernetes.io/job-name": "rustfs"}, - {"klio.cnpg.io/type": "base"}, + // klio.cnpg.io/klio-server is set to the Server's name, so any Klio + // server pod (whatever the server is called) is matched by its mere + // presence rather than a fixed label value. + klioServerRequirement, err := labels.NewRequirement( + "klio.cnpg.io/klio-server", selection.Exists, nil) + if err != nil { + return ctx, err + } + + labelSelectors := []labels.Selector{ + labels.SelectorFromSet(labels.Set{"app.kubernetes.io/name": "klio"}), + labels.SelectorFromSet(labels.Set{"app.kubernetes.io/name": "cloudnative-pg"}), + labels.SelectorFromSet(labels.Set{"app.kubernetes.io/name": "postgresql"}), + labels.SelectorFromSet(labels.Set{"app.kubernetes.io/instance": "rustfs"}), + labels.SelectorFromSet(labels.Set{"batch.kubernetes.io/job-name": "rustfs"}), + labels.NewSelector().Add(*klioServerRequirement), } for _, ls := range labelSelectors { sternDoneChs = append(sternDoneChs, - sternmultitailer.StreamLogs(sternCtx, client, - labels.SelectorFromSet(ls), logDir)) + sternmultitailer.StreamLogs(sternCtx, client, ls, logDir)) } return ctx, nil diff --git a/operator/test/e2e/maintenance_test.go b/operator/test/e2e/maintenance_test.go index b5dccfd0d..50de875aa 100644 --- a/operator/test/e2e/maintenance_test.go +++ b/operator/test/e2e/maintenance_test.go @@ -42,7 +42,7 @@ const maintenanceClusterName = "test-cluster" // listTier1WALFiles returns the WAL segment file names stored in tier1 for a // cluster, sorted ascending. Partial files are excluded. // -// WAL files live at /data/wal/{clusterName}/{16-char-prefix}/{24-char-name} on +// WAL files live at /klio/data/wal/{clusterName}/{16-char-prefix}/{24-char-name} on // the server pod; 'find' is unavailable in the minimal container, so we rely on // shell globbing. func listTier1WALFiles( @@ -53,7 +53,7 @@ func listTier1WALFiles( var stdout, stderr bytes.Buffer listCmd := []string{ "sh", "-c", - fmt.Sprintf("ls /data/wal/%s/*/0000* 2>/dev/null | sort", clusterName), + fmt.Sprintf("ls /klio/data/wal/%s/*/0000* 2>/dev/null | sort", clusterName), } // ls exits non-zero when nothing matches, which is fine: we return an empty diff --git a/operator/test/e2e/pvc_resize_test.go b/operator/test/e2e/pvc_resize_test.go index e2e33bd5d..549bf29b8 100644 --- a/operator/test/e2e/pvc_resize_test.go +++ b/operator/test/e2e/pvc_resize_test.go @@ -176,7 +176,7 @@ func NewPVCResizeFeatureConfig(name string, namespace string) klioFeatures.PVCRe // Encryption secret ageSecrets := secrets.GetKlioAgeEncryptionSecrets("encryption", namespace, "testencryptionpassword123") - // Klio Server with tier1 and queue + // Klio Server with tier1 and the unified storage PVC klioServer := klio.GetServerObject( klioServerName, namespace, @@ -207,14 +207,12 @@ func NewPVCResizeFeatureConfig(name string, namespace string) klioFeatures.PVCRe } return klioFeatures.PVCResizeFeatureConfig{ - Name: name, - Setup: scenario.Setup, - Teardown: scenario.Teardown, - KlioServer: klioServer, - Namespace: namespace, - NewDataSize: resource.MustParse("2Gi"), - NewCacheSize: resource.MustParse("2Gi"), - NewQueueSize: resource.MustParse("200Mi"), + Name: name, + Setup: scenario.Setup, + Teardown: scenario.Teardown, + KlioServer: klioServer, + Namespace: namespace, + NewStorageSize: resource.MustParse("4Gi"), } } diff --git a/operator/test/e2e/server_reconfig_test.go b/operator/test/e2e/server_reconfig_test.go index 54d4cbd6a..59aa4b54c 100644 --- a/operator/test/e2e/server_reconfig_test.go +++ b/operator/test/e2e/server_reconfig_test.go @@ -21,7 +21,6 @@ package e2e import ( "context" - "slices" "testing" "time" @@ -31,7 +30,6 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/pkg/envconf" @@ -72,7 +70,7 @@ type serverReconfigScenario struct { encryptionSecret *corev1.Secret identitySecret *corev1.Secret - // Klio Server (initially tier1+queue only, no tier2) + // Klio Server (initially tier1 only, no tier2) klioServer *kliov1alpha1.Server // Tier2 configuration to add during Run @@ -156,13 +154,15 @@ func (f *serverReconfigFeature) Setup() types.StepFunc { // Run executes the server tier reconfiguration test. // -// This test verifies that adding tier2 to an existing tier1+queue server -// triggers the StatefulSet delete/recreate flow (due to immutable VCTs) -// and that: +// Every server, whatever tiers are enabled, gets the same single unified +// "klio" PVC (mounted at /klio). Adding tier2 to a tier1-only server +// therefore no longer touches the StatefulSet's VolumeClaimTemplates, so it +// should update the StatefulSet in place rather than delete/recreate it. +// This test verifies that: // 1. The server Pod comes back ready with the new configuration. -// 2. The StatefulSet has the expected VolumeClaimTemplates including cachetier2. -// 3. The new cachetier2 PVC is created. -// 4. The original PVCs (data, cachetier1, queue) are retained (same UIDs). +// 2. The StatefulSet still has exactly one VolumeClaimTemplate, named "klio". +// 3. The klio PVC's UID is unchanged (it was not deleted/recreated) and no +// new PVC was created for the server. func (f *serverReconfigFeature) Run() types.StepFunc { return func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { t.Helper() @@ -173,24 +173,32 @@ func (f *serverReconfigFeature) Run() types.StepFunc { server := f.scenario.klioServer stsName := server.Name + "-klio" + klioPVCName := "klio-" + stsName + "-0" - // Record UIDs of existing PVCs before reconfiguration - originalPVCNames := []string{ - "data-" + stsName + "-0", - "cachetier1-" + stsName + "-0", - "queue-" + stsName + "-0", - } - originalPVCUIDs := make(map[string]k8stypes.UID, len(originalPVCNames)) - - for _, pvcName := range originalPVCNames { - pvc := &corev1.PersistentVolumeClaim{} - require.NoError(t, - r.Get(ctx, pvcName, f.scenario.namespace.Name, pvc), - "failed to get original PVC %s", pvcName, - ) - originalPVCUIDs[pvcName] = pvc.UID - t.Logf("Recorded PVC %s with UID %s", pvcName, pvc.UID) - } + // Record the unified klio PVC's UID and the total PVC count for this + // server before reconfiguration. + originalPVC := &corev1.PersistentVolumeClaim{} + require.NoError(t, + r.Get(ctx, klioPVCName, f.scenario.namespace.Name, originalPVC), + "failed to get original klio PVC %s", klioPVCName, + ) + originalPVCUID := originalPVC.UID + t.Logf("Recorded klio PVC %s with UID %s", klioPVCName, originalPVCUID) + + originalPVCCount := countServerPVCs(ctx, t, r, f.scenario.namespace.Name, server.Name) + + // Record the StatefulSet's reconcile-hash annotation before the + // update. Adding tier2 no longer touches VolumeClaimTemplates, so + // unlike before, the pod isn't forced to restart by an immutable + // field rejection; without this gate, "wait for pod ready" could + // trivially pass against the still-running pre-update pod before + // the operator has reconciled anything. + stsBefore := &appsv1.StatefulSet{} + require.NoError(t, + r.Get(ctx, stsName, f.scenario.namespace.Name, stsBefore), + "failed to get StatefulSet before reconfiguration", + ) + hashBefore := stsBefore.Annotations["klio.cnpg.io/klio-server-hash"] // Fetch current Server and add tier2 configuration currentServer := &kliov1alpha1.Server{} @@ -199,12 +207,28 @@ func (f *serverReconfigFeature) Run() types.StepFunc { "failed to get current Server", ) - tier2Config := klio.BuildTier2Configuration( - f.scenario.s3Opts, f.scenario.tier2Encryption, f.scenario.storageClass) + tier2Config := klio.BuildTier2Configuration(f.scenario.s3Opts, f.scenario.tier2Encryption) currentServer.Spec.Tier2 = &tier2Config require.NoError(t, r.Update(ctx, currentServer), "failed to update Server with tier2") t.Log("Server updated with tier2 configuration") + // Wait for the operator to actually reconcile the tier2 change into + // the StatefulSet before checking readiness/PVC identity. + t.Log("Waiting for the StatefulSet to pick up the tier2 configuration...") + err = wait.For( + func(ctx context.Context) (bool, error) { + sts := &appsv1.StatefulSet{} + if getErr := r.Get(ctx, stsName, f.scenario.namespace.Name, sts); getErr != nil { + return false, nil //nolint:nilerr + } + + return sts.Annotations["klio.cnpg.io/klio-server-hash"] != hashBefore, nil + }, + wait.WithTimeout(2*time.Minute), + wait.WithInterval(5*time.Second), + ) + require.NoError(t, err, "StatefulSet was not reconciled with the tier2 configuration") + // Wait for server Pod to become ready again t.Log("Waiting for server Pod to be ready after reconfiguration...") err = wait.For( @@ -215,48 +239,37 @@ func (f *serverReconfigFeature) Run() types.StepFunc { require.NoError(t, err, "server Pod not ready after tier2 reconfiguration") t.Log("Server Pod is ready after reconfiguration") - // Verify StatefulSet has the expected VolumeClaimTemplates + // Verify the StatefulSet still has exactly one VolumeClaimTemplate: + // the unified "klio" one, unaffected by the tier1/tier2 change. sts := &appsv1.StatefulSet{} require.NoError(t, r.Get(ctx, stsName, f.scenario.namespace.Name, sts), "failed to get StatefulSet", ) - - vctNames := make([]string, len(sts.Spec.VolumeClaimTemplates)) - for i, vct := range sts.Spec.VolumeClaimTemplates { - vctNames[i] = vct.Name - } - t.Logf("StatefulSet VolumeClaimTemplates: %v", vctNames) - - for _, expected := range []string{"data", "cachetier1", "queue", "cachetier2"} { - require.True(t, - slices.Contains(vctNames, expected), - "StatefulSet missing VolumeClaimTemplate %q, got %v", expected, vctNames, - ) - } - - // Verify cachetier2 PVC exists - cachetier2PVC := &corev1.PersistentVolumeClaim{} - cachetier2PVCName := "cachetier2-" + stsName + "-0" + require.Len(t, sts.Spec.VolumeClaimTemplates, 1, + "StatefulSet should have exactly one VolumeClaimTemplate, got %d", + len(sts.Spec.VolumeClaimTemplates)) + require.Equal(t, "klio", sts.Spec.VolumeClaimTemplates[0].Name, + "StatefulSet's single VolumeClaimTemplate should be named %q", "klio") + + // Verify the klio PVC was retained (same UID), not deleted/recreated. + finalPVC := &corev1.PersistentVolumeClaim{} require.NoError(t, - r.Get(ctx, cachetier2PVCName, f.scenario.namespace.Name, cachetier2PVC), - "cachetier2 PVC %s not found", cachetier2PVCName, + r.Get(ctx, klioPVCName, f.scenario.namespace.Name, finalPVC), + "klio PVC %s no longer exists after reconfiguration", klioPVCName, + ) + require.Equal(t, originalPVCUID, finalPVC.UID, + "klio PVC %s was recreated (UID changed from %s to %s), data may have been lost", + klioPVCName, originalPVCUID, finalPVC.UID, + ) + t.Logf("klio PVC %s retained with original UID %s", klioPVCName, finalPVC.UID) + + // Verify no new PVC appeared for this server. + finalPVCCount := countServerPVCs(ctx, t, r, f.scenario.namespace.Name, server.Name) + require.Equal(t, originalPVCCount, finalPVCCount, + "number of PVCs for server %s changed after reconfiguration (%d -> %d)", + server.Name, originalPVCCount, finalPVCCount, ) - t.Logf("cachetier2 PVC %s exists with UID %s", cachetier2PVCName, cachetier2PVC.UID) - - // Verify original PVCs are retained (same UIDs) - for _, pvcName := range originalPVCNames { - pvc := &corev1.PersistentVolumeClaim{} - require.NoError(t, - r.Get(ctx, pvcName, f.scenario.namespace.Name, pvc), - "original PVC %s no longer exists after reconfiguration", pvcName, - ) - require.Equal(t, originalPVCUIDs[pvcName], pvc.UID, - "PVC %s was recreated (UID changed from %s to %s), data may have been lost", - pvcName, originalPVCUIDs[pvcName], pvc.UID, - ) - t.Logf("PVC %s retained with original UID %s", pvcName, pvc.UID) - } t.Log("Server tier reconfiguration test passed: all verifications succeeded") @@ -264,12 +277,39 @@ func (f *serverReconfigFeature) Run() types.StepFunc { } } +// countServerPVCs returns the number of PersistentVolumeClaims labelled as +// belonging to the given Klio server, in the given namespace. +func countServerPVCs( + ctx context.Context, + t *testing.T, + r *resources.Resources, + namespace string, + serverName string, +) int { + t.Helper() + + var pvcList corev1.PersistentVolumeClaimList + require.NoError(t, + r.List(ctx, &pvcList, resources.WithLabelSelector("klio.cnpg.io/klio-server="+serverName)), + "failed to list PVCs for server %s", serverName, + ) + + count := 0 + for i := range pvcList.Items { + if pvcList.Items[i].Namespace == namespace { + count++ + } + } + + return count +} + // Teardown cleans up resources after the test. func (f *serverReconfigFeature) Teardown() types.StepFunc { return f.scenario.Teardown } -// ServerTierReconfiguration returns a Feature that tests adding tier2 to an existing tier1+queue server. +// ServerTierReconfiguration returns a Feature that tests adding tier2 to an existing tier1-only server. func ServerTierReconfiguration(namespace string) *serverReconfigFeature { const ( klioServerName = "klio" @@ -321,7 +361,7 @@ func ServerTierReconfiguration(namespace string) *serverReconfigFeature { S3CABundleSecretName: rustfsCertificate.Spec.SecretName, } - // Create tier1+queue only server (no tier2) + // Create tier1-only server (no tier2) klioServer := klio.GetServerObject( klioServerName, namespace, diff --git a/operator/test/e2e/tier2_recovery_common_test.go b/operator/test/e2e/tier2_recovery_common_test.go index f1669f142..a549e78a6 100644 --- a/operator/test/e2e/tier2_recovery_common_test.go +++ b/operator/test/e2e/tier2_recovery_common_test.go @@ -237,6 +237,12 @@ func deployTier2RecoveryServer( return fmt.Errorf("recovery server not ready: %w", err) } + // The recovery server is read-only (tier2-only, no tier1): confirm it + // too gets the unified "klio" PVC/mount, same as a tier1 server. + if err := checkRecoveryServerHasKlioPVCAndMount(ctx, r, namespace, resources.RecoveryServer); err != nil { + return err + } + // Create PluginConfiguration for recovery (points to second server) if err := r.Create(ctx, resources.PluginConfigurationRecovery); err != nil { return fmt.Errorf("failed to create recovery plugin configuration: %w", err) @@ -245,6 +251,53 @@ func deployTier2RecoveryServer( return nil } +// checkRecoveryServerHasKlioPVCAndMount verifies that a read-only +// (tier2-only) Server also gets the unified "klio" PVC, and that it is +// actually mounted at /klio in the server's StatefulSet, exactly like a +// tier1 server does. +func checkRecoveryServerHasKlioPVCAndMount( + ctx context.Context, + r *resources.Resources, + namespace string, + server *kliov1alpha1.Server, +) error { + stsName := server.Name + "-klio" + pvcName := "klio-" + stsName + "-0" + + pvc := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, pvcName, namespace, pvc); err != nil { + return fmt.Errorf("read-only server's klio PVC %s not found: %w", pvcName, err) + } + + if serverLabel := pvc.Labels["klio.cnpg.io/klio-server"]; serverLabel != server.Name { + return fmt.Errorf("read-only server's PVC %s has klio-server label %q, want %q", + pvcName, serverLabel, server.Name) + } + + sts := &appsv1.StatefulSet{} + if err := r.Get(ctx, stsName, namespace, sts); err != nil { + return fmt.Errorf("read-only server's StatefulSet %s not found: %w", stsName, err) + } + + if len(sts.Spec.Template.Spec.Containers) == 0 { + return fmt.Errorf("read-only server's StatefulSet %s has no containers", stsName) + } + + mounted := false + for _, m := range sts.Spec.Template.Spec.Containers[0].VolumeMounts { + if m.Name == "klio" && m.MountPath == "/klio" { + mounted = true + + break + } + } + if !mounted { + return fmt.Errorf("read-only server's StatefulSet %s has no klio volume mounted at /klio", stsName) + } + + return nil +} + // tier2ScenarioResources contains all resources created by buildTier2ScenarioResources. type tier2ScenarioResources struct { // Common resources diff --git a/operator/test/e2e/wal_retention_test.go b/operator/test/e2e/wal_retention_test.go index aa6162444..79c33cf9b 100644 --- a/operator/test/e2e/wal_retention_test.go +++ b/operator/test/e2e/wal_retention_test.go @@ -182,12 +182,12 @@ func (s *walRetentionScenario) getWALFilesInTier1( containerName := "server" // List WAL files in the tier1 WAL directory. - // WAL files are stored at /data/wal/{clusterName}/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX. + // WAL files are stored at /klio/data/wal/{clusterName}/XXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX. // Note: 'find' is not available in the minimal container, so we use ls with shell globbing. var stdout, stderr bytes.Buffer listCmd := []string{ "sh", "-c", - fmt.Sprintf("ls /data/wal/%s/*/0000* 2>/dev/null | sort", s.cnpgCluster.Name), + fmt.Sprintf("ls /klio/data/wal/%s/*/0000* 2>/dev/null | sort", s.cnpgCluster.Name), } // ls returns exit code 1 if no files match, which is fine - we just return empty list. diff --git a/operator/test/klio/features/pvc_resize.go b/operator/test/klio/features/pvc_resize.go index 320b4a938..ccd2c1952 100644 --- a/operator/test/klio/features/pvc_resize.go +++ b/operator/test/klio/features/pvc_resize.go @@ -36,25 +36,19 @@ import ( kliov1alpha1 "github.com/cloudnative-pg/klio/operator/api/v1alpha1" ) -const ( - // klioServerLabelKey is the label key used to identify PVCs belonging to a Klio server. - klioServerLabelKey = "klio.cnpg.io/klio-server" - // pvcTypeLabelKey is the label key used to identify the type of PVC. - pvcTypeLabelKey = "klio.cnpg.io/pvcType" -) +// klioServerLabelKey is the label key used to identify PVCs belonging to a Klio server. +const klioServerLabelKey = "klio.cnpg.io/klio-server" // PVCResizeFeature defines a feature for testing PVC resize functionality. type PVCResizeFeature struct { - name string - setup types.StepFunc - teardown types.StepFunc - klioServer *kliov1alpha1.Server - namespace string - newDataSize resource.Quantity - newCacheSize resource.Quantity - newQueueSize resource.Quantity - timeout time.Duration - interval time.Duration + name string + setup types.StepFunc + teardown types.StepFunc + klioServer *kliov1alpha1.Server + namespace string + newStorageSize resource.Quantity + timeout time.Duration + interval time.Duration } // PVCResizeFeatureConfig holds the configuration for creating a PVC resize feature test. @@ -69,12 +63,8 @@ type PVCResizeFeatureConfig struct { KlioServer *kliov1alpha1.Server // Namespace is the namespace where resources are created. Namespace string - // NewDataSize is the new size for the data PVC. - NewDataSize resource.Quantity - // NewCacheSize is the new size for the cache PVC. - NewCacheSize resource.Quantity - // NewQueueSize is the new size for the queue PVC. - NewQueueSize resource.Quantity + // NewStorageSize is the new size for the unified storage PVC. + NewStorageSize resource.Quantity // Timeout for waiting for PVC resize (defaults to 5 minutes). Timeout time.Duration // Interval for checking PVC resize status (defaults to 5 seconds). @@ -91,16 +81,14 @@ func NewPVCResizeFeature(config PVCResizeFeatureConfig) *PVCResizeFeature { } return &PVCResizeFeature{ - name: config.Name, - setup: config.Setup, - teardown: config.Teardown, - klioServer: config.KlioServer, - namespace: config.Namespace, - newDataSize: config.NewDataSize, - newCacheSize: config.NewCacheSize, - newQueueSize: config.NewQueueSize, - timeout: config.Timeout, - interval: config.Interval, + name: config.Name, + setup: config.Setup, + teardown: config.Teardown, + klioServer: config.KlioServer, + namespace: config.Namespace, + newStorageSize: config.NewStorageSize, + timeout: config.Timeout, + interval: config.Interval, } } @@ -173,22 +161,10 @@ func (f *PVCResizeFeature) updateServerPVCSizes( expectedSizes := make(map[string]resource.Quantity) - if server.Spec.Tier1 != nil && !f.newDataSize.IsZero() { - server.Spec.Tier1.Data.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage] = f.newDataSize - expectedSizes["data"] = f.newDataSize - t.Logf("Updating data PVC size to %s", f.newDataSize.String()) - } - - if server.Spec.Tier1 != nil && !f.newCacheSize.IsZero() { - server.Spec.Tier1.Cache.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage] = f.newCacheSize - expectedSizes["cachetier1"] = f.newCacheSize - t.Logf("Updating cachetier1 PVC size to %s", f.newCacheSize.String()) - } - - if server.Spec.Queue != nil && !f.newQueueSize.IsZero() { - server.Spec.Queue.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage] = f.newQueueSize - expectedSizes["queue"] = f.newQueueSize - t.Logf("Updating queue PVC size to %s", f.newQueueSize.String()) + if !f.newStorageSize.IsZero() { + server.Spec.Storage.PersistentVolumeClaimTemplate.Resources.Requests[corev1.ResourceStorage] = f.newStorageSize + expectedSizes["klio-"+f.klioServer.Name+klioPodSuffix] = f.newStorageSize + t.Logf("Updating klio PVC size to %s", f.newStorageSize.String()) } err = r.Update(ctx, &server) @@ -206,18 +182,18 @@ func verifyPVCSizes( ) { t.Helper() - for pvcType, expectedSize := range expectedSizes { - actualSize, exists := finalSizes[pvcType] - require.True(t, exists, "PVC type %s not found", pvcType) + for pvcName, expectedSize := range expectedSizes { + actualSize, exists := finalSizes[pvcName] + require.True(t, exists, "PVC %s not found", pvcName) require.GreaterOrEqual(t, actualSize.Cmp(expectedSize), 0, "PVC %s size %s is less than expected %s", - pvcType, (&actualSize).String(), (&expectedSize).String()) - initialSize := initialSizes[pvcType] - t.Logf("PVC %s resized successfully: %s -> %s", pvcType, (&initialSize).String(), (&actualSize).String()) + pvcName, (&actualSize).String(), (&expectedSize).String()) + initialSize := initialSizes[pvcName] + t.Logf("PVC %s resized successfully: %s -> %s", pvcName, (&initialSize).String(), (&actualSize).String()) } } -// getPVCSizes returns a map of PVC type labels to their current sizes. +// getPVCSizes returns a map of PVC names to their current sizes. func getPVCSizes( ctx context.Context, r *resources.Resources, @@ -239,13 +215,8 @@ func getPVCSizes( continue } - pvcType, exists := pvc.Labels[pvcTypeLabelKey] - if !exists { - continue - } - if size, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; ok { - sizes[pvcType] = size + sizes[pvc.Name] = size } } @@ -267,8 +238,8 @@ func checkPVCsResized( return false, nil //nolint:nilerr } - for pvcType, expectedSize := range expectedSizes { - actualSize, exists := sizes[pvcType] + for pvcName, expectedSize := range expectedSizes { + actualSize, exists := sizes[pvcName] if !exists { return false, nil } diff --git a/operator/test/utils/templates/klio/klio.go b/operator/test/utils/templates/klio/klio.go index 3612e8647..01dab1293 100644 --- a/operator/test/utils/templates/klio/klio.go +++ b/operator/test/utils/templates/klio/klio.go @@ -73,30 +73,13 @@ func newFileSource(secretName, fileName string) kliov1alpha1.FileSource { } } -// BuildTier2Configuration creates a Tier2Configuration from S3 options, encryption options, -// and a storage class name. If storageClass is empty, the cluster's default storage class is used. +// BuildTier2Configuration creates a Tier2Configuration from S3 options and +// encryption options. func BuildTier2Configuration( s3Opts Tier2S3Options, encOpts EncryptionOptions, - storageClass string, ) kliov1alpha1.Tier2Configuration { - var sc *string - if storageClass != "" { - sc = new(storageClass) - } - return kliov1alpha1.Tier2Configuration{ - Cache: kliov1alpha1.Cache{ - PersistentVolumeClaimTemplate: corev1.PersistentVolumeClaimSpec{ - StorageClassName: sc, - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, S3: &kliov1alpha1.S3Configuration{ BucketName: s3Opts.S3BucketName, Prefix: s3Opts.S3Prefix, @@ -132,9 +115,9 @@ type ServerTemplateOptions struct { // test image is used. Image string - // StorageClass is the Kubernetes storage class used for all PVC templates - // (tier1 cache, tier1 data, queue). If empty, the cluster's default - // storage class is used. + // StorageClass is the Kubernetes storage class used for the unified + // storage PVC template. If empty, the cluster's default storage class + // is used. StorageClass string // TLSSecretName is the secret to be used to expose the Klio server. @@ -155,6 +138,11 @@ func newBaseServer(name, namespace string, opts ServerTemplateOptions) *kliov1al ImagePullPolicy: corev1.PullAlways, } + var sc *string + if opts.StorageClass != "" { + sc = new(opts.StorageClass) + } + return &kliov1alpha1.Server{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -166,63 +154,34 @@ func newBaseServer(name, namespace string, opts ServerTemplateOptions) *kliov1al TLSSecretName: opts.TLSSecretName, ClientCASecretName: opts.ClientCASecretName, }, + Storage: kliov1alpha1.Storage{ + PersistentVolumeClaimTemplate: corev1.PersistentVolumeClaimSpec{ + StorageClassName: sc, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, }, } } -// GetServerObject returns a Klio server Object with tier1 and queue configuration. +// GetServerObject returns a Klio server Object with tier1 and unified storage configuration. func GetServerObject( name, namespace string, opts ServerTemplateOptions, ) *kliov1alpha1.Server { - var sc *string - if opts.StorageClass != "" { - sc = new(opts.StorageClass) - } - server := newBaseServer(name, namespace, opts) server.Spec.Mode = kliov1alpha1.ModeStandard server.Spec.Tier1 = &kliov1alpha1.Tier1Configuration{ - Cache: kliov1alpha1.Cache{ - PersistentVolumeClaimTemplate: corev1.PersistentVolumeClaimSpec{ - StorageClassName: sc, - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - Data: kliov1alpha1.Data{ - PersistentVolumeClaimTemplate: corev1.PersistentVolumeClaimSpec{ - StorageClassName: sc, - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, EncryptionKeyFile: newFileSource(opts.Encryption.EncryptionKeySecretName, opts.Encryption.EncryptionKeyFileName), IdentityFile: newFileSource(opts.Encryption.IdentitySecretName, opts.Encryption.IdentityFileName), } - // Queue is mandatory when tier1 is configured - server.Spec.Queue = &kliov1alpha1.Queue{ - PersistentVolumeClaimTemplate: corev1.PersistentVolumeClaimSpec{ - StorageClassName: sc, - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("100Mi"), - }, - }, - }, - } - return server } @@ -290,24 +249,24 @@ type ServerWithTier2TemplateOptions struct { S3 Tier2S3Options } -// GetServerWithTier2Object returns a Klio server Object with tier1, tier2, and queue configuration. +// GetServerWithTier2Object returns a Klio server Object with tier1 and tier2 configuration. func GetServerWithTier2Object( name, namespace string, opts ServerWithTier2TemplateOptions, ) *kliov1alpha1.Server { - // GetServerObject already includes tier1 and queue configuration + // GetServerObject already includes tier1 and the unified storage PVC server := GetServerObject(name, namespace, opts.ServerTemplateOptions) // Add tier2 configuration - tier2Config := BuildTier2Configuration(opts.S3, opts.Tier2Encryption, opts.StorageClass) + tier2Config := BuildTier2Configuration(opts.S3, opts.Tier2Encryption) server.Spec.Tier2 = &tier2Config return server } // GetReadOnlyTier2ServerObject returns a read-only Klio server Object with only tier2 configuration. -// This server does not have tier1 or queue, only tier2 for recovery purposes. +// This server does not have tier1, only tier2 for recovery purposes. func GetReadOnlyTier2ServerObject( name, namespace string, @@ -315,7 +274,7 @@ func GetReadOnlyTier2ServerObject( ) *kliov1alpha1.Server { server := newBaseServer(name, namespace, opts.ServerTemplateOptions) server.Spec.Mode = kliov1alpha1.ModeReadOnly - tier2Config := BuildTier2Configuration(opts.S3, opts.Tier2Encryption, opts.StorageClass) + tier2Config := BuildTier2Configuration(opts.S3, opts.Tier2Encryption) server.Spec.Tier2 = &tier2Config return server