From c532e4c54201b3a37f5a4e065051205a9b8f97a8 Mon Sep 17 00:00:00 2001 From: Fedor Tarasenko Date: Mon, 14 Sep 2026 10:58:27 +0000 Subject: [PATCH 1/3] feat(s3): add bucket create/delete, object delete and du MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the write side of "koc s3": the group could list, show, download and upload, but nothing could create a bucket or remove anything, so a bucket's whole lifecycle still needed s5cmd or s3cmd next to koc. koc s3 bucket create # s5cmd mb koc s3 bucket delete [...] # s5cmd rb koc s3 object delete / [...] # s5cmd rm koc s3 object delete / --recursive koc s3 du [/] [--group] # s5cmd du Notes on the shapes chosen: - "bucket create" states the bucket's location as the signing region, which is the only value a store can accept — a request signed for one region is not served by another — so there is no separate location flag. us-east-1 is special-cased: AWS rejects a body that names it. Re-creating a bucket the credentials own fails rather than succeeding quietly, so a script can tell "I made this" from "it was there". - "bucket delete" never removes objects implicitly; BucketNotEmpty is turned into the recursive-delete command that fixes it. - "object delete --recursive" is the one destructive shape in the group, so it is spelled out rather than inferred from a trailing slash or a wildcard, and --dry-run prints exactly the keys it would remove. It is also how a bucket is emptied before "bucket delete". Deleting is idempotent because S3 answers a delete of an absent key with success; that is documented rather than papered over. - "du" is a listing folded into a sum, because S3 has no size call. It streams via the new ListObjectsFunc, so emptying or totalling a bucket costs one page of memory whatever its size; ListObjects is now a thin wrapper over it and its behaviour is unchanged. - Deletes follow koc's batch contract (internal/cli/batchdelete): every ref is attempted, failures are joined. parseBucketRef rejects a ref carrying a key, so "bucket create a/b" is an error instead of silently making a bucket named "a". Exercised end to end against a mock S3 endpoint (create, re-create, upload, list, du, du --group, recursive dry-run, recursive delete, not-empty refusal, delete, and the s3:// ref spelling), plus unit tests at both the client and the runXxx seam. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TQ2Xt7Epc3n8XAXiAgPXiC --- README.md | 20 ++- docs/coverage.md | 16 +-- internal/cli/s3/bucket.go | 108 ++++++++++++++ internal/cli/s3/bucket_test.go | 135 ++++++++++++++++++ internal/cli/s3/client.go | 15 ++ internal/cli/s3/du.go | 132 +++++++++++++++++ internal/cli/s3/du_test.go | 73 ++++++++++ internal/cli/s3/object.go | 1 + internal/cli/s3/object_delete.go | 127 +++++++++++++++++ internal/cli/s3/object_delete_test.go | 116 +++++++++++++++ internal/cli/s3/s3.go | 16 ++- internal/s3/lifecycle_test.go | 196 ++++++++++++++++++++++++++ internal/s3/s3.go | 103 +++++++++++++- 13 files changed, 1040 insertions(+), 18 deletions(-) create mode 100644 internal/cli/s3/bucket_test.go create mode 100644 internal/cli/s3/du.go create mode 100644 internal/cli/s3/du_test.go create mode 100644 internal/cli/s3/object_delete.go create mode 100644 internal/cli/s3/object_delete_test.go create mode 100644 internal/s3/lifecycle_test.go diff --git a/README.md b/README.md index fe76103..294bf52 100644 --- a/README.md +++ b/README.md @@ -521,17 +521,35 @@ S3 credentials alone are used, so it works on a host with no cloud credentials. ```sh koc s3 bucket list +koc s3 bucket create scratch +koc s3 bucket delete scratch # the bucket must be empty koc s3 object list db-backups koc s3 object show db-backups/ # HEAD only, no transfer +koc s3 object delete db-backups/ +koc s3 object delete db-backups/e2e- -r # every key under a prefix +koc s3 du db-backups # objects and exact bytes +koc s3 du db-backups --group # per storage class koc s3 download db-backups/ ./dump.mbs.gz.enc koc s3 download db-backups/.sha256 - # "-" streams to stdout, so it pipes koc s3 upload ./dump.mbs.gz.enc db-backups/ ``` +Every ref also accepts the `s3:///` spelling, so a path copied from +`s5cmd` or `aws s3` pastes in unchanged. + `bucket list` is scoped to the **access key**, not to the store: Garage answers with the buckets that key is granted, so a key made for one bucket lists exactly that one. +`object delete --recursive` is the one destructive shape in the group, so it is +never inferred from a trailing slash or a wildcard — and `--dry-run` prints +exactly the keys it would remove without touching any of them. It is also how a +bucket is emptied before `bucket delete`, which never removes objects +implicitly. + +`du` is a listing folded into a sum, because S3 has no "how big is this" call: +no object is downloaded, but every key is walked, one request per 1000 of them. + Credentials come from flags, from the environment, or from a Kubernetes Secret: | flag | env | default | @@ -590,7 +608,7 @@ internal/kube/ minimal read-only k8s REST client (no client-go) internal/vault/ minimal Vault REST client (AppRole/token + KV v2) internal/s3/ minimal S3 REST client (SigV4, no aws-sdk/minio-go) internal/cli/vault/ "koc vault kv" list/get/copy/export/decrypt, no Keystone auth -internal/cli/s3/ "koc s3" bucket/object/download/upload, no Keystone auth +internal/cli/s3/ "koc s3" bucket/object/du/download/upload, no Keystone auth internal/output/ -f/-c formatter (table/json/yaml/value/csv) internal/cli/ root command wiring internal/cli/resolve/ cross-service name→ID resolution diff --git a/docs/coverage.md b/docs/coverage.md index 7961a6d..0fb10c2 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -3,7 +3,7 @@ How much of the upstream OpenStack CLI surface `koc` implements, measured against primary sources rather than documentation. -**Snapshot:** 2026-09-03 · `koc` @ this commit (base `3ba95db`) · 555 leaf +**Snapshot:** 2026-09-14 · `koc` @ this commit (base `96dabfc`) · 559 leaf commands (visible tree; 2 more are hidden duplicates). **Keep this file current** — see "Updating this document" below. Any commit that @@ -28,8 +28,8 @@ PyPI is the source of record. ## Headline -**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 555 leaf commands, -515 are upstream-equivalent and 40 are koc-native. +**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 559 leaf commands, +515 are upstream-equivalent and 44 are koc-native. The denominator grew by 13 against the 2026-08-07 snapshot without a single command changing: `python-ironic-inspector-client` is now a **baseline** rather @@ -433,14 +433,14 @@ limitations"; the fix is to make the other resolvers match `server`'s behaviour. ## koc-native commands -No upstream equivalent, by design — **40 leaves**, itemised so the total -reconciles with the headline (555 = 515 + 40): +No upstream equivalent, by design — **44 leaves**, itemised so the total +reconciles with the headline (559 = 515 + 44): | Count | Commands | Why it has no upstream equivalent | | --- | --- | --- | | 18 | `koc keyvrm …` — `app-config` ×2, `availability-zone` ×2, `event` ×4, `host-aggregate-config` ×5, `recommendation` ×5 | in-house KeyVRM catalog service; no gophercloud package and no OSC plugin | | 5 | `koc vault kv list/get/copy/export/decrypt` | Vault is not an OpenStack service; `copy` fills a gap in the Vault CLI itself | -| 5 | `koc s3 bucket list`, `koc s3 object list/show`, `koc s3 download/upload` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | +| 9 | `koc s3 bucket list/create/delete`, `koc s3 object list/show/delete`, `koc s3 du`, `koc s3 download/upload` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | | 2 | `koc dns pool list/show` | designate's API and its Python SDK both expose `/v2/pools`, but `python-designateclient` registers no `openstack` command for it. Reads only — pool *writes* are a `designate-manage`/config operation on the servers | | 2 | `koc server add/remove server-group` | KeyStack dynamic server groups | | 2 | `koc network trunk subport add`/`remove` | upstream folds these into `network trunk set`/`unset --subport` flags rather than giving them verbs (`network subport list` does exist and is counted — see "Naming deviations") | @@ -481,7 +481,7 @@ The tables are derived, not hand-maintained. To re-derive after a version bump or a batch of new commands: ```sh -# 1. koc's own command tree (553 leaf commands at the snapshot above) +# 1. koc's own command tree (559 leaf commands at the snapshot above) make build # Walk `--help` recursively. Count a command when it is *runnable*, not merely when # it is childless: `koc image import ` is a verb that also parents `koc image @@ -511,7 +511,7 @@ Then **check the arithmetic**, because that is the only thing that makes these tables worth reading. Three identities must hold at every snapshot: 1. every raw row numerator summed = the headline numerator (515); -2. leaf commands = headline numerator + koc-native (555 = 515 + 40); +2. leaf commands = headline numerator + koc-native (559 = 515 + 44); 3. every raw row denominator summed = 901, and minus the two not-targeted rows (swift 17 + manila 40) = the in-scope denominator (844). diff --git a/internal/cli/s3/bucket.go b/internal/cli/s3/bucket.go index ea84bde..4316f78 100644 --- a/internal/cli/s3/bucket.go +++ b/internal/cli/s3/bucket.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/cli/batchdelete" "github.com/ftarasenko/go-openstackclient/internal/output" "github.com/ftarasenko/go-openstackclient/internal/s3" ) @@ -20,6 +21,8 @@ func newBucketCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.C Short: "Manage buckets", } cmd.AddCommand(newBucketListCommand(a, o, f)) + cmd.AddCommand(newBucketCreateCommand(a, o, f)) + cmd.AddCommand(newBucketDeleteCommand(a, o, f)) return cmd } @@ -63,3 +66,108 @@ func runBucketList(ctx context.Context, client *s3.Client, o *output.Options, w } return o.WriteList(w, output.Table{Columns: []string{"Name", "Created"}, Rows: rows}) } + +const bucketCreateLong = `Create a bucket. + +The request states the bucket's location as the signing region (--s3-region, +default "garage"), which is the only value a store will accept: a request signed +for one region is not served by another. So there is no separate location flag — +set --s3-region and the bucket lands there. + +Creating a bucket the credentials already own fails rather than succeeding +quietly, so a script can tell "I created this" from "it was already there". On +Garage the key that creates a bucket owns it and needs no further grant.` + +const bucketCreateExample = ` koc s3 bucket create scratch + koc s3 bucket create s3://scratch` + +func newBucketCreateCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return &cobra.Command{ + Use: "create ", + Short: "Create a bucket", + Long: bucketCreateLong, + Example: bucketCreateExample, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, err := parseBucketRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketCreate(ctx, client, o, bucket, cmd.OutOrStdout()) + }, + } +} + +// runBucketCreate is the test seam for "bucket create". +func runBucketCreate(ctx context.Context, client *s3.Client, o *output.Options, + bucket string, w io.Writer) error { + if err := client.CreateBucket(ctx, bucket); err != nil { + switch s3.ErrorCode(err) { + case "BucketAlreadyOwnedByYou": + return fmt.Errorf("bucket %q already exists and is owned by these credentials", bucket) + case "BucketAlreadyExists": + return fmt.Errorf("bucket %q already exists and is owned by someone else", bucket) + } + return fmt.Errorf("creating bucket %q on %s: %w", bucket, client.Endpoint(), err) + } + return o.WriteSingle(w, []string{"Bucket", "Region"}, []any{bucket, client.Region()}) +} + +const bucketDeleteLong = `Delete one or more empty buckets. + +A bucket that still holds objects is refused by the server — nothing is ever +removed implicitly. Empty it first: + + koc s3 object delete / --recursive + +Every bucket named is attempted even if an earlier one fails, and the command +exits non-zero naming each failure.` + +func newBucketDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return &cobra.Command{ + Use: "delete [ ...]", + Short: "Delete empty buckets", + Long: bucketDeleteLong, + Aliases: []string{"remove"}, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketDelete(ctx, client, args, cmd.OutOrStdout()) + }, + } +} + +// runBucketDelete is the test seam for "bucket delete". It follows koc's batch +// contract: attempt every ref, join the failures. +func runBucketDelete(ctx context.Context, client *s3.Client, refs []string, w io.Writer) error { + return batchdelete.Each(refs, func(ref string) error { + bucket, err := parseBucketRef(ref) + if err != nil { + return err + } + if err := client.DeleteBucket(ctx, bucket); err != nil { + if s3.ErrorCode(err) == "BucketNotEmpty" { + return fmt.Errorf("bucket %q is not empty: delete its objects first "+ + "(koc s3 object delete %s/ --recursive)", bucket, bucket) + } + return fmt.Errorf("deleting bucket %q: %w", bucket, err) + } + _, err = fmt.Fprintf(w, "Deleted bucket: %s\n", bucket) + return err + }) +} diff --git a/internal/cli/s3/bucket_test.go b/internal/cli/s3/bucket_test.go new file mode 100644 index 0000000..d5d963b --- /dev/null +++ b/internal/cli/s3/bucket_test.go @@ -0,0 +1,135 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" +) + +func TestRunBucketCreate(t *testing.T) { + var gotMethod, gotPath string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketCreate(context.Background(), client, valueOpts(), "scratch", &buf); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut || gotPath != "/scratch" { + t.Errorf("request = %s %s, want PUT /scratch", gotMethod, gotPath) + } + // WriteSingle in value format is one field per line. + if got, want := buf.String(), "scratch\ngarage\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// An existing bucket must produce advice, not a raw 409: the two S3 codes mean +// different things to the operator ("yours" vs "someone else's"). +func TestRunBucketCreateAlreadyExists(t *testing.T) { + for _, tc := range []struct{ code, want string }{ + {"BucketAlreadyOwnedByYou", "owned by these credentials"}, + {"BucketAlreadyExists", "owned by someone else"}, + } { + t.Run(tc.code, func(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprintf(w, `%sx`, tc.code) + }) + + var buf bytes.Buffer + err := runBucketCreate(context.Background(), client, valueOpts(), "scratch", &buf) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + if buf.Len() != 0 { + t.Errorf("wrote %q on failure, want nothing", buf.String()) + } + }) + } +} + +func TestRunBucketDelete(t *testing.T) { + var paths []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runBucketDelete(context.Background(), client, []string{"scratch", "s3://spare"}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(paths, ","), "DELETE /scratch,DELETE /spare"; got != want { + t.Errorf("requests = %q, want %q", got, want) + } + want := "Deleted bucket: scratch\nDeleted bucket: spare\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A non-empty bucket is the expected failure, and the message must name the +// command that fixes it rather than leaving the operator with "409 Conflict". +func TestRunBucketDeleteNotEmptyExplains(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprint(w, `BucketNotEmptyx`) + }) + + var buf bytes.Buffer + err := runBucketDelete(context.Background(), client, []string{"db-backups"}, &buf) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "koc s3 object delete db-backups/ --recursive") { + t.Errorf("error = %q, want it to suggest the recursive delete", err) + } +} + +// The batch contract: a failing ref must not skip the ones after it. +func TestRunBucketDeleteAttemptsEveryRef(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/locked" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runBucketDelete(context.Background(), client, []string{"locked", "scratch"}, &buf) + if err == nil { + t.Fatal("expected the failing ref to be reported") + } + if got, want := buf.String(), "Deleted bucket: scratch\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A bucket ref carrying a key is a typo, and on a create it would silently make +// a bucket the operator did not name. +func TestParseBucketRefRejectsKey(t *testing.T) { + if _, err := parseBucketRef("scratch/keys"); err == nil { + t.Error("expected scratch/keys to be rejected as a bucket ref") + } + for _, ref := range []string{"scratch", "s3://scratch", "scratch/"} { + got, err := parseBucketRef(ref) + if err != nil { + t.Errorf("parseBucketRef(%q) = %v", ref, err) + continue + } + if got != "scratch" { + t.Errorf("parseBucketRef(%q) = %q, want scratch", ref, got) + } + } +} diff --git a/internal/cli/s3/client.go b/internal/cli/s3/client.go index e423f27..350c3aa 100644 --- a/internal/cli/s3/client.go +++ b/internal/cli/s3/client.go @@ -289,6 +289,21 @@ func parseRef(ref string) (bucket, key string, err error) { return bucket, key, nil } +// parseBucketRef is parseRef where a key is not accepted at all. "scratch/x" as +// a bucket name is almost always a mistyped object ref, and on a create that +// would make a bucket literally named "scratch" while the operator believed +// they had addressed a path. +func parseBucketRef(ref string) (string, error) { + bucket, key, err := parseRef(ref) + if err != nil { + return "", err + } + if strings.Trim(key, "/") != "" { + return "", fmt.Errorf("%q names an object, not a bucket: expected or s3://", ref) + } + return bucket, nil +} + // parseObjectRef is parseRef where the key is mandatory. func parseObjectRef(ref string) (bucket, key string, err error) { bucket, key, err = parseRef(ref) diff --git a/internal/cli/s3/du.go b/internal/cli/s3/du.go new file mode 100644 index 0000000..ac4f6e6 --- /dev/null +++ b/internal/cli/s3/du.go @@ -0,0 +1,132 @@ +package s3cli + +import ( + "context" + "fmt" + "io" + "sort" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// duFlags holds the options accepted by "du". +type duFlags struct { + group bool +} + +const duLong = `Total the size of a bucket, or of one prefix in it. + +S3 has no "how big is this" call, so this is a full listing folded into a sum: +the objects are never downloaded, but every key is walked, which costs one +request per 1000 of them. The listing streams, so the size of the bucket does +not become the size of koc's memory. + +Sizes are exact bytes, like the rest of koc, so they stay usable in +--format value/csv and in arithmetic. + +--group reports one row per storage class instead of one total, which is what +tells a mixed bucket's cheap tier from its expensive one. Objects a store +reports with no storage class are grouped as STANDARD, which is what it means.` + +const duExample = ` # A whole bucket + koc s3 du db-backups + + # Just the end-to-end test dumps + koc s3 du db-backups/e2e- + + # Per storage class + koc s3 du db-backups --group` + +func newDuCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + df := &duFlags{} + cmd := &cobra.Command{ + Use: "du [/]", + Short: "Total the size of a bucket or prefix", + Long: duLong, + Example: duExample, + Aliases: []string{"usage"}, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, prefix, err := parseRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runDu(ctx, client, o, bucket, prefix, df, cmd.OutOrStdout()) + }, + } + cmd.Flags().BoolVar(&df.group, "group", false, "one row per storage class instead of one total") + return cmd +} + +// usage is the running total for one storage class, or for the whole listing. +type usage struct { + objects int64 + bytes int64 +} + +// runDu is the test seam for "du". +func runDu(ctx context.Context, client *s3.Client, o *output.Options, + bucket, prefix string, f *duFlags, w io.Writer) error { + var total usage + byClass := map[string]*usage{} + + err := client.ListObjectsFunc(ctx, bucket, prefix, 0, func(obj s3.Object) error { + total.objects++ + total.bytes += obj.Size + if f.group { + class := obj.StorageClass + if class == "" { + class = "STANDARD" + } + u, ok := byClass[class] + if !ok { + u = &usage{} + byClass[class] = u + } + u.objects++ + u.bytes += obj.Size + } + return nil + }) + if err != nil { + return fmt.Errorf("totalling %s/%s: %w", bucket, prefix, err) + } + + if f.group { + return writeUsageByClass(w, o, byClass) + } + return o.WriteSingle(w, + []string{"Bucket", "Prefix", "Objects", "Size"}, + []any{bucket, prefix, total.objects, total.bytes}) +} + +// writeUsageByClass renders --group's per-storage-class rows, ordered by class +// name so two runs of the same bucket diff cleanly. +func writeUsageByClass(w io.Writer, o *output.Options, byClass map[string]*usage) error { + classes := make([]string, 0, len(byClass)) + for class := range byClass { + classes = append(classes, class) + } + sort.Strings(classes) + + rows := make([][]any, len(classes)) + for i, class := range classes { + rows[i] = []any{class, byClass[class].objects, byClass[class].bytes} + } + return o.WriteList(w, output.Table{ + Columns: []string{"Storage Class", "Objects", "Size"}, + Rows: rows, + }) +} diff --git a/internal/cli/s3/du_test.go b/internal/cli/s3/du_test.go new file mode 100644 index 0000000..cc575f1 --- /dev/null +++ b/internal/cli/s3/du_test.go @@ -0,0 +1,73 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/ftarasenko/go-openstackclient/internal/output" +) + +func TestRunDu(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + err := runDu(context.Background(), client, valueOpts(), "db-backups", "", &duFlags{}, &buf) + if err != nil { + t.Fatal(err) + } + // Bucket, prefix, objects, exact bytes — 100 + 20 + 3, one field per line. + if got, want := buf.String(), "db-backups\n\n3\n123\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestRunDuGroupByStorageClass(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + err := runDu(context.Background(), client, valueOpts(), "db-backups", "", &duFlags{group: true}, &buf) + if err != nil { + t.Fatal(err) + } + // Classes sort by name, and an object with no class counts as STANDARD. + want := "GLACIER\t1\t20\nSTANDARD\t2\t103\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// duListHandler serves one page of three objects across two storage classes, +// one of which the store reports with no class at all. +func duListHandler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("du issued %s, want GET only", r.Method) + } + _, _ = fmt.Fprint(w, `false + a100STANDARD + b20GLACIER + c3 + `) + } +} + +// du must be able to render as a table too, not only in --format value. +func TestRunDuTable(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + o := &output.Options{Format: output.FormatJSON} + if err := runDu(context.Background(), client, o, "db-backups", "e2e-", &duFlags{}, &buf); err != nil { + t.Fatal(err) + } + for _, want := range []string{`"Objects": 3`, `"Size": 123`, `"Prefix": "e2e-"`} { + if !strings.Contains(buf.String(), want) { + t.Errorf("JSON output %s missing %q", buf.String(), want) + } + } +} diff --git a/internal/cli/s3/object.go b/internal/cli/s3/object.go index a1be07c..3fdcbb7 100644 --- a/internal/cli/s3/object.go +++ b/internal/cli/s3/object.go @@ -22,6 +22,7 @@ func newObjectCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.C } cmd.AddCommand(newObjectListCommand(a, o, f)) cmd.AddCommand(newObjectShowCommand(a, o, f)) + cmd.AddCommand(newObjectDeleteCommand(a, o, f)) return cmd } diff --git a/internal/cli/s3/object_delete.go b/internal/cli/s3/object_delete.go new file mode 100644 index 0000000..b7c8b21 --- /dev/null +++ b/internal/cli/s3/object_delete.go @@ -0,0 +1,127 @@ +package s3cli + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/cli/batchdelete" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// objectDeleteFlags holds the options accepted by "object delete". +type objectDeleteFlags struct { + recursive bool + dryRun bool +} + +const objectDeleteLong = `Delete objects. + +Each argument is one object: "/", or "s3:///". Every +argument is attempted even if an earlier one fails, and the command exits +non-zero naming each failure. + +S3 answers a delete of a key that was never there with success, so deleting is +idempotent: a key that is already gone is reported deleted rather than as an +error. That is the protocol's behaviour, not a choice koc makes — do not use the +exit status to test whether a key existed, use "koc s3 object show". + +--recursive turns each argument's key into a prefix and deletes every object +under it, which is also how a bucket is emptied before "koc s3 bucket delete": + + koc s3 object delete db-backups/e2e- --recursive # one prefix + koc s3 object delete db-backups/ --recursive # the whole bucket + +That is the one destructive shape in this group, so it is spelled out rather +than inferred from a trailing slash or a wildcard, and --dry-run prints exactly +the keys it would delete without touching any of them.` + +func newObjectDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + df := &objectDeleteFlags{} + cmd := &cobra.Command{ + Use: "delete / [/ ...]", + Short: "Delete objects", + Long: objectDeleteLong, + Aliases: []string{"remove"}, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runObjectDelete(ctx, client, args, df, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVarP(&df.recursive, "recursive", "r", false, + "treat each key as a prefix and delete every object under it") + fl.BoolVar(&df.dryRun, "dry-run", false, + "print the objects that would be deleted and delete nothing") + return cmd +} + +// runObjectDelete is the test seam for "object delete". +func runObjectDelete(ctx context.Context, client *s3.Client, refs []string, + f *objectDeleteFlags, w io.Writer) error { + return batchdelete.Each(refs, func(ref string) error { + if f.recursive { + // A bare bucket is a legal prefix ref here — "/" is how the + // whole bucket is named — so parseRef, not parseObjectRef. + bucket, prefix, err := parseRef(ref) + if err != nil { + return err + } + return deletePrefix(ctx, client, bucket, prefix, f.dryRun, w) + } + bucket, key, err := parseObjectRef(ref) + if err != nil { + return err + } + return deleteOne(ctx, client, bucket, key, f.dryRun, w) + }) +} + +// deletePrefix deletes every object under prefix, one signed DELETE per key. +// +// Keys are deleted as the listing streams rather than after collecting it, so +// emptying a bucket costs one page of memory whatever its size. There is no +// batch DeleteObjects call: it would cut the request count, but it needs a +// Content-MD5 over a hand-built XML body and reports per-key failures in a +// 200 response, and the buckets koc is aimed at hold backups in the dozens. +func deletePrefix(ctx context.Context, client *s3.Client, bucket, prefix string, + dryRun bool, w io.Writer) error { + seen := 0 + err := client.ListObjectsFunc(ctx, bucket, prefix, 0, func(obj s3.Object) error { + seen++ + return deleteOne(ctx, client, bucket, obj.Key, dryRun, w) + }) + if err != nil { + return fmt.Errorf("deleting %s/%s recursively: %w", bucket, prefix, err) + } + if seen == 0 { + _, err = fmt.Fprintf(w, "No objects under %s/%s\n", bucket, prefix) + } + return err +} + +// deleteOne deletes a single key, or says what it would have deleted. +func deleteOne(ctx context.Context, client *s3.Client, bucket, key string, + dryRun bool, w io.Writer) error { + if dryRun { + _, err := fmt.Fprintf(w, "Would delete object: %s/%s\n", bucket, key) + return err + } + if err := client.DeleteObject(ctx, bucket, key); err != nil { + return fmt.Errorf("deleting %s/%s: %w", bucket, key, err) + } + _, err := fmt.Fprintf(w, "Deleted object: %s/%s\n", bucket, key) + return err +} diff --git a/internal/cli/s3/object_delete_test.go b/internal/cli/s3/object_delete_test.go new file mode 100644 index 0000000..265f87d --- /dev/null +++ b/internal/cli/s3/object_delete_test.go @@ -0,0 +1,116 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" +) + +func TestRunObjectDelete(t *testing.T) { + var paths []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + refs := []string{"db-backups/a.sql.gz", "s3://db-backups/b.sql.gz"} + if err := runObjectDelete(context.Background(), client, refs, &objectDeleteFlags{}, &buf); err != nil { + t.Fatal(err) + } + want := "DELETE /db-backups/a.sql.gz,DELETE /db-backups/b.sql.gz" + if got := strings.Join(paths, ","); got != want { + t.Errorf("requests = %q, want %q", got, want) + } + wantOut := "Deleted object: db-backups/a.sql.gz\nDeleted object: db-backups/b.sql.gz\n" + if got := buf.String(); got != wantOut { + t.Errorf("output = %q, want %q", got, wantOut) + } +} + +// Without --recursive a bare bucket is not a target: it would be a one-word +// typo away from emptying the bucket. +func TestRunObjectDeleteRequiresKey(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("no request should be made for a keyless ref") + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups"}, &objectDeleteFlags{}, &buf) + if err == nil { + t.Fatal("expected a keyless ref to be rejected") + } +} + +func TestRunObjectDeleteRecursive(t *testing.T) { + var deleted []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + deleted = append(deleted, r.URL.Path) + w.WriteHeader(http.StatusNoContent) + return + } + if got := r.URL.Query().Get("prefix"); got != "e2e-" { + t.Errorf("prefix = %q, want e2e-", got) + } + _, _ = fmt.Fprint(w, `false + e2e-a.sql.gz1 + e2e-b.sql.gz2 + `) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/e2e-"}, + &objectDeleteFlags{recursive: true}, &buf) + if err != nil { + t.Fatal(err) + } + want := "/db-backups/e2e-a.sql.gz,/db-backups/e2e-b.sql.gz" + if got := strings.Join(deleted, ","); got != want { + t.Errorf("deleted = %q, want %q", got, want) + } +} + +// --dry-run must list the same keys it would remove and issue no DELETE at all. +func TestRunObjectDeleteDryRun(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + t.Errorf("--dry-run issued DELETE %s", r.URL.Path) + } + _, _ = fmt.Fprint(w, `false + a.sql.gz + `) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/"}, + &objectDeleteFlags{recursive: true, dryRun: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "Would delete object: db-backups/a.sql.gz\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// An empty prefix must say so rather than reporting a silent success that reads +// like "everything was already deleted". +func TestRunObjectDeleteRecursiveNoMatches(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false`) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/gone-"}, + &objectDeleteFlags{recursive: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "No objects under db-backups/gone-\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} diff --git a/internal/cli/s3/s3.go b/internal/cli/s3/s3.go index 2d6f1c1..90132a5 100644 --- a/internal/cli/s3/s3.go +++ b/internal/cli/s3/s3.go @@ -10,7 +10,7 @@ import ( // s3Long documents the credential sources on the group itself. The connection // flags are this group's primary input and are registered here rather than // globally, so this is the only place they are discoverable. -const s3Long = `List buckets and objects in an S3-compatible store, and move files in and out. +const s3Long = `Manage buckets and objects in an S3-compatible store, and move files in and out. This group is koc-specific (S3 is not an OpenStack service, and upstream's object-store commands speak Swift) and does not authenticate against Keystone: @@ -42,7 +42,10 @@ variables, so export it from either: Addressing is path-style by default (//), matching the --host-bucket setting the backup pipeline gives s3cmd; pass --no-path-style for -a store fronted by wildcard DNS.` +a store fronted by wildcard DNS. + +Every reference accepts either "/" or the "s3:///" +spelling, so a path copied out of s5cmd or "aws s3" pastes in unchanged.` const s3Example = ` # Everything the credentials can see koc s3 bucket list @@ -54,6 +57,14 @@ const s3Example = ` # Everything the credentials can see koc s3 download db-backups/ ./dump.sql.gz koc s3 download db-backups/.sha256 - + # How much space the backups take, per storage class + koc s3 du db-backups --group + + # A scratch bucket, and its removal once emptied + koc s3 bucket create scratch + koc s3 object delete scratch/ --recursive + koc s3 bucket delete scratch + # GitLab's own key, straight from the cluster koc s3 --s3-creds-from-ns lcm-gitlab bucket list` @@ -72,5 +83,6 @@ func NewCommand(a *auth.Options, o *output.Options) *cobra.Command { cmd.AddCommand(newObjectCommand(a, o, f)) cmd.AddCommand(newDownloadCommand(a, o, f)) cmd.AddCommand(newUploadCommand(a, o, f)) + cmd.AddCommand(newDuCommand(a, o, f)) return cmd } diff --git a/internal/s3/lifecycle_test.go b/internal/s3/lifecycle_test.go new file mode 100644 index 0000000..330b317 --- /dev/null +++ b/internal/s3/lifecycle_test.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "testing" +) + +func TestCreateBucketSendsLocationConstraint(t *testing.T) { + var gotMethod, gotPath, gotBody string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + body, _ := io.ReadAll(r.Body) + gotMethod, gotPath, gotBody = r.Method, r.URL.Path, string(body) + w.WriteHeader(http.StatusOK) + }) + + if err := client.CreateBucket(context.Background(), "scratch"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut { + t.Errorf("method = %q, want PUT", gotMethod) + } + if gotPath != "/scratch" { + t.Errorf("path = %q, want /scratch", gotPath) + } + // The signing region is the location constraint: a request signed for one + // region is not served by another, so there is nothing else it could be. + want := "garage" + if gotBody != want { + t.Errorf("body = %q, want %q", gotBody, want) + } +} + +// us-east-1 is the one region AWS rejects a location constraint for, so the +// body must be absent rather than naming it. +func TestCreateBucketOmitsBodyForUSEast1(t *testing.T) { + var gotBody string + var gotLen int64 + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + gotBody, gotLen = string(body), r.ContentLength + w.WriteHeader(http.StatusOK) + }) + client.cfg.Region = usEast1 + + if err := client.CreateBucket(context.Background(), "scratch"); err != nil { + t.Fatal(err) + } + if gotBody != "" { + t.Errorf("body = %q, want empty", gotBody) + } + if gotLen > 0 { + t.Errorf("Content-Length = %d, want 0", gotLen) + } +} + +func TestCreateBucketAlreadyOwnedReportsCode(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprint(w, `BucketAlreadyOwnedByYou`+ + `Bucket already exists`) + }) + + err := client.CreateBucket(context.Background(), "scratch") + if err == nil { + t.Fatal("expected an error") + } + if got := ErrorCode(err); got != "BucketAlreadyOwnedByYou" { + t.Errorf("ErrorCode = %q, want BucketAlreadyOwnedByYou", got) + } +} + +func TestDeleteBucket(t *testing.T) { + var gotMethod, gotPath string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusNoContent) + }) + + if err := client.DeleteBucket(context.Background(), "scratch"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodDelete || gotPath != "/scratch" { + t.Errorf("request = %s %s, want DELETE /scratch", gotMethod, gotPath) + } +} + +func TestDeleteBucketNotEmptyReportsCode(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprint(w, `BucketNotEmptynope`) + }) + + err := client.DeleteBucket(context.Background(), "scratch") + if got := ErrorCode(err); got != "BucketNotEmpty" { + t.Errorf("ErrorCode = %q, want BucketNotEmpty (err = %v)", got, err) + } +} + +func TestDeleteObject(t *testing.T) { + var gotMethod, gotPath string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusNoContent) + }) + + if err := client.DeleteObject(context.Background(), "db-backups", "dump.sql.gz"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodDelete || gotPath != "/db-backups/dump.sql.gz" { + t.Errorf("request = %s %s, want DELETE /db-backups/dump.sql.gz", gotMethod, gotPath) + } +} + +// ErrorCode must not claim a code for an error that never came from S3. +func TestErrorCodeOnForeignError(t *testing.T) { + if got := ErrorCode(io.EOF); got != "" { + t.Errorf("ErrorCode(io.EOF) = %q, want empty", got) + } + if got := ErrorCode(nil); got != "" { + t.Errorf("ErrorCode(nil) = %q, want empty", got) + } +} + +// ListObjectsFunc must follow continuation tokens and stop the walk the moment +// the limit is reached, without asking for a page it cannot use. +func TestListObjectsFuncPagesAndCapsLimit(t *testing.T) { + var maxKeys []string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + maxKeys = append(maxKeys, r.URL.Query().Get("max-keys")) + if r.URL.Query().Get("continuation-token") == "" { + _, _ = fmt.Fprint(w, `true + tok + a1 + b2 + `) + return + } + _, _ = fmt.Fprint(w, `false + c4 + `) + }) + + var keys []string + var sum int64 + err := client.ListObjectsFunc(context.Background(), "b", "", 3, func(o Object) error { + keys = append(keys, o.Key) + sum += o.Size + return nil + }) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(keys, ","); got != "a,b,c" { + t.Errorf("keys = %q, want a,b,c", got) + } + if sum != 7 { + t.Errorf("sum = %d, want 7", sum) + } + // Page two may ask for at most the one key still wanted. + if len(maxKeys) != 2 { + t.Fatalf("requests = %d, want 2", len(maxKeys)) + } + if n, _ := strconv.Atoi(maxKeys[1]); n != 1 { + t.Errorf("second max-keys = %q, want 1", maxKeys[1]) + } +} + +// An error from the callback stops the walk and reaches the caller unwrapped, +// which is what lets a delete-as-you-list abort on the first failure. +func TestListObjectsFuncPropagatesCallbackError(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + ab + `) + }) + + seen := 0 + err := client.ListObjectsFunc(context.Background(), "b", "", 0, func(Object) error { + seen++ + return io.ErrUnexpectedEOF + }) + if err != io.ErrUnexpectedEOF { //nolint:errorlint // identity is the assertion + t.Errorf("err = %v, want io.ErrUnexpectedEOF", err) + } + if seen != 1 { + t.Errorf("callback ran %d times, want 1", seen) + } +} diff --git a/internal/s3/s3.go b/internal/s3/s3.go index db942af..fae92f9 100644 --- a/internal/s3/s3.go +++ b/internal/s3/s3.go @@ -13,6 +13,7 @@ package s3 import ( + "bytes" "context" "crypto/sha256" "crypto/tls" @@ -190,6 +191,18 @@ func IsNotFound(err error) bool { return ae.StatusCode == http.StatusNotFound } +// ErrorCode returns the S3 error code carried by err ("BucketNotEmpty", +// "BucketAlreadyOwnedByYou"), or "" if err is not an S3 API error. Callers use +// it to turn a specific server answer into advice; matching on the message +// would break with the server's wording. +func ErrorCode(err error) string { + var ae *APIError + if !errors.As(err, &ae) { + return "" + } + return ae.Code +} + // Bucket is one entry of a ListBuckets result. type Bucket struct { Name string @@ -240,6 +253,56 @@ func (c *Client) ListBuckets(ctx context.Context) ([]Bucket, error) { return out, nil } +// usEast1 is the one region whose name must NOT appear in a +// CreateBucketConfiguration: AWS treats it as the default and rejects the body +// that names it. +const usEast1 = "us-east-1" + +// createBucketConfiguration is the CreateBucket request body. It exists because +// every region other than us-east-1 requires the bucket's location to be stated +// explicitly, and Garage validates the value against its configured region. +type createBucketConfiguration struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + LocationConstraint string `xml:"LocationConstraint"` +} + +// CreateBucket creates a bucket. The location constraint is the signing region, +// which is necessarily the right answer: a request signed for region R is only +// accepted by a store serving R. +// +// Creating a bucket the credentials already own is not an error on AWS outside +// us-east-1 either — it answers BucketAlreadyOwnedByYou, which the caller can +// recognise with ErrorCode. +func (c *Client) CreateBucket(ctx context.Context, bucket string) error { + req := request{method: http.MethodPut, url: c.url(bucket, "", nil), payloadHash: emptySHA256} + + if region := c.cfg.Region; region != "" && region != usEast1 { + body, err := xml.Marshal(createBucketConfiguration{LocationConstraint: region}) + if err != nil { + return fmt.Errorf("encoding create-bucket body: %w", err) + } + req.body = bytes.NewReader(body) + req.payloadHash = hexSHA256(body) + req.size = int64(len(body)) + req.header = map[string]string{"Content-Type": "application/xml"} + } + + return c.do(ctx, req, drainBody) +} + +// DeleteBucket removes an empty bucket. A bucket with objects in it is refused +// by the server with BucketNotEmpty; nothing is deleted implicitly. +func (c *Client) DeleteBucket(ctx context.Context, bucket string) error { + return c.do(ctx, request{method: http.MethodDelete, url: c.url(bucket, "", nil), payloadHash: emptySHA256}, drainBody) +} + +// drainBody is the sink for a call whose reply carries nothing the caller wants. +// The body is still read so the connection goes back to the pool. +func drainBody(resp *http.Response) error { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, errorBodyLimit)) + return nil +} + // listObjectsPage is one ListObjectsV2 response. type listObjectsPage struct { IsTruncated bool `xml:"IsTruncated"` @@ -259,9 +322,25 @@ type listObjectsPage struct { // same rule the rest of koc's --limit flags follow. func (c *Client) ListObjects(ctx context.Context, bucket, prefix string, limit int) ([]Object, error) { var out []Object - token := "" + err := c.ListObjectsFunc(ctx, bucket, prefix, limit, func(o Object) error { + out = append(out, o) + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +// ListObjectsFunc is ListObjects without materialising the result: fn is called +// once per key, in listing order, and an error from it stops the walk and is +// returned as-is. Callers that only fold over the listing (summing sizes, +// deleting as they go) use this so a bucket with a million keys costs one page +// of memory instead of all of them. +func (c *Client) ListObjectsFunc(ctx context.Context, bucket, prefix string, limit int, fn func(Object) error) error { + token, seen := "", 0 for { - q := url.Values{"list-type": {"2"}, "max-keys": {strconv.Itoa(pageSize(limit, len(out)))}} + q := url.Values{"list-type": {"2"}, "max-keys": {strconv.Itoa(pageSize(limit, seen))}} if prefix != "" { q.Set("prefix", prefix) } @@ -271,22 +350,25 @@ func (c *Client) ListObjects(ctx context.Context, bucket, prefix string, limit i var page listObjectsPage if err := c.getXML(ctx, c.url(bucket, "", q), &page); err != nil { - return nil, err + return err } for _, o := range page.Contents { - out = append(out, Object{ + err := fn(Object{ Key: o.Key, Size: o.Size, LastModified: parseS3Time(o.LastModified), ETag: strings.Trim(o.ETag, `"`), StorageClass: o.StorageClass, }) - if limit > 0 && len(out) >= limit { - return out, nil + if err != nil { + return err + } + if seen++; limit > 0 && seen >= limit { + return nil } } if !page.IsTruncated || page.NextContinuationToken == "" { - return out, nil + return nil } token = page.NextContinuationToken } @@ -330,6 +412,13 @@ func (c *Client) GetObject(ctx context.Context, bucket, key string, w io.Writer) return n, err } +// DeleteObject removes one object. S3 answers 204 for a key that was never +// there, so a delete is idempotent and "no such key" is not reported — the +// caller asked for the key to be gone, and it is. +func (c *Client) DeleteObject(ctx context.Context, bucket, key string) error { + return c.do(ctx, request{method: http.MethodDelete, url: c.url(bucket, key, nil), payloadHash: emptySHA256}, drainBody) +} + // PutObject uploads body as a single part. The reader must be seekable because // SigV4 signs a hash of the payload: the body is read once to hash it, then // rewound and sent. That rules out streaming from a pipe, and is the reason From 82afd0a2127dd2619d383e505cf2632d0c99614d Mon Sep 17 00:00:00 2001 From: Fedor Tarasenko Date: Mon, 14 Sep 2026 11:37:00 +0000 Subject: [PATCH 2/3] feat(s3): multipart, recursive transfers, copy/move, presign, versioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the rest of the gap against s5cmd. Every item below was something "koc s3" could not do at all, and several were hard limits rather than missing conveniences. Transfers - Uploads are multipart past --part-size (16 MiB default, --concurrency parts at once), so the 5 GiB single-PUT ceiling is gone: a dump of any size uploads. A failure aborts the upload, so a half-written object never becomes visible and the stored parts stop accruing cost. - "koc s3 upload -" reads standard input, which multipart is what makes possible: SigV4 has to hash a single-PUT body before sending it, and a pipe cannot be rewound. "mysqldump | gzip | koc s3 upload - b/k" now works without staging the dump on disk. - --recursive (and a wildcard reference) on download, upload, copy, move and object delete works over a whole prefix or tree, --concurrency objects at a time — a thousand small objects were a thousand serial round trips. --include/--exclude take globs where "*" spans "/", as in s5cmd; --dry-run prints exactly what would move. - A recursive download mirrors each key's path below the prefix, skips files that already exist (so an interrupted restore resumes by re-running), and refuses a key that would land outside the destination: "../../etc/cron.d/x" is a legal S3 key, and writing it where it says would let whoever can write to the bucket write anywhere koc can. New verbs - "copy"/"move": server-side, so the bytes never travel through koc and a 100 GiB object is one small request. A move is the copy and then the delete, in that order — a failure leaves the source intact. - "presign": a URL that reads (or, --method put, writes) one object with no key attached. Pure local signing, no request, so it works offline. - "bucket show": HEAD-based existence/reachability probe plus the versioning state, which is also how to tell Garage from a store that has versioning. - "bucket set --versioning enabled|suspended". Other - A retryable failure (5xx, 429, 408, SlowDown, or a transport error) is retried with exponential backoff, --s3-retries times, default 5. A failure raised by the response sink is never replayed: it has already written part of the object to a file or to stdout. - --s3-anonymous sends requests unsigned, for a publicly readable bucket; it is the only mode that needs no credentials. - Recursive deletes go out in batches of up to 1000 keys per request instead of one DELETE per key, with the Content-MD5 AWS requires. Per-key refusals arrive inside a 200 and are reported as failures while the rest of the batch is still reported deleted. - "object list --delimiter /" collapses each subtree into one entry, so a deep bucket walks one level at a time. - --human renders sizes for a reader on object list, object show and du; exact bytes stay the default everywhere, because a rounded "14.2 GiB" is not a number a script can add up. The formatter lives in internal/output, not inline. - --all-versions / --version-id on list, show, delete, download and copy. Garage implements no versioning, so these are for AWS, Ceph RGW or MinIO; Garage's own answer ("unversioned") is reported honestly. Implementation notes - request.body is now an io.ReadSeeker so a retry can replay it, rewound to the caller's offset rather than to byte zero. - ListObjects/ListObjectsFunc take a ListOptions struct (prefix, delimiter, limit, versions) instead of three positional arguments. - A copy and a multipart completion can both be refused inside a 200 response; both paths check the body for an document rather than decoding only the fields they want and reporting success. - Batch deletes and PutBucketVersioning need MD5 (Content-MD5), which is fixed by the protocol and not a security choice; the reasoning sits in internal/s3/integrity.go next to the nolint. - Under --recursive the pool's workers are objects and each object's own multipart then goes out a part at a time, so --concurrency is not squared into that many part buffers. Verification. Presigned URLs are asserted against golden signatures from an independent SigV4 query-signing implementation written from the AWS spec — a URL is only useful if a server accepts it, so agreeing with a second implementation is the property worth testing. The whole surface was then exercised end to end against a mock endpoint that verifies SigV4 itself (header and query) and speaks the multipart protocol: stdin upload, a 12 MiB three-part upload, recursive download/upload with skip-on-exists, wildcard selection, copy, move, batch delete with a checked Content-MD5, bucket show, du --group --human, and a presigned URL fetched with curl (plus a tampered one rejected). Every request in that run verified; the only signature failures logged were the deliberate ones. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TQ2Xt7Epc3n8XAXiAgPXiC --- AGENTS.md | 7 +- README.md | 63 +++- docs/coverage.md | 16 +- internal/cli/s3/bucket.go | 128 +++++++ internal/cli/s3/client.go | 30 +- internal/cli/s3/copy.go | 300 ++++++++++++++++ internal/cli/s3/copy_test.go | 292 +++++++++++++++ internal/cli/s3/download.go | 339 ++++++++++++++++++ internal/cli/s3/du.go | 54 ++- internal/cli/s3/filter.go | 124 +++++++ internal/cli/s3/filter_test.go | 103 ++++++ internal/cli/s3/flags_test.go | 311 ++++++++++++++++ internal/cli/s3/object.go | 155 ++++++-- internal/cli/s3/object_delete.go | 199 +++++++++-- internal/cli/s3/object_delete_test.go | 182 +++++++++- internal/cli/s3/pool.go | 94 +++++ internal/cli/s3/presign.go | 104 ++++++ internal/cli/s3/s3.go | 29 +- internal/cli/s3/s3_test.go | 35 +- internal/cli/s3/stop.go | 11 + internal/cli/s3/transfer.go | 207 ----------- internal/cli/s3/transfer_bulk_test.go | 477 +++++++++++++++++++++++++ internal/cli/s3/transfer_paths_test.go | 21 +- internal/cli/s3/upload.go | 370 +++++++++++++++++++ internal/output/output.go | 37 ++ internal/s3/copy.go | 82 +++++ internal/s3/copy_delete_test.go | 206 +++++++++++ internal/s3/delete.go | 118 ++++++ internal/s3/errors_test.go | 8 +- internal/s3/integrity.go | 20 ++ internal/s3/lifecycle_test.go | 6 +- internal/s3/multipart.go | 389 ++++++++++++++++++++ internal/s3/multipart_test.go | 275 ++++++++++++++ internal/s3/presign.go | 97 +++++ internal/s3/presign_test.go | 125 +++++++ internal/s3/retry.go | 91 +++++ internal/s3/retry_test.go | 234 ++++++++++++ internal/s3/s3.go | 280 ++++++++++++--- internal/s3/s3_test.go | 12 +- internal/s3/sign.go | 4 + internal/s3/transport.go | 6 + internal/s3/versioning.go | 185 ++++++++++ internal/s3/versioning_test.go | 260 ++++++++++++++ 43 files changed, 5697 insertions(+), 389 deletions(-) create mode 100644 internal/cli/s3/copy.go create mode 100644 internal/cli/s3/copy_test.go create mode 100644 internal/cli/s3/download.go create mode 100644 internal/cli/s3/filter.go create mode 100644 internal/cli/s3/filter_test.go create mode 100644 internal/cli/s3/flags_test.go create mode 100644 internal/cli/s3/pool.go create mode 100644 internal/cli/s3/presign.go create mode 100644 internal/cli/s3/stop.go delete mode 100644 internal/cli/s3/transfer.go create mode 100644 internal/cli/s3/transfer_bulk_test.go create mode 100644 internal/cli/s3/upload.go create mode 100644 internal/s3/copy.go create mode 100644 internal/s3/copy_delete_test.go create mode 100644 internal/s3/delete.go create mode 100644 internal/s3/integrity.go create mode 100644 internal/s3/multipart.go create mode 100644 internal/s3/multipart_test.go create mode 100644 internal/s3/presign.go create mode 100644 internal/s3/presign_test.go create mode 100644 internal/s3/retry.go create mode 100644 internal/s3/retry_test.go create mode 100644 internal/s3/versioning.go create mode 100644 internal/s3/versioning_test.go diff --git a/AGENTS.md b/AGENTS.md index a04f2bc..4e51ebd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -197,11 +197,14 @@ internal/auth/ one authenticated ProviderClient per invocation + per debug.go --debug transport: redacts tokens+secrets, elides large bodies internal/kube/ minimal read-only k8s REST client (kubeconfig + secret/Ironic reads); no client-go internal/vault/ minimal Vault REST client (AppRole login / token + KV v2 read + X-Vault-Namespace) -internal/s3/ minimal S3 REST client: hand-rolled SigV4 + list/head/get/put (no aws-sdk-go-v2 / minio-go) +internal/s3/ minimal S3 REST client, no aws-sdk-go-v2 / minio-go: hand-rolled SigV4 + (header + presigned query), list/head/get/put, multipart upload, + server-side copy, batch delete, versioning, retry with backoff internal/output/ -f/--format {table,json,yaml,value,csv} and -c/--column layer internal/cli/keyvrm/ KeyVRM (in-house catalog service); typed request layer (types.go/requests.go) + cobra verbs internal/cli/vault/ "koc vault kv" list/get/copy/export/decrypt (package vaultcli); Vault creds only -internal/cli/s3/ "koc s3" bucket/object list+show, download, upload (package s3cli); S3 creds only +internal/cli/s3/ "koc s3" bucket/object lifecycle, du, download/upload (multipart, stdin, + recursive), server-side copy/move, presign (package s3cli); S3 creds only internal/cli/quota/ "koc quota show|set" — the one cross-service noun (nova+cinder+neutron) internal/cli/ root.go wires every service's command group onto the root internal/cli/resolve/ cross-service name→ID (image→glance, network→neutron, project→keystone) diff --git a/README.md b/README.md index 294bf52..e8e0405 100644 --- a/README.md +++ b/README.md @@ -521,34 +521,79 @@ S3 credentials alone are used, so it works on a host with no cloud credentials. ```sh koc s3 bucket list +koc s3 bucket show db-backups # exists? reachable? versioned? koc s3 bucket create scratch koc s3 bucket delete scratch # the bucket must be empty koc s3 object list db-backups +koc s3 object list db-backups --delimiter / # one level, like a directory koc s3 object show db-backups/ # HEAD only, no transfer koc s3 object delete db-backups/ koc s3 object delete db-backups/e2e- -r # every key under a prefix -koc s3 du db-backups # objects and exact bytes -koc s3 du db-backups --group # per storage class +koc s3 du db-backups --human --group # size, per storage class koc s3 download db-backups/ ./dump.mbs.gz.enc koc s3 download db-backups/.sha256 - # "-" streams to stdout, so it pipes +koc s3 download db-backups/2026/ ./restore -r koc s3 upload ./dump.mbs.gz.enc db-backups/ +koc s3 upload ./restore db-backups/2026/ -r +koc s3 copy db-backups/ db-backups/latest.mbs.gz.enc +koc s3 move db-backups/ archive/ +koc s3 presign db-backups/ --expire 1h ``` Every ref also accepts the `s3:///` spelling, so a path copied from -`s5cmd` or `aws s3` pastes in unchanged. +`s5cmd` or `aws s3` pastes in unchanged, and a wildcard (`"db-backups/2026/*.gz"`) +selects many where a command takes one. `bucket list` is scoped to the **access key**, not to the store: Garage answers with the buckets that key is granted, so a key made for one bucket lists exactly that one. +**Transfers.** `upload` has no 5 GiB ceiling: anything past `--part-size` (16 MiB +by default) goes out as a multipart upload, `--concurrency` parts at a time, and +a failure aborts it so a half-written object never becomes visible. `-` as the +source reads **standard input**, which is what makes a dump streamable without +staging it on disk: + +```sh +mysqldump --all-databases | gzip | koc s3 upload - db-backups/nightly.sql.gz +``` + +`--recursive` on `download`, `upload`, `copy`, `move` and `object delete` works +on a whole prefix or tree, `--concurrency` objects at a time — which is most of +why a bulk restore finishes, since a thousand small objects are otherwise a +thousand serial round trips. `--include`/`--exclude` take globs (where `*` spans +`/`, as in s5cmd) and `--dry-run` prints exactly what would move. + +`copy` and `move` are **server-side**: the source is named in a header, so the +bytes never travel through koc and a 100 GiB object is one small request. A move +is the copy and then the delete, in that order — a failure leaves the source +intact rather than losing the object. + `object delete --recursive` is the one destructive shape in the group, so it is -never inferred from a trailing slash or a wildcard — and `--dry-run` prints -exactly the keys it would remove without touching any of them. It is also how a -bucket is emptied before `bucket delete`, which never removes objects -implicitly. +never inferred from a trailing slash — and `--dry-run` prints exactly the keys it +would remove without touching any of them. It is also how a bucket is emptied +before `bucket delete`, which never removes objects implicitly. Keys go out in +batches of up to 1000 per request, so emptying a large bucket costs a thousandth +of the round trips. `du` is a listing folded into a sum, because S3 has no "how big is this" call: no object is downloaded, but every key is walked, one request per 1000 of them. +Sizes are exact bytes everywhere unless `--human` is given — a rounded +"14.2 GiB" is not a number a script can add up. + +`presign` prints a URL that reads (or, with `--method put`, writes) one object +with no credentials attached to the recipient. It makes **no request** — pure +local signing — so it works offline, and the URL is a bearer credential for that +key until it expires: treat it like a password and keep `--expire` short. + +**Versioning** (`bucket show`, `bucket set --versioning`, `--all-versions`, +`--version-id`) is implemented but Garage has none — it reports every bucket as +unversioned — so those are for a koc pointed at AWS, Ceph RGW or MinIO. + +A failed request is retried with exponential backoff (`--s3-retries`, 5 by +default): on a cluster network a reset connection mid-transfer should not cost +the whole command. `--s3-anonymous` sends requests unsigned, for a bucket +granted to everyone. Credentials come from flags, from the environment, or from a Kubernetes Secret: @@ -560,6 +605,8 @@ Credentials come from flags, from the environment, or from a Kubernetes Secret: | `--s3-region` | `AWS_REGION`, `AWS_DEFAULT_REGION`, `S3_REGION`, `s3_region` | `garage` | | `--s3-cacert` | `AWS_CA_BUNDLE`, `S3_CACERT` | system roots | | `--s3-creds-from-ns` | `KOC_S3_CREDS_FROM_NS` | — | +| `--s3-anonymous` | `S3_ANONYMOUS` | off (no credentials needed or used) | +| `--s3-retries` | — | `5` | | `--insecure-s3` | `S3_SKIP_VERIFY` | off (the global `--insecure` also applies) | The three env families are deliberate: `AWS_*` so an existing `aws`/`boto` @@ -608,7 +655,7 @@ internal/kube/ minimal read-only k8s REST client (no client-go) internal/vault/ minimal Vault REST client (AppRole/token + KV v2) internal/s3/ minimal S3 REST client (SigV4, no aws-sdk/minio-go) internal/cli/vault/ "koc vault kv" list/get/copy/export/decrypt, no Keystone auth -internal/cli/s3/ "koc s3" bucket/object/du/download/upload, no Keystone auth +internal/cli/s3/ "koc s3" bucket/object/du/transfer/copy/presign, no Keystone auth internal/output/ -f/-c formatter (table/json/yaml/value/csv) internal/cli/ root command wiring internal/cli/resolve/ cross-service name→ID resolution diff --git a/docs/coverage.md b/docs/coverage.md index 0fb10c2..fa78fc1 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -3,7 +3,7 @@ How much of the upstream OpenStack CLI surface `koc` implements, measured against primary sources rather than documentation. -**Snapshot:** 2026-09-14 · `koc` @ this commit (base `96dabfc`) · 559 leaf +**Snapshot:** 2026-09-14 · `koc` @ this commit (base `c532e4c`) · 564 leaf commands (visible tree; 2 more are hidden duplicates). **Keep this file current** — see "Updating this document" below. Any commit that @@ -28,8 +28,8 @@ PyPI is the source of record. ## Headline -**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 559 leaf commands, -515 are upstream-equivalent and 44 are koc-native. +**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 564 leaf commands, +515 are upstream-equivalent and 49 are koc-native. The denominator grew by 13 against the 2026-08-07 snapshot without a single command changing: `python-ironic-inspector-client` is now a **baseline** rather @@ -433,14 +433,14 @@ limitations"; the fix is to make the other resolvers match `server`'s behaviour. ## koc-native commands -No upstream equivalent, by design — **44 leaves**, itemised so the total -reconciles with the headline (559 = 515 + 44): +No upstream equivalent, by design — **49 leaves**, itemised so the total +reconciles with the headline (564 = 515 + 49): | Count | Commands | Why it has no upstream equivalent | | --- | --- | --- | | 18 | `koc keyvrm …` — `app-config` ×2, `availability-zone` ×2, `event` ×4, `host-aggregate-config` ×5, `recommendation` ×5 | in-house KeyVRM catalog service; no gophercloud package and no OSC plugin | | 5 | `koc vault kv list/get/copy/export/decrypt` | Vault is not an OpenStack service; `copy` fills a gap in the Vault CLI itself | -| 9 | `koc s3 bucket list/create/delete`, `koc s3 object list/show/delete`, `koc s3 du`, `koc s3 download/upload` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | +| 14 | `koc s3 bucket list/create/delete/show/set`, `koc s3 object list/show/delete`, `koc s3 du`, `koc s3 download/upload`, `koc s3 copy/move`, `koc s3 presign` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | | 2 | `koc dns pool list/show` | designate's API and its Python SDK both expose `/v2/pools`, but `python-designateclient` registers no `openstack` command for it. Reads only — pool *writes* are a `designate-manage`/config operation on the servers | | 2 | `koc server add/remove server-group` | KeyStack dynamic server groups | | 2 | `koc network trunk subport add`/`remove` | upstream folds these into `network trunk set`/`unset --subport` flags rather than giving them verbs (`network subport list` does exist and is counted — see "Naming deviations") | @@ -481,7 +481,7 @@ The tables are derived, not hand-maintained. To re-derive after a version bump or a batch of new commands: ```sh -# 1. koc's own command tree (559 leaf commands at the snapshot above) +# 1. koc's own command tree (564 leaf commands at the snapshot above) make build # Walk `--help` recursively. Count a command when it is *runnable*, not merely when # it is childless: `koc image import ` is a verb that also parents `koc image @@ -511,7 +511,7 @@ Then **check the arithmetic**, because that is the only thing that makes these tables worth reading. Three identities must hold at every snapshot: 1. every raw row numerator summed = the headline numerator (515); -2. leaf commands = headline numerator + koc-native (559 = 515 + 44); +2. leaf commands = headline numerator + koc-native (564 = 515 + 49); 3. every raw row denominator summed = 901, and minus the two not-targeted rows (swift 17 + manila 40) = the in-scope denominator (844). diff --git a/internal/cli/s3/bucket.go b/internal/cli/s3/bucket.go index 4316f78..a22ab60 100644 --- a/internal/cli/s3/bucket.go +++ b/internal/cli/s3/bucket.go @@ -2,8 +2,10 @@ package s3cli import ( "context" + "errors" "fmt" "io" + "strings" "github.com/spf13/cobra" @@ -23,6 +25,8 @@ func newBucketCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.C cmd.AddCommand(newBucketListCommand(a, o, f)) cmd.AddCommand(newBucketCreateCommand(a, o, f)) cmd.AddCommand(newBucketDeleteCommand(a, o, f)) + cmd.AddCommand(newBucketShowCommand(a, o, f)) + cmd.AddCommand(newBucketSetCommand(a, o, f)) return cmd } @@ -171,3 +175,127 @@ func runBucketDelete(ctx context.Context, client *s3.Client, refs []string, w io return err }) } + +const bucketShowLong = `Show a bucket: that it exists, that the credentials reach it, and its +versioning state. + +The existence check is a HEAD, which costs the server no listing work — the +cheapest possible probe, and the one to use in a script that must know whether +a bucket is there before writing to it. + +Versioning is reported as Enabled, Suspended, or Unversioned for a bucket that +was never configured. Garage answers Unversioned for every bucket because it +implements no versioning at all, so this is also how to tell which kind of store +is on the other end. A store that does not answer the versioning call leaves the +field unknown rather than failing the command.` + +func newBucketShowCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return &cobra.Command{ + Use: "show ", + Short: "Show a bucket's existence and versioning state", + Long: bucketShowLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, err := parseBucketRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketShow(ctx, client, o, bucket, cmd.OutOrStdout()) + }, + } +} + +// runBucketShow is the test seam for "bucket show". +func runBucketShow(ctx context.Context, client *s3.Client, o *output.Options, + bucket string, w io.Writer) error { + if err := client.HeadBucket(ctx, bucket); err != nil { + if s3.IsNotFound(err) { + return fmt.Errorf("no bucket %q on %s (or the credentials cannot see it)", bucket, client.Endpoint()) + } + return fmt.Errorf("reading bucket %q: %w", bucket, err) + } + + // The bucket is there, which is the answer asked for; a store that does not + // implement the versioning call must not turn that into a failure. + versioning := "unknown" + if status, err := client.GetBucketVersioning(ctx, bucket); err == nil { + versioning = status + } + return o.WriteSingle(w, + []string{"Bucket", "Endpoint", "Region", "Versioning"}, + []any{bucket, client.Endpoint(), client.Region(), versioning}) +} + +const bucketSetLong = `Change a bucket's settings. Only versioning is settable. + +--versioning enabled starts keeping every version of every key; suspended stops +without discarding what is already kept. S3 has no way back to the +never-versioned state, which is why "unversioned" is not accepted here. + +Garage implements no versioning, so this fails against it — that is the store +saying so, not koc. Use "koc s3 bucket show" to see which kind of store answers.` + +func newBucketSetCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + var versioning string + cmd := &cobra.Command{ + Use: "set ", + Short: "Change a bucket's settings", + Long: bucketSetLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, err := parseBucketRef(args[0]) + if err != nil { + return err + } + if !cmd.Flags().Changed("versioning") { + return errors.New("nothing to set: pass --versioning enabled|suspended") + } + status, err := versioningStatus(versioning) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketSet(ctx, client, o, bucket, status, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&versioning, "versioning", "", + "object versioning: enabled or suspended") + return cmd +} + +// versioningStatus maps the flag's lower-case spelling to the protocol's. +// Accepting both keeps "--versioning Enabled" from being a puzzling error. +func versioningStatus(v string) (string, error) { + switch strings.ToLower(v) { + case "enabled": + return s3.VersioningEnabled, nil + case "suspended": + return s3.VersioningSuspended, nil + default: + return "", fmt.Errorf("--versioning must be enabled or suspended, got %q", v) + } +} + +// runBucketSet is the test seam for "bucket set". +func runBucketSet(ctx context.Context, client *s3.Client, o *output.Options, + bucket, status string, w io.Writer) error { + if err := client.SetBucketVersioning(ctx, bucket, status); err != nil { + return fmt.Errorf("setting versioning on %q: %w", bucket, err) + } + return o.WriteSingle(w, []string{"Bucket", "Versioning"}, []any{bucket, status}) +} diff --git a/internal/cli/s3/client.go b/internal/cli/s3/client.go index 350c3aa..7ec015d 100644 --- a/internal/cli/s3/client.go +++ b/internal/cli/s3/client.go @@ -1,5 +1,7 @@ -// Package s3cli implements the koc-specific "koc s3 ..." command group: list -// buckets and objects, and move files in and out of an S3-compatible store. +// Package s3cli implements the koc-specific "koc s3 ..." command group: the +// bucket and object lifecycle of an S3-compatible store, and moving data in and +// out of it — including multipart and streamed uploads, recursive and wildcard +// transfers, server-side copy/move, and presigned URLs. // // It has no python-openstackclient equivalent. S3 is not an OpenStack service — // upstream's object-store commands speak Swift, which KeyStack does not deploy — @@ -33,6 +35,13 @@ import ( // the group's Long help for where the other keys live). const defaultCredsSecret = "gitlab-object-storage" //nolint:gosec // G101: name of a k8s Secret object, not a credential value +// defaultRetries is --s3-retries' default. A koc S3 call can be a transfer of +// several gigabytes across a cluster network, where one reset connection should +// not cost the whole command; s5cmd defaults to ten for the same reason, and +// this is lower only because koc's backoff is not jittered across hundreds of +// workers. +const defaultRetries = 5 + // connFlags holds the "koc s3" connection and credential flags. They are // persistent flags on the group rather than global ones: every other koc command // needs Keystone, none of them needs these, and koc --help is long enough. @@ -45,6 +54,8 @@ type connFlags struct { credsFromNS string insecure bool noPathStyle bool + anonymous bool + retries int fs *pflag.FlagSet // to tell "unset" from "explicitly empty" } @@ -72,6 +83,10 @@ func (f *connFlags) addTo(fs *pflag.FlagSet) { "disable TLS verification for the S3 endpoint (env S3_SKIP_VERIFY)") fs.BoolVar(&f.noPathStyle, "no-path-style", false, "address buckets as . instead of /; needs a wildcard DNS record") + fs.BoolVar(&f.anonymous, "s3-anonymous", auth.EnvBool("S3_ANONYMOUS"), + "send requests unsigned, for a publicly readable bucket (env S3_ANONYMOUS); no credentials are needed or used") + fs.IntVar(&f.retries, "s3-retries", defaultRetries, + "retries for a failed request, with exponential backoff (0 = none)") } // client builds the S3 client. --timeout, --debug, --insecure, --kubeconfig and @@ -85,9 +100,14 @@ func (f *connFlags) client(ctx context.Context, a *auth.Options) (*s3.Client, er PathStyle: !f.noPathStyle, // The global --insecure is honoured too: a user who typed it once meant // it for the endpoint they are talking to. - Insecure: f.insecure || a.Insecure, - Timeout: a.Timeout, - Debug: a.Debug, + Insecure: f.insecure || a.Insecure, + Timeout: a.Timeout, + Debug: a.Debug, + Anonymous: f.anonymous, + MaxRetries: f.retries, + } + if f.retries < 0 { + return nil, fmt.Errorf("--s3-retries cannot be negative, got %d", f.retries) } if f.credsFromNS != "" { diff --git a/internal/cli/s3/copy.go b/internal/cli/s3/copy.go new file mode 100644 index 0000000..c07150b --- /dev/null +++ b/internal/cli/s3/copy.go @@ -0,0 +1,300 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// copyFlags holds the options accepted by "copy" and "move". +type copyFlags struct { + recursive bool + dryRun bool + flatten bool + versionID string + contentType string + concurrency int + filter keyFilter + // remove makes this a move: the source is deleted once the copy lands. + remove bool +} + +const copyLong = `Copy objects inside the store, without the bytes travelling through koc. + +The source is named in a header, so the store does the copying: a 100 GiB +object is one small request, at no egress cost and no local disk. That is what +makes this the way to promote, rename or archive a backup — as opposed to a +download followed by an upload, which moves every byte twice through whatever +link koc is on. + +--recursive copies every object under the source key, treated as a prefix, to +the destination key as a prefix; a wildcard source does the same without the +flag. Each object keeps its path relative to the source prefix, or goes straight +under the destination with --flatten. + +--version-id copies one specific version of one object (Garage implements no +versioning).` + +const copyExample = ` # Promote last night's dump to a stable name + koc s3 copy db-backups/nightly-2026-09-13.sql.gz db-backups/latest.sql.gz + + # Archive a month into another bucket + koc s3 copy db-backups/2026/08/ archive/2026-08/ --recursive + + # Everything matching a pattern + koc s3 copy "db-backups/*.sha256" checksums/ --recursive` + +const moveLong = `Move objects inside the store: a server-side copy, then a delete of the source. + +S3 has no atomic move, so this is the two calls in order — the source is only +removed once the copy has landed, and a failure therefore leaves the source +intact rather than losing the object. A move interrupted between the two leaves +both copies, never neither. + +Every flag behaves as for "koc s3 copy".` + +func newCopyCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return newCopyLikeCommand(a, o, f, copySpec{ + use: "copy / /", + short: "Copy objects inside the store (server-side)", + long: copyLong, + example: copyExample, + aliases: []string{"cp"}, + }) +} + +func newMoveCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return newCopyLikeCommand(a, o, f, copySpec{ + use: "move / /", + short: "Move objects inside the store (server-side copy, then delete)", + long: moveLong, + aliases: []string{"mv"}, + remove: true, + }) +} + +// copySpec is the per-verb text of the two commands copy.go builds, which are +// the same command bar the trailing delete. +type copySpec struct { + use, short, long, example string + aliases []string + remove bool +} + +func newCopyLikeCommand(a *auth.Options, o *output.Options, f *connFlags, spec copySpec) *cobra.Command { + cf := ©Flags{remove: spec.remove} + cmd := &cobra.Command{ + Use: spec.use, + Short: spec.short, + Long: spec.long, + Example: spec.example, + Aliases: spec.aliases, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + src, dst, err := parseCopyRefs(args[0], args[1], cf) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runCopy(ctx, client, o, src, dst, cf, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVarP(&cf.recursive, "recursive", "r", false, + "treat the source key as a prefix and copy everything under it") + fl.BoolVar(&cf.dryRun, "dry-run", false, "print what would be copied and copy nothing") + fl.BoolVar(&cf.flatten, "flatten", false, + "put every object directly under the destination prefix instead of mirroring its path") + fl.StringVar(&cf.versionID, "version-id", "", "copy this version of the source instead of the current one") + fl.StringVar(&cf.contentType, "content-type", "", + "Content-Type for the destination; without it the source's metadata is carried over") + fl.IntVar(&cf.concurrency, "concurrency", s3.DefaultConcurrency, + "objects to copy at once under --recursive") + cf.filter.addTo(fl) + return cmd +} + +// parseCopyRefs validates the two references and the flag combination. +func parseCopyRefs(srcRef, dstRef string, f *copyFlags) (src, dst s3.ObjectRef, err error) { + srcBucket, srcKey, err := parseRef(srcRef) + if err != nil { + return src, dst, err + } + dstBucket, dstKey, err := parseRef(dstRef) + if err != nil { + return src, dst, err + } + src = s3.ObjectRef{Bucket: srcBucket, Key: srcKey} + dst = s3.ObjectRef{Bucket: dstBucket, Key: dstKey} + + bulk := f.recursive || hasGlob(srcKey) + switch { + case bulk && f.versionID != "": + return src, dst, errors.New("--version-id names one object; it cannot be combined with --recursive") + case !bulk && srcKey == "": + return src, dst, fmt.Errorf("%q names no object: expected /", srcRef) + case !bulk && f.filter.active(): + return src, dst, errors.New("--include/--exclude filter a listing; pass --recursive or a wildcard source") + case !bulk && f.flatten: + return src, dst, errors.New("--flatten only applies to a recursive copy") + case !bulk && src == dst: + return src, dst, fmt.Errorf("source and destination are the same object (%s)", src) + } + return src, dst, f.filter.compile() +} + +// runCopy is the test seam for "copy" and "move". +func runCopy(ctx context.Context, client *s3.Client, o *output.Options, + src, dst s3.ObjectRef, f *copyFlags, w io.Writer) error { + if f.recursive || hasGlob(src.Key) { + return runCopyRecursive(ctx, client, src, dst, f, w) + } + + // A destination ending in "/" — or naming only a bucket — takes the + // source's basename, the same rule "upload" follows. + if dst.Key == "" || strings.HasSuffix(dst.Key, "/") { + dst.Key += keyBase(src.Key) + } + if f.dryRun { + return sayCopy(w, src, dst, f.remove, true) + } + + if err := copyOne(ctx, client, src, dst, f); err != nil { + return err + } + // One object, one summary row — the per-object progress lines belong to the + // recursive form, where there is more than one to follow. + return o.WriteSingle(w, []string{"Source", "Destination"}, []any{src.String(), dst.String()}) +} + +// runCopyRecursive copies every matching object under a prefix. +func runCopyRecursive(ctx context.Context, client *s3.Client, src, dst s3.ObjectRef, + f *copyFlags, w io.Writer) error { + prefix := globPrefix(src.Key) + var glob *regexp.Regexp + if hasGlob(src.Key) { + var err error + if glob, err = compileGlob(src.Key); err != nil { + return fmt.Errorf("%s: %w", src, err) + } + } + + out := &syncWriter{w: w} + p, workCtx := newPool(ctx, f.concurrency) + taken := 0 + + err := client.ListObjectsFunc(ctx, src.Bucket, s3.ListOptions{Prefix: prefix}, func(obj s3.Object) error { + if glob != nil && !glob.MatchString(obj.Key) { + return nil + } + if !f.filter.match(obj.Key) { + return nil + } + from := s3.ObjectRef{Bucket: src.Bucket, Key: obj.Key} + to := s3.ObjectRef{Bucket: dst.Bucket, Key: copyDestKey(dst.Key, prefix, obj.Key, f.flatten)} + if from == to { + // Copying an object onto itself is refused by S3 unless metadata + // changes, and in a recursive run it is a mistake, not a request. + return nil + } + taken++ + if !p.run(workCtx, func() error { + if f.dryRun { + return sayCopy(out, from, to, f.remove, true) + } + if err := copyOne(workCtx, client, from, to, f); err != nil { + return err + } + return sayCopy(out, from, to, f.remove, false) + }) { + return errStopListing + } + return nil + }) + if waitErr := p.wait(); err == nil || isStopListing(err) { + err = waitErr + } + if err != nil && !isStopListing(err) { + return fmt.Errorf("copying %s: %w", src, err) + } + if taken == 0 { + _, err = fmt.Fprintf(w, "No objects under %s\n", src) + return err + } + return nil +} + +// copyOne performs the copy and, for a move, the delete that follows it. +func copyOne(ctx context.Context, client *s3.Client, src, dst s3.ObjectRef, f *copyFlags) error { + if _, err := client.CopyObject(ctx, src, dst, f.versionID, f.contentType); err != nil { + return err + } + if !f.remove { + return nil + } + // Only now: a delete before the copy has landed would lose the object. + if err := client.DeleteObject(ctx, src.Bucket, src.Key, f.versionID); err != nil { + return fmt.Errorf("copied %s to %s but could not delete the source: %w", src, dst, err) + } + return nil +} + +// copyDestKey maps a source key to its destination key. +func copyDestKey(dstPrefix, srcPrefix, key string, flatten bool) string { + rel := key + if flatten { + rel = keyBase(key) + } else if srcPrefix != "" { + rel = strings.TrimPrefix(strings.TrimPrefix(key, srcPrefix), "/") + } + if rel == "" { + rel = keyBase(key) + } + if dstPrefix == "" { + return rel + } + return strings.TrimSuffix(dstPrefix, "/") + "/" + rel +} + +// keyBase is path.Base for an object key: the part after the last "/". It is +// not filepath.Base, which would split on "\\" on Windows and mangle a key that +// legitimately contains one. +func keyBase(key string) string { + if i := strings.LastIndex(key, "/"); i >= 0 { + return key[i+1:] + } + return key +} + +// sayCopy prints one progress line, naming the operation the flags asked for. +func sayCopy(w io.Writer, src, dst s3.ObjectRef, remove, dryRun bool) error { + var verb string + switch { + case dryRun && remove: + verb = "Would move" + case dryRun: + verb = "Would copy" + case remove: + verb = "Moved" + default: + verb = "Copied" + } + _, err := fmt.Fprintf(w, "%s: %s -> %s\n", verb, src, dst) + return err +} diff --git a/internal/cli/s3/copy_test.go b/internal/cli/s3/copy_test.go new file mode 100644 index 0000000..e43bd2d --- /dev/null +++ b/internal/cli/s3/copy_test.go @@ -0,0 +1,292 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "sort" + "strings" + "testing" + + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +func testCopyFlags() *copyFlags { return ©Flags{concurrency: 1} } + +func TestRunCopySingleObject(t *testing.T) { + var gotPath, gotSource string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotPath, gotSource = r.URL.Path, r.Header.Get("x-amz-copy-source") + _, _ = fmt.Fprint(w, `"e"`) + }) + + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "nightly.sql.gz"} + dst := s3.ObjectRef{Bucket: "db-backups", Key: "latest.sql.gz"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, testCopyFlags(), &buf); err != nil { + t.Fatal(err) + } + if gotPath != "/db-backups/latest.sql.gz" { + t.Errorf("path = %q, want the destination", gotPath) + } + if gotSource != "/db-backups/nightly.sql.gz" { + t.Errorf("copy source = %q", gotSource) + } + if got, want := buf.String(), "db-backups/nightly.sql.gz\ndb-backups/latest.sql.gz\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A destination ending in "/" — or naming only a bucket — takes the source's +// basename, the same rule "upload" follows. +func TestRunCopyDestinationPrefixTakesTheBasename(t *testing.T) { + var gotPath string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = fmt.Fprint(w, `"e"`) + }) + + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "2026/08/a.sql.gz"} + dst := s3.ObjectRef{Bucket: "archive", Key: "restored/"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, testCopyFlags(), &buf); err != nil { + t.Fatal(err) + } + if gotPath != "/archive/restored/a.sql.gz" { + t.Errorf("path = %q, want the basename appended", gotPath) + } +} + +// A move is the copy plus a delete of the source, in that order: the source is +// only removed once the copy has landed. +func TestRunMoveCopiesThenDeletes(t *testing.T) { + var order []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + order = append(order, r.Method+" "+r.URL.Path) + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + _, _ = fmt.Fprint(w, `"e"`) + }) + + f := testCopyFlags() + f.remove = true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "a.sql.gz"} + dst := s3.ObjectRef{Bucket: "archive", Key: "a.sql.gz"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err != nil { + t.Fatal(err) + } + want := "PUT /archive/a.sql.gz,DELETE /db-backups/a.sql.gz" + if got := strings.Join(order, ","); got != want { + t.Errorf("requests = %q, want %q", got, want) + } +} + +// A failed copy must leave the source alone: losing the object is the one +// outcome a move must never have. +func TestRunMoveKeepsTheSourceWhenTheCopyFails(t *testing.T) { + var deleted bool + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + deleted = true + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `AccessDenied`) + }) + + f := testCopyFlags() + f.remove = true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "a.sql.gz"} + dst := s3.ObjectRef{Bucket: "archive", Key: "a.sql.gz"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err == nil { + t.Fatal("a refused copy was reported as a successful move") + } + if deleted { + t.Fatal("the source was deleted even though the copy failed") + } +} + +// A recursive copy mirrors each key's path below the source prefix. +func TestRunCopyRecursive(t *testing.T) { + var copies []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false + 2026/08/a.sql.gz + 2026/09/b.sql.gz + `) + return + } + copies = append(copies, r.Header.Get("x-amz-copy-source")+" -> "+r.URL.Path) + _, _ = fmt.Fprint(w, `"e"`) + }) + + f := testCopyFlags() + f.recursive = true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "2026/"} + dst := s3.ObjectRef{Bucket: "archive", Key: "old/"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err != nil { + t.Fatal(err) + } + sort.Strings(copies) + want := []string{ + "/db-backups/2026/08/a.sql.gz -> /archive/old/08/a.sql.gz", + "/db-backups/2026/09/b.sql.gz -> /archive/old/09/b.sql.gz", + } + if got := strings.Join(copies, "|"); got != strings.Join(want, "|") { + t.Errorf("copies = %q, want %q", got, want) + } +} + +// A wildcard source is its own recursion. +func TestRunCopyWildcardSource(t *testing.T) { + var copies []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false + a.sql.gz + a.sha256 + `) + return + } + copies = append(copies, r.URL.Path) + _, _ = fmt.Fprint(w, `"e"`) + }) + + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "*.sha256"} + dst := s3.ObjectRef{Bucket: "checksums", Key: ""} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, testCopyFlags(), &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(copies, ","), "/checksums/a.sha256"; got != want { + t.Errorf("copies = %q, want only the pattern match", got) + } +} + +// Copying an object onto itself is refused by S3 unless metadata changes, and +// in a recursive run over the same prefix it is a mistake, not a request. +func TestRunCopyRecursiveSkipsIdentityCopies(t *testing.T) { + var copies int + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false + a.sql.gz`) + return + } + copies++ + _, _ = fmt.Fprint(w, `"e"`) + }) + + f := testCopyFlags() + f.recursive = true + var buf bytes.Buffer + ref := s3.ObjectRef{Bucket: "db-backups", Key: ""} + if err := runCopy(context.Background(), client, valueOpts(), ref, ref, f, &buf); err != nil { + t.Fatal(err) + } + if copies != 0 { + t.Errorf("copies = %d, want none — every key maps onto itself", copies) + } + if !strings.Contains(buf.String(), "No objects under") { + t.Errorf("output = %q, want it to say nothing was copied", buf.String()) + } +} + +func TestRunCopyRecursiveDryRun(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if !r.URL.Query().Has("list-type") { + t.Errorf("--dry-run sent %s %s", r.Method, r.URL.Path) + } + _, _ = fmt.Fprint(w, `false + a.sql.gz`) + }) + + f := testCopyFlags() + f.recursive, f.dryRun, f.remove = true, true, true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: ""} + dst := s3.ObjectRef{Bucket: "archive", Key: ""} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err != nil { + t.Fatal(err) + } + want := "Would move: db-backups/a.sql.gz -> archive/a.sql.gz\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestParseCopyRefsValidation(t *testing.T) { + for _, tc := range []struct { + name string + src, dst string + flags copyFlags + want string + }{ + {"no source key", "b", "b/k", copyFlags{}, "names no object"}, + {"same object", "b/k", "b/k", copyFlags{}, "the same object"}, + {"version with recursive", "b/p", "b/q", copyFlags{recursive: true, versionID: "v"}, "--recursive"}, + {"filters without recursion", "b/k", "b/j", + copyFlags{filter: keyFilter{include: []string{"*"}}}, "--recursive"}, + {"flatten without recursion", "b/k", "b/j", copyFlags{flatten: true}, "--flatten"}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := tc.flags + if _, _, err := parseCopyRefs(tc.src, tc.dst, &flags); err == nil || + !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } + + // The s3:// spelling is accepted on both sides. + src, dst, err := parseCopyRefs("s3://a/k", "s3://b/j", ©Flags{}) + if err != nil { + t.Fatal(err) + } + if src.Bucket != "a" || dst.Bucket != "b" { + t.Errorf("refs = %s, %s", src, dst) + } +} + +func TestCopyDestKey(t *testing.T) { + for _, tc := range []struct { + name string + dstPrefix, srcPrefix, key string + flatten bool + want string + }{ + {"mirrors below the prefix", "old/", "2026/", "2026/08/a.gz", false, "old/08/a.gz"}, + {"flattened", "old/", "2026/", "2026/08/a.gz", true, "old/a.gz"}, + {"no destination prefix", "", "2026/", "2026/a.gz", false, "a.gz"}, + {"destination without a slash", "old", "", "a.gz", false, "old/a.gz"}, + {"prefix is not a path boundary", "old/", "nightly-", "nightly-a.gz", false, "old/a.gz"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := copyDestKey(tc.dstPrefix, tc.srcPrefix, tc.key, tc.flatten); got != tc.want { + t.Errorf("copyDestKey = %q, want %q", got, tc.want) + } + }) + } +} + +// keyBase must split on "/" only: a backslash is a legal character in an S3 +// key, and filepath.Base would mangle it on Windows. +func TestKeyBase(t *testing.T) { + for _, tc := range []struct{ key, want string }{ + {"a/b/c.gz", "c.gz"}, + {"c.gz", "c.gz"}, + {`weird\name.gz`, `weird\name.gz`}, + } { + if got := keyBase(tc.key); got != tc.want { + t.Errorf("keyBase(%q) = %q, want %q", tc.key, got, tc.want) + } + } +} diff --git a/internal/cli/s3/download.go b/internal/cli/s3/download.go new file mode 100644 index 0000000..c97eb61 --- /dev/null +++ b/internal/cli/s3/download.go @@ -0,0 +1,339 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// downloadFlags holds the options accepted by "download". +type downloadFlags struct { + force bool + recursive bool + dryRun bool + flatten bool + versionID string + concurrency int + filter keyFilter +} + +// downloadRequest is one "download" invocation: the object or prefix to fetch, +// where to put it, and the flags that govern it. The positional arguments travel +// with the flags because runDownload needs all of them and a parameter list that +// long says nothing about which is which (go:S107). +type downloadRequest struct { + bucket string + // key is one object's key, or — under --recursive or with a wildcard — the + // prefix or glob selecting many. + key string + // dest is the destination path: "" means the key's basename, "-" means + // stream to stdout. Under --recursive it is a directory. + dest string + flags *downloadFlags +} + +const downloadLong = `Download an object, a prefix, or a wildcard match. + +With no FILE the object's key basename is used. FILE "-" streams the object to +stdout as raw bytes and prints nothing else, so it pipes. + +An existing file is never overwritten without --force: these are backups, and a +half-typed key that clobbers the local copy of one is not a recoverable +mistake. A transfer that fails partway removes the partial file rather than +leaving a truncated dump that looks complete. + +--recursive downloads every object under the key, treated as a prefix, into +FILE taken as a directory; a wildcard reference does the same without the flag: + + koc s3 download db-backups/2026/ ./restore --recursive + koc s3 download "db-backups/2026/*.sql.gz" ./restore + +Each object lands at its key's path relative to the prefix, so the layout is +preserved; --flatten puts every object straight in the directory instead, which +fails loudly rather than silently if two keys share a basename. In recursive +mode an existing file is *skipped* with a notice unless --force, so an +interrupted restore resumes by re-running it — unlike the single-object form, +where naming one key and doing nothing would hide the no-op. + +--concurrency downloads that many objects at once, which is most of the reason +a bulk restore finishes: a thousand small objects are a thousand round trips +serially. Progress lines then appear in completion order, not listing order. + +--include/--exclude narrow which keys are taken. --version-id fetches one +specific version of one object (Garage implements no versioning).` + +func newDownloadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + df := &downloadFlags{} + cmd := &cobra.Command{ + Use: "download / [file]", + Short: "Download an object, a prefix, or a wildcard match", + Long: downloadLong, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + req, err := newDownloadRequest(args, df) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runDownload(ctx, client, o, req, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVar(&df.force, "force", false, "overwrite the destination file if it exists") + fl.BoolVarP(&df.recursive, "recursive", "r", false, + "treat the key as a prefix and download everything under it into a directory") + fl.BoolVar(&df.dryRun, "dry-run", false, "print what would be downloaded and transfer nothing") + fl.BoolVar(&df.flatten, "flatten", false, + "put every object directly in the destination directory instead of mirroring its key path") + fl.StringVar(&df.versionID, "version-id", "", "download this version of the object instead of the current one") + fl.IntVar(&df.concurrency, "concurrency", s3.DefaultConcurrency, + "objects to download at once under --recursive") + df.filter.addTo(fl) + return cmd +} + +// newDownloadRequest parses the positional arguments and rejects the flag +// combinations that cannot mean anything. +func newDownloadRequest(args []string, f *downloadFlags) (downloadRequest, error) { + bucket, key, err := parseRef(args[0]) + if err != nil { + return downloadRequest{}, err + } + req := downloadRequest{bucket: bucket, key: key, flags: f} + if len(args) == 2 { + req.dest = args[1] + } + + bulk := f.recursive || hasGlob(key) + switch { + case bulk && req.dest == "-": + return req, errors.New("a recursive download needs a directory, not stdout") + case bulk && f.versionID != "": + return req, errors.New("--version-id names one object; it cannot be combined with --recursive") + case !bulk && key == "": + return req, fmt.Errorf("%q names no object: expected /", args[0]) + case !bulk && f.filter.active(): + return req, errors.New("--include/--exclude filter a listing; pass --recursive or a wildcard reference") + case !bulk && f.flatten: + return req, errors.New("--flatten only applies to a recursive download") + } + return req, f.filter.compile() +} + +// runDownload is the test seam for "download". w receives either the object's +// raw bytes (dest "-"), the per-object progress lines of a recursive run, or the +// summary table of a single one. +func runDownload(ctx context.Context, client *s3.Client, o *output.Options, + r downloadRequest, w io.Writer) error { + if r.flags.recursive || hasGlob(r.key) { + return runDownloadRecursive(ctx, client, r, w) + } + if r.dest == "-" { + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to stdout\n", r.bucket, r.key) + return err + } + if _, err := client.GetObject(ctx, r.bucket, r.key, r.flags.versionID, w); err != nil { + return downloadError(r.bucket, r.key, err) + } + return nil + } + + dest := r.dest + if dest == "" { + dest = filepath.Base(r.key) + } + if info, err := os.Stat(dest); err == nil && info.IsDir() { + dest = filepath.Join(dest, filepath.Base(r.key)) + } + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to %s\n", r.bucket, r.key, dest) + return err + } + + n, err := downloadToFile(ctx, client, r.bucket, r.key, r.flags.versionID, dest, r.flags.force) + if err != nil { + return err + } + return o.WriteSingle(w, + []string{"Bucket", "Key", "File", "Size"}, + []any{r.bucket, r.key, dest, n}) +} + +// runDownloadRecursive downloads every matching object under a prefix. +func runDownloadRecursive(ctx context.Context, client *s3.Client, r downloadRequest, w io.Writer) error { + dir := r.dest + if dir == "" { + dir = "." + } + prefix := globPrefix(r.key) + var glob *regexp.Regexp + if hasGlob(r.key) { + var err error + if glob, err = compileGlob(r.key); err != nil { + return fmt.Errorf("%s/%s: %w", r.bucket, r.key, err) + } + } + + out := &syncWriter{w: w} + p, workCtx := newPool(ctx, r.flags.concurrency) + taken := 0 + + err := client.ListObjectsFunc(ctx, r.bucket, s3.ListOptions{Prefix: prefix}, func(obj s3.Object) error { + if strings.HasSuffix(obj.Key, "/") { + // A "directory marker": a zero-length key ending in "/", which some + // tools create. Mirroring it as a file would shadow the directory. + return nil + } + if glob != nil && !glob.MatchString(obj.Key) { + return nil + } + if !r.flags.filter.match(obj.Key) { + return nil + } + dest, err := destForKey(dir, prefix, obj.Key, r.flags.flatten) + if err != nil { + return err + } + taken++ + if !p.run(workCtx, func() error { + return downloadOne(workCtx, client, r.bucket, obj.Key, dest, r.flags, out) + }) { + return errStopListing + } + return nil + }) + if waitErr := p.wait(); err == nil || isStopListing(err) { + err = waitErr + } + if err != nil && !isStopListing(err) { + return fmt.Errorf("downloading %s/%s: %w", r.bucket, r.key, err) + } + if taken == 0 { + _, err = fmt.Fprintf(w, "No objects under %s/%s\n", r.bucket, r.key) + return err + } + return nil +} + +// downloadOne fetches one object of a recursive run, or says what it would have +// fetched. +func downloadOne(ctx context.Context, client *s3.Client, bucket, key, dest string, + f *downloadFlags, w io.Writer) error { + if f.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to %s\n", bucket, key, dest) + return err + } + // In bulk mode an existing file is a resumption point, not a mistake: a + // restore that was interrupted is finished by re-running the same command. + if !f.force { + if _, err := os.Stat(dest); err == nil { + _, err := fmt.Fprintf(w, "Skipped (exists): %s\n", dest) + return err + } + } + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(dest), err) + } + + n, err := downloadToFile(ctx, client, bucket, key, "", dest, f.force) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "Downloaded: %s/%s -> %s (%d bytes)\n", bucket, key, dest, n) + return err +} + +// destForKey maps one key to its local path, and refuses a key that would +// escape the destination directory. +// +// A key is server-supplied data: "../../etc/cron.d/x", or an absolute one, is a +// perfectly legal S3 key, and a restore that wrote it where it says would let +// whoever can write to the bucket write anywhere koc can. +func destForKey(dir, prefix, key string, flatten bool) (string, error) { + rel := key + if flatten { + rel = filepath.Base(key) + } else if prefix != "" { + rel = strings.TrimPrefix(key, prefix) + // A prefix that is not a path boundary ("e2e-") leaves a bare file + // name, which is what should land in the directory. + rel = strings.TrimPrefix(rel, "/") + } + if rel == "" { + rel = filepath.Base(key) + } + + dest := filepath.Join(dir, filepath.FromSlash(rel)) + clean := filepath.Clean(dir) + if rel := mustRel(clean, dest); rel == "" { + return "", fmt.Errorf("object key %q would be written outside %s; refusing", key, clean) + } + return dest, nil +} + +// mustRel returns the path of target under base, or "" if it is not under it. +func mustRel(base, target string) string { + rel, err := filepath.Rel(base, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "" + } + return rel +} + +// downloadToFile streams the object into dest, leaving nothing behind if it +// fails. +func downloadToFile(ctx context.Context, client *s3.Client, bucket, key, versionID, dest string, + force bool) (n int64, err error) { + flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL + if force { + flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC + } + out, err := os.OpenFile(dest, flags, 0o600) //nolint:gosec // G304: operator-supplied output path + if err != nil { + if errors.Is(err, os.ErrExist) { + return 0, fmt.Errorf("%s already exists (pass --force to overwrite)", dest) + } + return 0, fmt.Errorf("creating %q: %w", dest, err) + } + defer func() { + closeErr := out.Close() + if err == nil && closeErr != nil { + err = fmt.Errorf("closing %q: %w", dest, closeErr) + } + if err != nil { + _ = os.Remove(dest) + } + }() + + if n, err = client.GetObject(ctx, bucket, key, versionID, out); err != nil { + return 0, downloadError(bucket, key, err) + } + return n, nil +} + +// downloadError turns a missing key into the message an operator can act on. +func downloadError(bucket, key string, err error) error { + if s3.IsNotFound(err) { + return fmt.Errorf("no object %q in bucket %q", key, bucket) + } + return fmt.Errorf("downloading %s/%s: %w", bucket, key, err) +} diff --git a/internal/cli/s3/du.go b/internal/cli/s3/du.go index ac4f6e6..b078006 100644 --- a/internal/cli/s3/du.go +++ b/internal/cli/s3/du.go @@ -15,7 +15,10 @@ import ( // duFlags holds the options accepted by "du". type duFlags struct { - group bool + group bool + human bool + allVersions bool + filter keyFilter } const duLong = `Total the size of a bucket, or of one prefix in it. @@ -26,20 +29,27 @@ request per 1000 of them. The listing streams, so the size of the bucket does not become the size of koc's memory. Sizes are exact bytes, like the rest of koc, so they stay usable in ---format value/csv and in arithmetic. +--format value/csv and in arithmetic; --human renders them for a reader. --group reports one row per storage class instead of one total, which is what tells a mixed bucket's cheap tier from its expensive one. Objects a store -reports with no storage class are grouped as STANDARD, which is what it means.` +reports with no storage class are grouped as STANDARD, which is what it means — +Garage reports every object that way, since it implements no storage classes. + +--all-versions counts every version and delete marker, which is the only way to +see what an old version still costs on a store that has versioning.` const duExample = ` # A whole bucket koc s3 du db-backups - # Just the end-to-end test dumps - koc s3 du db-backups/e2e- + # Just the end-to-end test dumps, for a reader + koc s3 du db-backups/e2e- --human # Per storage class - koc s3 du db-backups --group` + koc s3 du db-backups --group + + # Only the compressed dumps + koc s3 du db-backups --include "*.sql.gz"` func newDuCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { df := &duFlags{} @@ -58,6 +68,9 @@ func newDuCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Comma if err != nil { return err } + if err := df.filter.compile(); err != nil { + return err + } ctx := cmd.Context() client, err := f.client(ctx, a) if err != nil { @@ -66,7 +79,11 @@ func newDuCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Comma return runDu(ctx, client, o, bucket, prefix, df, cmd.OutOrStdout()) }, } - cmd.Flags().BoolVar(&df.group, "group", false, "one row per storage class instead of one total") + fl := cmd.Flags() + fl.BoolVar(&df.group, "group", false, "one row per storage class instead of one total") + fl.BoolVar(&df.human, "human", false, "render sizes for a reader (KiB/MiB/GiB) instead of exact bytes") + fl.BoolVar(&df.allVersions, "all-versions", false, "count every version and delete marker") + df.filter.addTo(fl) return cmd } @@ -82,7 +99,11 @@ func runDu(ctx context.Context, client *s3.Client, o *output.Options, var total usage byClass := map[string]*usage{} - err := client.ListObjectsFunc(ctx, bucket, prefix, 0, func(obj s3.Object) error { + opts := s3.ListOptions{Prefix: prefix, Versions: f.allVersions} + err := client.ListObjectsFunc(ctx, bucket, opts, func(obj s3.Object) error { + if !f.filter.match(obj.Key) { + return nil + } total.objects++ total.bytes += obj.Size if f.group { @@ -105,16 +126,16 @@ func runDu(ctx context.Context, client *s3.Client, o *output.Options, } if f.group { - return writeUsageByClass(w, o, byClass) + return writeUsageByClass(w, o, byClass, f.human) } return o.WriteSingle(w, []string{"Bucket", "Prefix", "Objects", "Size"}, - []any{bucket, prefix, total.objects, total.bytes}) + []any{bucket, prefix, total.objects, renderSize(total.bytes, f.human)}) } // writeUsageByClass renders --group's per-storage-class rows, ordered by class // name so two runs of the same bucket diff cleanly. -func writeUsageByClass(w io.Writer, o *output.Options, byClass map[string]*usage) error { +func writeUsageByClass(w io.Writer, o *output.Options, byClass map[string]*usage, human bool) error { classes := make([]string, 0, len(byClass)) for class := range byClass { classes = append(classes, class) @@ -123,10 +144,19 @@ func writeUsageByClass(w io.Writer, o *output.Options, byClass map[string]*usage rows := make([][]any, len(classes)) for i, class := range classes { - rows[i] = []any{class, byClass[class].objects, byClass[class].bytes} + rows[i] = []any{class, byClass[class].objects, renderSize(byClass[class].bytes, human)} } return o.WriteList(w, output.Table{ Columns: []string{"Storage Class", "Objects", "Size"}, Rows: rows, }) } + +// renderSize is the one place --human is applied, so every size column in this +// group reads the same way. +func renderSize(n int64, human bool) any { + if human { + return output.HumanBytes(n) + } + return n +} diff --git a/internal/cli/s3/filter.go b/internal/cli/s3/filter.go new file mode 100644 index 0000000..641b17b --- /dev/null +++ b/internal/cli/s3/filter.go @@ -0,0 +1,124 @@ +package s3cli + +import ( + "fmt" + "regexp" + "strings" + + "github.com/spf13/pflag" +) + +// keyFilter is the --include/--exclude pair shared by every command that walks +// a listing. Both take glob patterns and both may be repeated: +// +// - with no --include, every key is a candidate; +// - with any --include, a key must match at least one of them; +// - --exclude then removes a key whatever --include said, because "all the +// dumps except the end-to-end ones" is the shape an operator actually +// needs. +type keyFilter struct { + include []string + exclude []string + + compiled struct { + include []*regexp.Regexp + exclude []*regexp.Regexp + } +} + +// addTo registers the flags. Every command that filters a listing uses the same +// two names and the same help, so an operator learns them once. +func (f *keyFilter) addTo(fs *pflag.FlagSet) { + fs.StringArrayVar(&f.include, "include", nil, + `only keys matching this glob, repeatable (e.g. --include "*.sql.gz")`) + fs.StringArrayVar(&f.exclude, "exclude", nil, + `skip keys matching this glob, repeatable; applied after --include`) +} + +// compile turns the patterns into matchers, rejecting a malformed one before any +// request is made rather than silently matching nothing. +func (f *keyFilter) compile() error { + var err error + if f.compiled.include, err = compileGlobs(f.include, "--include"); err != nil { + return err + } + f.compiled.exclude, err = compileGlobs(f.exclude, "--exclude") + return err +} + +// active reports whether any pattern was given, so a caller can skip the whole +// filtering path (and its per-key cost) when none was. +func (f *keyFilter) active() bool { return len(f.include) > 0 || len(f.exclude) > 0 } + +// match reports whether key survives the filter. +func (f *keyFilter) match(key string) bool { + if len(f.compiled.include) > 0 && !matchesAny(f.compiled.include, key) { + return false + } + return !matchesAny(f.compiled.exclude, key) +} + +func matchesAny(pats []*regexp.Regexp, key string) bool { + for _, p := range pats { + if p.MatchString(key) { + return true + } + } + return false +} + +func compileGlobs(pats []string, flag string) ([]*regexp.Regexp, error) { + out := make([]*regexp.Regexp, 0, len(pats)) + for _, p := range pats { + re, err := compileGlob(p) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", flag, p, err) + } + out = append(out, re) + } + return out, nil +} + +// compileGlob translates a glob to an anchored regexp. +// +// path.Match is not used: its "*" stops at a "/", which is right for a +// filesystem and wrong for an S3 keyspace, where the slashes in +// "2026/08/dump.sql.gz" are ordinary characters and --include "*.sql.gz" is +// expected to reach them. This matches s5cmd's wildcards, where "*" spans +// separators and "?" is one character. +// +// Only those two are special. Everything else — a dot, a bracket, a brace — is +// quoted to a literal, so a key containing one names itself rather than turning +// into a character class the operator did not write. +func compileGlob(pattern string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for _, r := range pattern { + switch r { + case '*': + b.WriteString(".*") + case '?': + b.WriteString(".") + default: + b.WriteString(regexp.QuoteMeta(string(r))) + } + } + b.WriteString("$") + return regexp.Compile(b.String()) +} + +// hasGlob reports whether ref carries wildcard characters, i.e. whether it +// names a set of keys rather than one. +func hasGlob(ref string) bool { + return strings.ContainsAny(ref, "*?") +} + +// globPrefix is the literal part of a glob before its first wildcard, which is +// what the server can filter on: "2026/08/*.gz" lists under "2026/08/" and the +// pattern then narrows the result locally. +func globPrefix(pattern string) string { + if i := strings.IndexAny(pattern, "*?"); i >= 0 { + return pattern[:i] + } + return pattern +} diff --git a/internal/cli/s3/filter_test.go b/internal/cli/s3/filter_test.go new file mode 100644 index 0000000..84fb9a2 --- /dev/null +++ b/internal/cli/s3/filter_test.go @@ -0,0 +1,103 @@ +package s3cli + +import "testing" + +// An S3 keyspace is flat: the slashes in "2026/08/dump.sql.gz" are ordinary +// characters, so "*" has to cross them. path.Match would not, which is why this +// translates to a regexp instead. +func TestCompileGlobSpansSlashes(t *testing.T) { + for _, tc := range []struct { + pattern, key string + want bool + }{ + {"*.sql.gz", "dump.sql.gz", true}, + {"*.sql.gz", "2026/08/dump.sql.gz", true}, + {"2026/*", "2026/08/dump.sql.gz", true}, + {"2026/*", "2025/08/dump.sql.gz", false}, + {"dump-?.gz", "dump-1.gz", true}, + {"dump-?.gz", "dump-12.gz", false}, + {"dump.sql.gz", "dump.sql.gz", true}, + {"dump.sql.gz", "xdump.sql.gz", false}, + // A dot is a literal, not "any character" — a glob is not a regexp. + {"a.c", "abc", false}, + {"a.c", "a.c", true}, + } { + re, err := compileGlob(tc.pattern) + if err != nil { + t.Fatalf("compileGlob(%q) = %v", tc.pattern, err) + } + if got := re.MatchString(tc.key); got != tc.want { + t.Errorf("%q matches %q = %v, want %v", tc.pattern, tc.key, got, tc.want) + } + } +} + +func TestKeyFilter(t *testing.T) { + for _, tc := range []struct { + name string + include, exclude []string + key string + want bool + }{ + {"no patterns takes everything", nil, nil, "anything", true}, + {"include must match", []string{"*.gz"}, nil, "dump.gz", true}, + {"include that misses drops the key", []string{"*.gz"}, nil, "dump.txt", false}, + {"any include is enough", []string{"*.gz", "*.txt"}, nil, "dump.txt", true}, + {"exclude wins over include", []string{"*"}, []string{"e2e-*"}, "e2e-a.gz", false}, + {"exclude alone", nil, []string{"e2e-*"}, "nightly.gz", true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := keyFilter{include: tc.include, exclude: tc.exclude} + if err := f.compile(); err != nil { + t.Fatal(err) + } + if got := f.match(tc.key); got != tc.want { + t.Errorf("match(%q) = %v, want %v", tc.key, got, tc.want) + } + }) + } +} + +// Only "*" and "?" are special, matching s5cmd: a bracket, a brace or a dot in +// a key is a literal, so "[2026]/dump" names exactly that key rather than a +// character class over one of four digits. +func TestCompileGlobTreatsEverythingElseAsLiteral(t *testing.T) { + re, err := compileGlob("[2026]/dump.gz") + if err != nil { + t.Fatal(err) + } + if !re.MatchString("[2026]/dump.gz") { + t.Error("a bracketed key did not match itself") + } + if re.MatchString("2/dump.gz") { + t.Error("brackets were treated as a character class") + } +} + +func TestGlobPrefix(t *testing.T) { + for _, tc := range []struct{ pattern, want string }{ + {"2026/08/*.gz", "2026/08/"}, + {"*.gz", ""}, + {"dump.sql.gz", "dump.sql.gz"}, + {"dump-?.gz", "dump-"}, + } { + if got := globPrefix(tc.pattern); got != tc.want { + t.Errorf("globPrefix(%q) = %q, want %q", tc.pattern, got, tc.want) + } + } +} + +func TestHasGlob(t *testing.T) { + for _, tc := range []struct { + ref string + want bool + }{ + {"b/k", false}, + {"b/*.gz", true}, + {"b/dump-?.gz", true}, + } { + if got := hasGlob(tc.ref); got != tc.want { + t.Errorf("hasGlob(%q) = %v, want %v", tc.ref, got, tc.want) + } + } +} diff --git a/internal/cli/s3/flags_test.go b/internal/cli/s3/flags_test.go new file mode 100644 index 0000000..bd2454d --- /dev/null +++ b/internal/cli/s3/flags_test.go @@ -0,0 +1,311 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/ftarasenko/go-openstackclient/internal/output" +) + +// --delimiter turns a flat keyspace into one directory level: the subtrees come +// first, with no size, so a table never claims an empty object is there. +func TestRunObjectListDelimiter(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("delimiter"); got != "/" { + t.Errorf("delimiter = %q, want /", got) + } + _, _ = fmt.Fprint(w, `false + 2026/ + latest.sql.gz7 + 2026-09-14T10:00:00.000Z"e" + `) + }) + + var buf bytes.Buffer + err := runObjectList(context.Background(), client, valueOpts(), "db-backups", + &objectListFlags{delimiter: "/"}, &buf) + if err != nil { + t.Fatal(err) + } + want := "2026/\t\t\t\nlatest.sql.gz\t7\t2026-09-14T10:00:00Z\te\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestRunObjectListHuman(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + big.sql.gz15252880 + `) + }) + + var buf bytes.Buffer + err := runObjectList(context.Background(), client, valueOpts(), "db-backups", + &objectListFlags{human: true}, &buf) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "14.5 MiB") { + t.Errorf("output = %q, want a human-readable size", buf.String()) + } +} + +// A local filter has to be applied before --limit is counted, or a bucket whose +// first keys are all excluded would answer an empty list under "--limit 1". +func TestRunObjectListFilterThenLimit(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + // With filtering on, the cap moves to koc, so the server must not be + // asked for a short page. + if got := r.URL.Query().Get("max-keys"); got != "1000" { + t.Errorf("max-keys = %q, want a full page when filtering locally", got) + } + _, _ = fmt.Fprint(w, `false + e2e-a.gz1 + nightly-a.gz2 + nightly-b.gz3 + `) + }) + + f := &objectListFlags{limit: 1, filter: keyFilter{include: []string{"nightly-*"}}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runObjectList(context.Background(), client, valueOpts(), "db-backups", f, &buf); err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "nightly-a.gz\t2\t\t\n"; got != want { + t.Errorf("output = %q, want the first matching key only", got) + } +} + +// --all-versions adds two columns, so the extra fields are not silently dropped +// in --format value. +func TestRunObjectListAllVersions(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if !r.URL.Query().Has("versions") { + t.Error("the request did not carry ?versions") + } + _, _ = fmt.Fprint(w, `false + dump.gzv2true + 92026-09-14T10:00:00.000Z + gone.gzv1 + 2026-09-14T09:00:00.000Z + `) + }) + + var buf bytes.Buffer + err := runObjectList(context.Background(), client, valueOpts(), "b", + &objectListFlags{allVersions: true}, &buf) + if err != nil { + t.Fatal(err) + } + out := buf.String() + if !strings.Contains(out, "v2") || !strings.Contains(out, "true") { + t.Errorf("output %q does not carry the version columns", out) + } + if !strings.Contains(out, "v1 (delete marker)") { + t.Errorf("output %q does not mark the delete marker", out) + } +} + +func TestRunDuHuman(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + err := runDu(context.Background(), client, valueOpts(), "db-backups", "", &duFlags{human: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "db-backups\n\n3\n123 B\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestRunDuFilter(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + a.sql.gz100 + a.sha2567 + `) + }) + + f := &duFlags{filter: keyFilter{include: []string{"*.sql.gz"}}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runDu(context.Background(), client, valueOpts(), "db-backups", "", f, &buf); err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "db-backups\n\n1\n100\n"; got != want { + t.Errorf("output = %q, want only the filtered object counted", got) + } +} + +func TestRunBucketShow(t *testing.T) { + var methods []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + if r.URL.Query().Has("versioning") { + _, _ = fmt.Fprint(w, `Enabled`) + return + } + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketShow(context.Background(), client, valueOpts(), "db-backups", &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(methods, ","), "HEAD,GET"; got != want { + t.Errorf("requests = %q, want a HEAD then the versioning read", got) + } + out := buf.String() + if !strings.Contains(out, "db-backups") || !strings.Contains(out, "Enabled") { + t.Errorf("output = %q", out) + } +} + +// The bucket being there is the answer the command was asked for; a store that +// does not implement the versioning call must not turn that into a failure. +func TestRunBucketShowToleratesNoVersioningSupport(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("versioning") { + w.WriteHeader(http.StatusNotImplemented) + return + } + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketShow(context.Background(), client, valueOpts(), "db-backups", &buf); err != nil { + t.Fatalf("a store without versioning failed the command: %v", err) + } + if !strings.Contains(buf.String(), "unknown") { + t.Errorf("output = %q, want the versioning field unknown", buf.String()) + } +} + +func TestRunBucketShowMissingBucket(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + var buf bytes.Buffer + err := runBucketShow(context.Background(), client, valueOpts(), "absent", &buf) + if err == nil { + t.Fatal("a missing bucket was reported as present") + } + if !strings.Contains(err.Error(), "absent") { + t.Errorf("error %q does not name the bucket", err) + } +} + +func TestRunBucketSet(t *testing.T) { + var gotMethod string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketSet(context.Background(), client, valueOpts(), "b", "Enabled", &buf); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut { + t.Errorf("method = %q, want PUT", gotMethod) + } + if got, want := buf.String(), "b\nEnabled\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// The flag takes either spelling, so "--versioning Enabled" is not a puzzling +// error — and anything else is refused by name. +func TestVersioningStatus(t *testing.T) { + for _, in := range []string{"enabled", "Enabled", "ENABLED"} { + if got, err := versioningStatus(in); err != nil || got != "Enabled" { + t.Errorf("versioningStatus(%q) = %q, %v", in, got, err) + } + } + if got, err := versioningStatus("suspended"); err != nil || got != "Suspended" { + t.Errorf("versioningStatus(suspended) = %q, %v", got, err) + } + if _, err := versioningStatus("unversioned"); err == nil { + t.Error("unversioned was accepted as a settable state") + } +} + +// Presigning makes no request at all, which is why runPresign takes no context. +func TestRunPresign(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("presigning sent %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + err := runPresign(client, valueOpts(), "db-backups", "dump.sql.gz", "", "get", time.Hour, &buf) + if err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{"X-Amz-Signature=", "X-Amz-Expires=3600", "/db-backups/dump.sql.gz"} { + if !strings.Contains(out, want) { + t.Errorf("output %q missing %q", out, want) + } + } +} + +func TestRunPresignPut(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + var buf bytes.Buffer + if err := runPresign(client, valueOpts(), "b", "k", "", "put", time.Minute, &buf); err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "put") { + t.Errorf("output %q does not name the method", buf.String()) + } +} + +func TestRunPresignRejectsBadInput(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + var buf bytes.Buffer + if err := runPresign(client, valueOpts(), "b", "k", "", "delete", time.Hour, &buf); err == nil || + !strings.Contains(err.Error(), "get or put") { + t.Errorf("err = %v, want it to name the allowed methods", err) + } + // A version ID addresses an object that exists; presigning an upload to one + // is a contradiction. + if err := runPresign(client, valueOpts(), "b", "k", "v1", "put", time.Hour, &buf); err == nil { + t.Error("--version-id with --method put was accepted") + } +} + +func TestHumanBytes(t *testing.T) { + for _, tc := range []struct { + n int64 + want string + }{ + {0, "0 B"}, + {1023, "1023 B"}, + {1024, "1.00 KiB"}, + {1536, "1.50 KiB"}, + {15252880, "14.5 MiB"}, + {1 << 30, "1.00 GiB"}, + {200 << 30, "200 GiB"}, + {-2048, "-2.00 KiB"}, + } { + if got := output.HumanBytes(tc.n); got != tc.want { + t.Errorf("HumanBytes(%d) = %q, want %q", tc.n, got, tc.want) + } + } +} diff --git a/internal/cli/s3/object.go b/internal/cli/s3/object.go index 3fdcbb7..472d6f9 100644 --- a/internal/cli/s3/object.go +++ b/internal/cli/s3/object.go @@ -28,23 +28,55 @@ func newObjectCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.C // objectListFlags holds the options accepted by "object list". type objectListFlags struct { - prefix string - limit int + prefix string + delimiter string + limit int + human bool + allVersions bool + filter keyFilter } const objectListLong = `List the objects in a bucket. -Sizes are exact bytes so they stay usable in scripts and in --format value/csv. ---limit is a hard cap on the result, not a page size: listing pages until the -cap is reached and stops there.` +Sizes are exact bytes so they stay usable in scripts and in --format value/csv; +--human renders them for a reader instead. --limit is a hard cap on the result, +not a page size: listing pages until the cap is reached and stops there. + +A bucket is a flat keyspace, so a deep one lists every key at once. +--delimiter / collapses everything below each "directory" into a single entry +marked with a trailing separator, which is how to walk a big bucket one level +at a time: + + koc s3 object list db-backups --delimiter / + koc s3 object list db-backups/2026/ --delimiter / + +--include and --exclude filter the keys locally after the server has listed +them, so they cost no extra requests but do not reduce the listing itself. + +--all-versions lists every version and delete marker rather than the current +object. Garage does not implement versioning — it reports every bucket as +unversioned — so this is for a koc pointed at AWS, Ceph RGW or MinIO.` + +const objectListExample = ` # Everything, exact bytes + koc s3 object list db-backups + + # One level, like a directory listing + koc s3 object list db-backups --delimiter / + + # Just the compressed dumps, sizes for a human + koc s3 object list db-backups --include "*.sql.gz" --human + + # Everything but the end-to-end test dumps + koc s3 object list db-backups --exclude "e2e-*"` func newObjectListCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { lf := &objectListFlags{} cmd := &cobra.Command{ - Use: "list ", - Short: "List objects in a bucket", - Long: objectListLong, - Args: cobra.ExactArgs(1), + Use: "list [/]", + Short: "List objects in a bucket", + Long: objectListLong, + Example: objectListExample, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if err := o.Validate(); err != nil { return err @@ -58,6 +90,9 @@ func newObjectListCommand(a *auth.Options, o *output.Options, f *connFlags) *cob if lf.prefix == "" { lf.prefix = prefix } + if err := lf.filter.compile(); err != nil { + return err + } ctx := cmd.Context() client, err := f.client(ctx, a) if err != nil { @@ -68,36 +103,97 @@ func newObjectListCommand(a *auth.Options, o *output.Options, f *connFlags) *cob } fl := cmd.Flags() fl.StringVar(&lf.prefix, "prefix", "", "only list keys starting with this prefix") + fl.StringVar(&lf.delimiter, "delimiter", "", + `collapse keys below each occurrence of this separator into one entry (usually "/")`) fl.IntVar(&lf.limit, "limit", 0, "maximum number of objects to return (0 = no limit)") + fl.BoolVar(&lf.human, "human", false, "render sizes for a reader (KiB/MiB/GiB) instead of exact bytes") + fl.BoolVar(&lf.allVersions, "all-versions", false, + "list every version and delete marker, not just the current object") + lf.filter.addTo(fl) return cmd } // runObjectList is the test seam for "object list". func runObjectList(ctx context.Context, client *s3.Client, o *output.Options, bucket string, f *objectListFlags, w io.Writer) error { - objects, err := client.ListObjects(ctx, bucket, f.prefix, f.limit) - if err != nil { + opts := s3.ListOptions{ + Prefix: f.prefix, + Delimiter: f.delimiter, + Limit: f.limit, + Versions: f.allVersions, + } + // A local filter has to be applied before --limit is counted, or a bucket + // whose first 100 keys are all excluded would answer an empty list under + // "--limit 10". So the cap moves here whenever filtering is on. + if f.filter.active() { + opts.Limit = 0 + } + + var rows [][]any + err := client.ListObjectsFunc(ctx, bucket, opts, func(obj s3.Object) error { + // A CommonPrefixes entry has no size or timestamp; it is a subtree. + if !obj.IsPrefix && !f.filter.match(obj.Key) { + return nil + } + rows = append(rows, objectRow(obj, f)) + if f.filter.active() && f.limit > 0 && len(rows) >= f.limit { + return errStopListing + } + return nil + }) + if err != nil && !isStopListing(err) { return fmt.Errorf("listing objects in %s: %w", bucket, err) } - rows := make([][]any, len(objects)) - for i, obj := range objects { - rows[i] = []any{obj.Key, obj.Size, formatTime(obj.LastModified), obj.ETag} + return o.WriteList(w, output.Table{Columns: objectListColumns(f), Rows: rows}) +} + +// objectListColumns is the header row, which grows by two under --all-versions +// so the extra fields are not silently dropped in --format value. +func objectListColumns(f *objectListFlags) []string { + cols := []string{"Key", "Size", "Last Modified", "ETag"} + if f.allVersions { + cols = append(cols, "Version ID", "Latest") + } + return cols +} + +// objectRow renders one listing entry. +func objectRow(obj s3.Object, f *objectListFlags) []any { + key, size := obj.Key, any(obj.Size) + switch { + case obj.IsPrefix: + // A subtree has no size of its own; saying "0" would read as an empty + // object that is really there. + size = "" + case f.human: + size = output.HumanBytes(obj.Size) } - return o.WriteList(w, output.Table{ - Columns: []string{"Key", "Size", "Last Modified", "ETag"}, - Rows: rows, - }) + + row := []any{key, size, formatTime(obj.LastModified), obj.ETag} + if f.allVersions { + version := obj.VersionID + if obj.DeleteMarker { + version += " (delete marker)" + } + row = append(row, version, obj.IsLatest) + } + return row } const objectShowLong = `Show one object's metadata, without downloading it. This is the equivalent of the "s3cmd info" call the backup pipeline makes to verify an upload: it is a HEAD request, so the object's body is never -transferred.` +transferred. + +--version-id addresses one specific version on a store that has versioning +(Garage does not).` func newObjectShowCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { - return &cobra.Command{ + var versionID string + var human bool + cmd := &cobra.Command{ Use: "show /", Short: "Show an object's metadata", Long: objectShowLong, @@ -115,15 +211,18 @@ func newObjectShowCommand(a *auth.Options, o *output.Options, f *connFlags) *cob if err != nil { return err } - return runObjectShow(ctx, client, o, bucket, key, cmd.OutOrStdout()) + return runObjectShow(ctx, client, o, bucket, key, versionID, human, cmd.OutOrStdout()) }, } + cmd.Flags().StringVar(&versionID, "version-id", "", "address this version of the object instead of the current one") + cmd.Flags().BoolVar(&human, "human", false, "render the size for a reader instead of exact bytes") + return cmd } // runObjectShow is the test seam for "object show". func runObjectShow(ctx context.Context, client *s3.Client, o *output.Options, - bucket, key string, w io.Writer) error { - info, err := client.HeadObject(ctx, bucket, key) + bucket, key, versionID string, human bool, w io.Writer) error { + info, err := client.HeadObject(ctx, bucket, key, versionID) if err != nil { if s3.IsNotFound(err) { return fmt.Errorf("no object %q in bucket %q", key, bucket) @@ -131,8 +230,16 @@ func runObjectShow(ctx context.Context, client *s3.Client, o *output.Options, return fmt.Errorf("reading %s/%s: %w", bucket, key, err) } + size := any(info.Size) + if human { + size = output.HumanBytes(info.Size) + } fields := []string{"Bucket", "Key", "Size", "Last Modified", "ETag", "Content Type"} - values := []any{info.Bucket, info.Key, info.Size, formatTime(info.LastModified), info.ETag, info.ContentType} + values := []any{info.Bucket, info.Key, size, formatTime(info.LastModified), info.ETag, info.ContentType} + if info.VersionID != "" { + fields = append(fields, "Version ID") + values = append(values, info.VersionID) + } metaKeys := make([]string, 0, len(info.Metadata)) for k := range info.Metadata { diff --git a/internal/cli/s3/object_delete.go b/internal/cli/s3/object_delete.go index b7c8b21..6822d1e 100644 --- a/internal/cli/s3/object_delete.go +++ b/internal/cli/s3/object_delete.go @@ -2,8 +2,10 @@ package s3cli import ( "context" + "errors" "fmt" "io" + "regexp" "github.com/spf13/cobra" @@ -15,8 +17,11 @@ import ( // objectDeleteFlags holds the options accepted by "object delete". type objectDeleteFlags struct { - recursive bool - dryRun bool + recursive bool + dryRun bool + versionID string + allVersions bool + filter keyFilter } const objectDeleteLong = `Delete objects. @@ -37,8 +42,19 @@ under it, which is also how a bucket is emptied before "koc s3 bucket delete": koc s3 object delete db-backups/ --recursive # the whole bucket That is the one destructive shape in this group, so it is spelled out rather -than inferred from a trailing slash or a wildcard, and --dry-run prints exactly -the keys it would delete without touching any of them.` +than inferred from a trailing slash, and --dry-run prints exactly the keys it +would delete without touching any of them. A recursive delete also accepts a +wildcard instead of --recursive: + + koc s3 object delete "db-backups/2026/*.sql.gz" + +A recursive delete goes out in batches of up to 1000 keys per request rather +than one request per key, so emptying a large bucket costs a thousandth of the +round trips. --include and --exclude narrow which keys it takes. + +--version-id deletes one specific version; --all-versions removes every version +and delete marker under the prefix. Garage implements no versioning, so both are +for a koc pointed at AWS, Ceph RGW or MinIO.` func newObjectDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { df := &objectDeleteFlags{} @@ -52,6 +68,9 @@ func newObjectDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *c if err := o.Validate(); err != nil { return err } + if err := df.validate(args); err != nil { + return err + } ctx := cmd.Context() client, err := f.client(ctx, a) if err != nil { @@ -65,63 +84,185 @@ func newObjectDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *c "treat each key as a prefix and delete every object under it") fl.BoolVar(&df.dryRun, "dry-run", false, "print the objects that would be deleted and delete nothing") + fl.StringVar(&df.versionID, "version-id", "", + "delete this version of the object instead of the current one") + fl.BoolVar(&df.allVersions, "all-versions", false, + "delete every version and delete marker, not just the current object") + df.filter.addTo(fl) return cmd } +// validate rejects the flag combinations that cannot mean anything, and +// compiles the globs so a bad pattern fails before a single key is removed. +func (f *objectDeleteFlags) validate(refs []string) error { + if f.versionID != "" { + switch { + case f.recursive: + return errors.New("--version-id names one object; it cannot be combined with --recursive") + case f.allVersions: + return errors.New("--version-id and --all-versions are mutually exclusive") + case len(refs) > 1: + return errors.New("--version-id names one object; pass a single reference") + } + } + if !f.recursive && !f.allVersions && f.filter.active() { + for _, ref := range refs { + if hasGlob(ref) { + return nil + } + } + return errors.New("--include/--exclude filter a listing; pass --recursive or a wildcard reference") + } + return f.filter.compile() +} + // runObjectDelete is the test seam for "object delete". func runObjectDelete(ctx context.Context, client *s3.Client, refs []string, f *objectDeleteFlags, w io.Writer) error { return batchdelete.Each(refs, func(ref string) error { - if f.recursive { + // A wildcard reference is a recursive delete that names its own + // pattern, so "delete 'b/2026/*.gz'" needs no second flag to say so. + if f.recursive || f.allVersions || hasGlob(ref) { // A bare bucket is a legal prefix ref here — "/" is how the // whole bucket is named — so parseRef, not parseObjectRef. - bucket, prefix, err := parseRef(ref) + bucket, pattern, err := parseRef(ref) if err != nil { return err } - return deletePrefix(ctx, client, bucket, prefix, f.dryRun, w) + return deletePrefix(ctx, client, bucket, pattern, f, w) } bucket, key, err := parseObjectRef(ref) if err != nil { return err } - return deleteOne(ctx, client, bucket, key, f.dryRun, w) + if f.dryRun { + return sayWouldDelete(w, bucket, key, f.versionID) + } + if err := client.DeleteObject(ctx, bucket, key, f.versionID); err != nil { + return fmt.Errorf("deleting %s: %w", describeTarget(bucket, key, f.versionID), err) + } + return sayDeleted(w, bucket, key, f.versionID) }) } -// deletePrefix deletes every object under prefix, one signed DELETE per key. +// deletePrefix deletes every object under a prefix (or matching a wildcard), +// batching the keys so a large bucket does not cost one round trip per object. // -// Keys are deleted as the listing streams rather than after collecting it, so -// emptying a bucket costs one page of memory whatever its size. There is no -// batch DeleteObjects call: it would cut the request count, but it needs a -// Content-MD5 over a hand-built XML body and reports per-key failures in a -// 200 response, and the buckets koc is aimed at hold backups in the dozens. -func deletePrefix(ctx context.Context, client *s3.Client, bucket, prefix string, - dryRun bool, w io.Writer) error { - seen := 0 - err := client.ListObjectsFunc(ctx, bucket, prefix, 0, func(obj s3.Object) error { +// Keys are collected a batch at a time as the listing streams rather than all +// at once, so emptying a bucket of a million objects costs one batch of memory +// rather than all of them. +func deletePrefix(ctx context.Context, client *s3.Client, bucket, pattern string, + f *objectDeleteFlags, w io.Writer) error { + var ( + pending []s3.DeleteTarget + seen int + ) + // The literal head of a wildcard is what the server can filter on; the rest + // of the pattern narrows the result here, against a matcher compiled once + // rather than per key. + prefix := globPrefix(pattern) + var glob *regexp.Regexp + if hasGlob(pattern) { + var err error + if glob, err = compileGlob(pattern); err != nil { + return fmt.Errorf("%s/%s: %w", bucket, pattern, err) + } + } + match := func(key string) bool { + if glob != nil && !glob.MatchString(key) { + return false + } + return f.filter.match(key) + } + + flush := func() error { + if len(pending) == 0 { + return nil + } + err := deleteBatch(ctx, client, bucket, pending, f.dryRun, w) + pending = pending[:0] + return err + } + + opts := s3.ListOptions{Prefix: prefix, Versions: f.allVersions} + err := client.ListObjectsFunc(ctx, bucket, opts, func(obj s3.Object) error { + if !match(obj.Key) { + return nil + } seen++ - return deleteOne(ctx, client, bucket, obj.Key, dryRun, w) + pending = append(pending, s3.DeleteTarget{Key: obj.Key, VersionID: obj.VersionID}) + if len(pending) < s3.MaxDeleteBatch { + return nil + } + return flush() }) if err != nil { - return fmt.Errorf("deleting %s/%s recursively: %w", bucket, prefix, err) + return fmt.Errorf("deleting %s/%s recursively: %w", bucket, pattern, err) + } + if err := flush(); err != nil { + return err } if seen == 0 { - _, err = fmt.Fprintf(w, "No objects under %s/%s\n", bucket, prefix) + _, err = fmt.Fprintf(w, "No objects under %s/%s\n", bucket, pattern) + return err } - return err + return nil } -// deleteOne deletes a single key, or says what it would have deleted. -func deleteOne(ctx context.Context, client *s3.Client, bucket, key string, - dryRun bool, w io.Writer) error { +// deleteBatch removes one batch of keys in a single request, or says what it +// would have removed. +func deleteBatch(ctx context.Context, client *s3.Client, bucket string, + targets []s3.DeleteTarget, dryRun bool, w io.Writer) error { if dryRun { - _, err := fmt.Fprintf(w, "Would delete object: %s/%s\n", bucket, key) + for _, t := range targets { + if err := sayWouldDelete(w, bucket, t.Key, t.VersionID); err != nil { + return err + } + } + return nil + } + + failures, err := client.DeleteObjects(ctx, bucket, targets) + if err != nil { return err } - if err := client.DeleteObject(ctx, bucket, key); err != nil { - return fmt.Errorf("deleting %s/%s: %w", bucket, key, err) + + // A batch delete reports per-key refusals inside a 200, so the successes + // are the targets the server did not name. + refused := make(map[string]bool, len(failures)) + for _, fail := range failures { + refused[fail.Key+"\x00"+fail.VersionID] = true } - _, err := fmt.Fprintf(w, "Deleted object: %s/%s\n", bucket, key) + for _, t := range targets { + if refused[t.Key+"\x00"+t.VersionID] { + continue + } + if err := sayDeleted(w, bucket, t.Key, t.VersionID); err != nil { + return err + } + } + + errs := make([]error, 0, len(failures)) + for _, fail := range failures { + errs = append(errs, fmt.Errorf("deleting %s/%s: %w", bucket, fail.Key, fail)) + } + return errors.Join(errs...) +} + +// describeTarget names one delete target for an error message. +func describeTarget(bucket, key, versionID string) string { + if versionID == "" { + return bucket + "/" + key + } + return fmt.Sprintf("%s/%s (version %s)", bucket, key, versionID) +} + +func sayDeleted(w io.Writer, bucket, key, versionID string) error { + _, err := fmt.Fprintf(w, "Deleted object: %s\n", describeTarget(bucket, key, versionID)) + return err +} + +func sayWouldDelete(w io.Writer, bucket, key, versionID string) error { + _, err := fmt.Fprintf(w, "Would delete object: %s\n", describeTarget(bucket, key, versionID)) return err } diff --git a/internal/cli/s3/object_delete_test.go b/internal/cli/s3/object_delete_test.go index 265f87d..7c5e820 100644 --- a/internal/cli/s3/object_delete_test.go +++ b/internal/cli/s3/object_delete_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io" "net/http" "strings" "testing" @@ -49,11 +50,17 @@ func TestRunObjectDeleteRequiresKey(t *testing.T) { func TestRunObjectDeleteRecursive(t *testing.T) { var deleted []string client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodDelete { - deleted = append(deleted, r.URL.Path) - w.WriteHeader(http.StatusNoContent) + // A recursive delete goes out as one batched POST ?delete, not one + // DELETE per key: that is what makes emptying a large bucket finish. + if r.Method == http.MethodPost && r.URL.Query().Has("delete") { + body, _ := io.ReadAll(r.Body) + deleted = append(deleted, keysInBatch(string(body))...) + _, _ = fmt.Fprint(w, ``) return } + if r.Method == http.MethodDelete { + t.Errorf("a per-key DELETE was sent for %s", r.URL.Path) + } if got := r.URL.Query().Get("prefix"); got != "e2e-" { t.Errorf("prefix = %q, want e2e-", got) } @@ -69,17 +76,87 @@ func TestRunObjectDeleteRecursive(t *testing.T) { if err != nil { t.Fatal(err) } - want := "/db-backups/e2e-a.sql.gz,/db-backups/e2e-b.sql.gz" - if got := strings.Join(deleted, ","); got != want { + if got, want := strings.Join(deleted, ","), "e2e-a.sql.gz,e2e-b.sql.gz"; got != want { t.Errorf("deleted = %q, want %q", got, want) } + want := "Deleted object: db-backups/e2e-a.sql.gz\nDeleted object: db-backups/e2e-b.sql.gz\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A wildcard reference is itself the instruction to go recursive, and the +// pattern narrows the listing the prefix could not. +func TestRunObjectDeleteWildcardRef(t *testing.T) { + var deleted []string + client := newMockClient(t, batchDeleteHandler(t, &deleted, + `2026/a.sql.gz`+ + `2026/a.sha256`)) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/2026/*.sql.gz"}, + &objectDeleteFlags{}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(deleted, ","), "2026/a.sql.gz"; got != want { + t.Errorf("deleted = %q, want only the pattern match", got) + } +} + +// --exclude removes a key the prefix would otherwise have taken. +func TestRunObjectDeleteExclude(t *testing.T) { + var deleted []string + client := newMockClient(t, batchDeleteHandler(t, &deleted, + `a.sql.gzkeep.sql.gz`)) + + f := &objectDeleteFlags{recursive: true, filter: keyFilter{exclude: []string{"keep*"}}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runObjectDelete(context.Background(), client, []string{"db-backups/"}, f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(deleted, ","), "a.sql.gz"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } +} + +// A key the server refuses inside the 200 must be reported as a failure, while +// the rest of the batch is still reported as deleted. +func TestRunObjectDeleteReportsPerKeyFailures(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + _, _ = fmt.Fprint(w, ` + lockedAccessDeniedno + `) + return + } + _, _ = fmt.Fprint(w, `false + finelocked + `) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/"}, + &objectDeleteFlags{recursive: true}, &buf) + if err == nil { + t.Fatal("a refused key was reported as success") + } + if !strings.Contains(err.Error(), "locked") { + t.Errorf("error %q does not name the refused key", err) + } + if got, want := buf.String(), "Deleted object: db-backups/fine\n"; got != want { + t.Errorf("output = %q, want only the key that was deleted", got) + } } -// --dry-run must list the same keys it would remove and issue no DELETE at all. +// --dry-run must list the same keys it would remove and issue no delete at all. func TestRunObjectDeleteDryRun(t *testing.T) { client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodDelete { - t.Errorf("--dry-run issued DELETE %s", r.URL.Path) + if r.Method != http.MethodGet { + t.Errorf("--dry-run issued %s %s", r.Method, r.URL.Path) } _, _ = fmt.Fprint(w, `false a.sql.gz @@ -114,3 +191,92 @@ func TestRunObjectDeleteRecursiveNoMatches(t *testing.T) { t.Errorf("output = %q, want %q", got, want) } } + +// --version-id names exactly one object, so every shape that means "many" is a +// mistake worth refusing before anything is removed. +func TestObjectDeleteFlagValidation(t *testing.T) { + for _, tc := range []struct { + name string + flags objectDeleteFlags + refs []string + want string + }{ + {"version with recursive", objectDeleteFlags{versionID: "v1", recursive: true}, + []string{"b/k"}, "--recursive"}, + {"version with all-versions", objectDeleteFlags{versionID: "v1", allVersions: true}, + []string{"b/k"}, "mutually exclusive"}, + {"version with two refs", objectDeleteFlags{versionID: "v1"}, + []string{"b/k", "b/j"}, "a single reference"}, + {"filters without recursion", objectDeleteFlags{filter: keyFilter{include: []string{"*"}}}, + []string{"b/k"}, "--recursive"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.flags.validate(tc.refs) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } + + // A wildcard reference is its own recursion, so filters are legal with one. + f := objectDeleteFlags{filter: keyFilter{include: []string{"*.gz"}}} + if err := f.validate([]string{"b/2026/*"}); err != nil { + t.Errorf("filters with a wildcard ref were refused: %v", err) + } +} + +// A version-scoped delete addresses the one version and says which. +func TestRunObjectDeleteVersion(t *testing.T) { + var gotVersion string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotVersion = r.URL.Query().Get("versionId") + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/dump.sql.gz"}, + &objectDeleteFlags{versionID: "v3"}, &buf) + if err != nil { + t.Fatal(err) + } + if gotVersion != "v3" { + t.Errorf("versionId = %q, want v3", gotVersion) + } + want := "Deleted object: db-backups/dump.sql.gz (version v3)\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// batchDeleteHandler serves one listing page and records the keys of every +// batch delete it is sent. +func batchDeleteHandler(t *testing.T, deleted *[]string, contents string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Query().Has("delete") { + body, _ := io.ReadAll(r.Body) + *deleted = append(*deleted, keysInBatch(string(body))...) + _, _ = fmt.Fprint(w, ``) + return + } + _, _ = fmt.Fprint(w, `false`+contents+``) + } +} + +// keysInBatch pulls the keys out of a DeleteObjects request body. +func keysInBatch(body string) []string { + var keys []string + for rest := body; ; { + i := strings.Index(rest, "") + if i < 0 { + return keys + } + rest = rest[i+len(""):] + j := strings.Index(rest, "") + if j < 0 { + return keys + } + keys = append(keys, rest[:j]) + rest = rest[j:] + } +} diff --git a/internal/cli/s3/pool.go b/internal/cli/s3/pool.go new file mode 100644 index 0000000..e8fa797 --- /dev/null +++ b/internal/cli/s3/pool.go @@ -0,0 +1,94 @@ +package s3cli + +import ( + "context" + "sync" +) + +// pool runs transfers with a bounded number of goroutines. It exists because +// a recursive transfer of a thousand small objects is almost entirely round-trip +// latency: serially it takes a thousand round trips end to end, and with even +// four workers a quarter of that. s5cmd's headline speed is this and little +// else. +// +// The first failure cancels the pool's context so the rest stop pushing bytes at +// a store that is already refusing, and is what wait returns. Later failures are +// dropped: they are usually the cancellation, and a page of identical "context +// canceled" lines buries the one error that mattered. +type pool struct { + slots chan struct{} + wg sync.WaitGroup + cancel context.CancelFunc + + mu sync.Mutex + err error +} + +// newPool returns a pool of n workers and the context its work must use. +func newPool(ctx context.Context, n int) (*pool, context.Context) { + if n < 1 { + n = 1 + } + ctx, cancel := context.WithCancel(ctx) + return &pool{slots: make(chan struct{}, n), cancel: cancel}, ctx +} + +// run starts fn in the background once a worker slot frees up. It reports false +// when the pool has already failed or the context is done, which is the signal +// for the caller's producer loop to stop. +func (p *pool) run(ctx context.Context, fn func() error) bool { + select { + case p.slots <- struct{}{}: + case <-ctx.Done(): + p.fail(ctx.Err()) + return false + } + + p.wg.Add(1) + go func() { + defer p.wg.Done() + defer func() { <-p.slots }() + if err := fn(); err != nil { + p.fail(err) + } + }() + return true +} + +// fail records the first error and stops the others. +func (p *pool) fail(err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.err == nil { + p.err = err + p.cancel() + } +} + +// wait blocks until every started worker has finished and returns the first +// failure. It must be called exactly once, and the pool's cancel is released +// with it. +func (p *pool) wait() error { + p.wg.Wait() + p.cancel() + p.mu.Lock() + defer p.mu.Unlock() + return p.err +} + +// syncWriter serialises the per-object progress lines a pool's workers emit, so +// two transfers finishing at once cannot interleave half a line each. +// +// Lines therefore appear in completion order, which with more than one worker is +// not the listing order — a caller that needs a stable order must sort, and the +// tests do. +type syncWriter struct { + mu sync.Mutex + w interface{ Write([]byte) (int, error) } +} + +func (s *syncWriter) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.w.Write(p) +} diff --git a/internal/cli/s3/presign.go b/internal/cli/s3/presign.go new file mode 100644 index 0000000..74e9c15 --- /dev/null +++ b/internal/cli/s3/presign.go @@ -0,0 +1,104 @@ +package s3cli + +import ( + "errors" + "fmt" + "io" + "time" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +const presignLong = `Print a URL that reads (or writes) one object, without a key. + +The credential moves from the Authorization header into the query string, so the +whole request becomes a link anyone can fetch with curl or a browser until it +expires. It is how to hand a colleague one backup, or let a machine with no S3 +credentials at all deliver a dump, without sharing the access key. + +No request is made: this is local computation over the credentials koc already +has, so it works offline and leaves nothing on the store. That also means the URL +is not checked — presigning a key that does not exist succeeds and the URL 404s. + +Treat the output like a password. Anyone holding it can read that object until +it expires, and it will appear in shell history, CI logs and any chat it is +pasted into. Keep --expire as short as the recipient needs. + +--method put presigns an upload to the key instead of a download. SigV4 caps a +presigned URL's life at 7 days, and --s3-anonymous has nothing to sign with.` + +const presignExample = ` # A link good for an hour + koc s3 presign db-backups/dump-2026-09-13.sql.gz + + # A week, the protocol's maximum + koc s3 presign db-backups/dump.sql.gz --expire 168h + + # Hand an upload slot to a machine with no credentials + koc s3 presign db-backups/incoming.sql.gz --method put --expire 30m` + +func newPresignCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + var ( + expire time.Duration + method string + verID string + ) + cmd := &cobra.Command{ + Use: "presign /", + Short: "Print a presigned URL for an object", + Long: presignLong, + Example: presignExample, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, key, err := parseObjectRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runPresign(client, o, bucket, key, verID, method, expire, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.DurationVar(&expire, "expire", time.Hour, "how long the URL stays valid (maximum 168h)") + fl.StringVar(&method, "method", "get", "what the URL does: get or put") + fl.StringVar(&verID, "version-id", "", "presign this version of the object instead of the current one") + return cmd +} + +// runPresign is the test seam for "presign". It takes no context: presigning +// makes no request. +func runPresign(client *s3.Client, o *output.Options, bucket, key, versionID, method string, + expire time.Duration, w io.Writer) error { + var ( + url string + err error + ) + switch method { + case "get", "GET": + url, err = client.PresignGetObject(bucket, key, versionID, expire) + case "put", "PUT": + if versionID != "" { + return errors.New("--version-id addresses an existing object; it cannot be combined with --method put") + } + url, err = client.PresignPutObject(bucket, key, expire) + default: + return fmt.Errorf("--method must be get or put, got %q", method) + } + if err != nil { + return fmt.Errorf("presigning %s/%s: %w", bucket, key, err) + } + + return o.WriteSingle(w, + []string{"URL", "Method", "Expires"}, + []any{url, method, expire.String()}) +} diff --git a/internal/cli/s3/s3.go b/internal/cli/s3/s3.go index 90132a5..6f7e418 100644 --- a/internal/cli/s3/s3.go +++ b/internal/cli/s3/s3.go @@ -10,7 +10,7 @@ import ( // s3Long documents the credential sources on the group itself. The connection // flags are this group's primary input and are registered here rather than // globally, so this is the only place they are discoverable. -const s3Long = `Manage buckets and objects in an S3-compatible store, and move files in and out. +const s3Long = `Manage buckets and objects in an S3-compatible store, and move data in and out. This group is koc-specific (S3 is not an OpenStack service, and upstream's object-store commands speak Swift) and does not authenticate against Keystone: @@ -45,7 +45,17 @@ Addressing is path-style by default (//), matching the a store fronted by wildcard DNS. Every reference accepts either "/" or the "s3:///" -spelling, so a path copied out of s5cmd or "aws s3" pastes in unchanged.` +spelling, so a path copied out of s5cmd or "aws s3" pastes in unchanged, and a +wildcard ("db-backups/2026/*.gz") selects many where a command takes one. + +Uploads are multipart past --part-size, so there is no 5 GiB ceiling, and "-" as +the source reads standard input. download, upload, copy, move and object delete +all take --recursive over a whole prefix or tree, --concurrency objects at a +time, with --include/--exclude and --dry-run. copy and move are server-side: the +bytes never travel through koc. + +A failed request is retried with backoff (--s3-retries); --s3-anonymous sends +requests unsigned, for a bucket granted to everyone.` const s3Example = ` # Everything the credentials can see koc s3 bucket list @@ -65,6 +75,18 @@ const s3Example = ` # Everything the credentials can see koc s3 object delete scratch/ --recursive koc s3 bucket delete scratch + # Stream a dump straight in, no staging on disk + mysqldump --all-databases | gzip | koc s3 upload - db-backups/nightly.sql.gz + + # Restore a month, four objects at a time + koc s3 download db-backups/2026/08/ ./restore --recursive --concurrency 4 + + # Promote last night's dump, server-side + koc s3 copy db-backups/nightly-2026-09-13.sql.gz db-backups/latest.sql.gz + + # Hand someone one backup for an hour, without a key + koc s3 presign db-backups/latest.sql.gz --expire 1h + # GitLab's own key, straight from the cluster koc s3 --s3-creds-from-ns lcm-gitlab bucket list` @@ -84,5 +106,8 @@ func NewCommand(a *auth.Options, o *output.Options) *cobra.Command { cmd.AddCommand(newDownloadCommand(a, o, f)) cmd.AddCommand(newUploadCommand(a, o, f)) cmd.AddCommand(newDuCommand(a, o, f)) + cmd.AddCommand(newCopyCommand(a, o, f)) + cmd.AddCommand(newMoveCommand(a, o, f)) + cmd.AddCommand(newPresignCommand(a, o, f)) return cmd } diff --git a/internal/cli/s3/s3_test.go b/internal/cli/s3/s3_test.go index 43db5a2..e1d14f0 100644 --- a/internal/cli/s3/s3_test.go +++ b/internal/cli/s3/s3_test.go @@ -36,6 +36,18 @@ func newMockClient(t *testing.T, h http.HandlerFunc) *s3.Client { func valueOpts() *output.Options { return &output.Options{Format: output.FormatValue} } +// testUploadFlags is the flag set an "upload" seam gets when a test drives it +// directly: real defaults, but one part at a time so a mock endpoint sees the +// requests in a deterministic order. +func testUploadFlags() *uploadFlags { + return &uploadFlags{partSizeMiB: s3.DefaultPartSize >> 20, concurrency: 1} +} + +// testDownloadFlags is the same for "download". +func testDownloadFlags() *downloadFlags { + return &downloadFlags{concurrency: 1} +} + func TestRunBucketList(t *testing.T) { client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = fmt.Fprint(w, ` @@ -88,7 +100,7 @@ func TestRunObjectShow(t *testing.T) { var buf bytes.Buffer err := runObjectShow(context.Background(), client, &output.Options{Format: output.FormatJSON}, - "db-backups", "e2e-mariadb.sql.gz.sha256", &buf) + "db-backups", "e2e-mariadb.sql.gz.sha256", "", false, &buf) if err != nil { t.Fatal(err) } @@ -98,7 +110,7 @@ func TestRunObjectShow(t *testing.T) { } } - err = runObjectShow(context.Background(), client, valueOpts(), "db-backups", "missing", &buf) + err = runObjectShow(context.Background(), client, valueOpts(), "db-backups", "missing", "", false, &buf) if err == nil || !strings.Contains(err.Error(), `no object "missing" in bucket "db-backups"`) { t.Errorf("err = %v, want a friendly not-found message", err) } @@ -115,7 +127,7 @@ func TestRunDownloadToFile(t *testing.T) { var buf bytes.Buffer dest := filepath.Join(dir, "out.gz") err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -132,12 +144,12 @@ func TestRunDownloadToFile(t *testing.T) { // A second run must refuse rather than clobber the local copy of a backup. err = runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, flags: &downloadFlags{}}, &buf) if err == nil || !strings.Contains(err.Error(), "--force") { t.Errorf("err = %v, want a refusal pointing at --force", err) } if err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, force: true}, &buf); err != nil { + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, flags: &downloadFlags{force: true}}, &buf); err != nil { t.Errorf("--force download failed: %v", err) } } @@ -150,7 +162,7 @@ func TestRunDownloadStdout(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz.sha256", dest: "-"}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz.sha256", dest: "-", flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -170,7 +182,7 @@ func TestRunDownloadFailureLeavesNoFile(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "missing", dest: dest}, &buf) + downloadRequest{bucket: "db-backups", key: "missing", dest: dest, flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("expected an error") } @@ -197,7 +209,8 @@ func TestRunUpload(t *testing.T) { // No key given: the file's basename is used. var buf bytes.Buffer - if err := runUpload(context.Background(), client, valueOpts(), file, "db-backups", "", &buf); err != nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: file, bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf); err != nil { t.Fatal(err) } if gotPath != "/db-backups/dump.sql.gz" { @@ -214,14 +227,16 @@ func TestRunUpload(t *testing.T) { } // A key ending in "/" is a prefix, not a key. - if err := runUpload(context.Background(), client, valueOpts(), file, "db-backups", "2026/", &buf); err != nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: file, bucket: "db-backups", key: "2026/", flags: testUploadFlags()}, &buf); err != nil { t.Fatal(err) } if gotPath != "/db-backups/2026/dump.sql.gz" { t.Errorf("path = %q, want the basename appended to the prefix", gotPath) } - if err := runUpload(context.Background(), client, valueOpts(), filepath.Dir(file), "db-backups", "", &buf); err == nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: filepath.Dir(file), bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf); err == nil { t.Error("uploading a directory was accepted") } } diff --git a/internal/cli/s3/stop.go b/internal/cli/s3/stop.go new file mode 100644 index 0000000..bc2ab82 --- /dev/null +++ b/internal/cli/s3/stop.go @@ -0,0 +1,11 @@ +package s3cli + +import "errors" + +// errStopListing ends a ListObjectsFunc walk early without making the caller +// treat it as a failure. The client propagates a callback error verbatim, which +// is exactly what a "stop here" signal needs — and what keeps a --limit from +// paging through the rest of a million-key bucket. +var errStopListing = errors.New("stop listing") + +func isStopListing(err error) bool { return errors.Is(err, errStopListing) } diff --git a/internal/cli/s3/transfer.go b/internal/cli/s3/transfer.go deleted file mode 100644 index aeb5932..0000000 --- a/internal/cli/s3/transfer.go +++ /dev/null @@ -1,207 +0,0 @@ -package s3cli - -import ( - "context" - "errors" - "fmt" - "io" - "mime" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - - "github.com/ftarasenko/go-openstackclient/internal/auth" - "github.com/ftarasenko/go-openstackclient/internal/output" - "github.com/ftarasenko/go-openstackclient/internal/s3" -) - -// downloadRequest is one "download" invocation: the object to fetch, where to -// put it, and the one flag that governs it. The positional arguments travel -// with the flag because runDownload needs all four and eight parameters is one -// too many (go:S107). -type downloadRequest struct { - bucket, key string - // dest is the destination path: "" means the key's basename, "-" means - // stream to stdout. - dest string - force bool -} - -const downloadLong = `Download an object to a file. - -With no FILE the object's key basename is used. FILE "-" streams the object to -stdout as raw bytes and prints nothing else, so it pipes. - -An existing file is never overwritten without --force: these are backups, and a -half-typed key that clobbers the local copy of one is not a recoverable -mistake. A transfer that fails partway removes the partial file rather than -leaving a truncated dump that looks complete.` - -func newDownloadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { - var force bool - cmd := &cobra.Command{ - Use: "download / [file]", - Short: "Download an object to a file", - Long: downloadLong, - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - if err := o.Validate(); err != nil { - return err - } - bucket, key, err := parseObjectRef(args[0]) - if err != nil { - return err - } - req := downloadRequest{bucket: bucket, key: key, force: force} - if len(args) == 2 { - req.dest = args[1] - } - ctx := cmd.Context() - client, err := f.client(ctx, a) - if err != nil { - return err - } - return runDownload(ctx, client, o, req, cmd.OutOrStdout()) - }, - } - cmd.Flags().BoolVar(&force, "force", false, "overwrite the destination file if it exists") - return cmd -} - -// runDownload is the test seam for "download". w receives either the object's -// raw bytes (dest "-") or the summary table. -func runDownload(ctx context.Context, client *s3.Client, o *output.Options, - r downloadRequest, w io.Writer) error { - if r.dest == "-" { - if _, err := client.GetObject(ctx, r.bucket, r.key, w); err != nil { - return downloadError(r.bucket, r.key, err) - } - return nil - } - dest := r.dest - if dest == "" { - dest = filepath.Base(r.key) - } - if info, err := os.Stat(dest); err == nil && info.IsDir() { - dest = filepath.Join(dest, filepath.Base(r.key)) - } - - n, err := downloadToFile(ctx, client, r.bucket, r.key, dest, r.force) - if err != nil { - return err - } - return o.WriteSingle(w, - []string{"Bucket", "Key", "File", "Size"}, - []any{r.bucket, r.key, dest, n}) -} - -// downloadToFile streams the object into dest, leaving nothing behind if it -// fails. -func downloadToFile(ctx context.Context, client *s3.Client, bucket, key, dest string, force bool) (n int64, err error) { - flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL - if force { - flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC - } - out, err := os.OpenFile(dest, flags, 0o600) //nolint:gosec // G304: operator-supplied output path - if err != nil { - if errors.Is(err, os.ErrExist) { - return 0, fmt.Errorf("%s already exists (pass --force to overwrite)", dest) - } - return 0, fmt.Errorf("creating %q: %w", dest, err) - } - defer func() { - closeErr := out.Close() - if err == nil && closeErr != nil { - err = fmt.Errorf("closing %q: %w", dest, closeErr) - } - if err != nil { - _ = os.Remove(dest) - } - }() - - if n, err = client.GetObject(ctx, bucket, key, out); err != nil { - return 0, downloadError(bucket, key, err) - } - return n, nil -} - -// downloadError turns a missing key into the message an operator can act on. -func downloadError(bucket, key string, err error) error { - if s3.IsNotFound(err) { - return fmt.Errorf("no object %q in bucket %q", key, bucket) - } - return fmt.Errorf("downloading %s/%s: %w", bucket, key, err) -} - -const uploadLong = `Upload a file to a bucket. - -With no key, or a key ending in "/", the file's basename is used. The upload is -a single signed PUT — there is no multipart support, so the server's own -single-part ceiling (5 GiB on Garage and on AWS) applies.` - -func newUploadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { - cmd := &cobra.Command{ - Use: "upload [/]", - Short: "Upload a file to a bucket", - Long: uploadLong, - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - if err := o.Validate(); err != nil { - return err - } - bucket, key, err := parseRef(args[1]) - if err != nil { - return err - } - ctx := cmd.Context() - client, err := f.client(ctx, a) - if err != nil { - return err - } - return runUpload(ctx, client, o, args[0], bucket, key, cmd.OutOrStdout()) - }, - } - return cmd -} - -// runUpload is the test seam for "upload". -func runUpload(ctx context.Context, client *s3.Client, o *output.Options, - file, bucket, key string, w io.Writer) error { - if key == "" || strings.HasSuffix(key, "/") { - key += filepath.Base(file) - } - - src, err := os.Open(file) //nolint:gosec // G304: operator-supplied upload path - if err != nil { - return fmt.Errorf("opening %q: %w", file, err) - } - defer func() { _ = src.Close() }() - - info, err := src.Stat() - if err != nil { - return fmt.Errorf("stat %q: %w", file, err) - } - if info.IsDir() { - return fmt.Errorf("%q is a directory; upload takes a single file", file) - } - - obj, err := client.PutObject(ctx, bucket, key, src, info.Size(), contentTypeFor(file)) - if err != nil { - return fmt.Errorf("uploading %s to %s/%s: %w", file, bucket, key, err) - } - return o.WriteSingle(w, - []string{"Bucket", "Key", "File", "Size", "ETag"}, - []any{bucket, key, file, obj.Size, obj.ETag}) -} - -// contentTypeFor guesses a Content-Type from the file extension, falling back to -// the S3 default. Getting this right matters for the objects koc uploads next to -// a backup — a .sha256 sibling should read as text in a browser, not download. -func contentTypeFor(file string) string { - if ct := mime.TypeByExtension(filepath.Ext(file)); ct != "" { - return ct - } - return "application/octet-stream" -} diff --git a/internal/cli/s3/transfer_bulk_test.go b/internal/cli/s3/transfer_bulk_test.go new file mode 100644 index 0000000..731fd7f --- /dev/null +++ b/internal/cli/s3/transfer_bulk_test.go @@ -0,0 +1,477 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// listingHandler serves one page of the given and the bytes "data" +// for every object GET — enough to drive a recursive transfer. +func listingHandler(t *testing.T, contents string, seen *[]string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false`+contents+``) + return + } + if seen != nil { + *seen = append(*seen, r.URL.Path) + } + _, _ = w.Write([]byte("data")) + } +} + +// sortedLines makes an assertion independent of completion order, which with +// more than one worker is not the listing order. +func sortedLines(s string) []string { + lines := strings.Split(strings.TrimSuffix(s, "\n"), "\n") + sort.Strings(lines) + return lines +} + +// A recursive download mirrors each key's path below the prefix, so a restore +// reproduces the layout rather than a flat pile of basenames. +func TestRunDownloadRecursiveMirrorsKeyPaths(t *testing.T) { + client := newMockClient(t, listingHandler(t, + `2026/08/a.sql.gz4`+ + `2026/09/b.sql.gz4`, nil)) + + dir := t.TempDir() + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "2026/", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + for _, rel := range []string{"08/a.sql.gz", "09/b.sql.gz"} { + if _, err := os.Stat(filepath.Join(dir, rel)); err != nil { + t.Errorf("%s was not written: %v", rel, err) + } + } +} + +// --flatten puts every object straight in the destination. +func TestRunDownloadRecursiveFlatten(t *testing.T) { + client := newMockClient(t, listingHandler(t, + `2026/08/a.sql.gz4`, nil)) + + dir := t.TempDir() + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive, f.flatten = true, true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "2026/", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "a.sql.gz")); err != nil { + t.Errorf("the object was not flattened into the destination: %v", err) + } +} + +// A wildcard reference is its own recursion, and narrows what the prefix took. +func TestRunDownloadWildcard(t *testing.T) { + var fetched []string + client := newMockClient(t, listingHandler(t, + `2026/a.sql.gz4`+ + `2026/a.sha2564`, &fetched)) + + dir := t.TempDir() + var buf bytes.Buffer + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "2026/*.sql.gz", dest: dir, flags: testDownloadFlags()}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(fetched, ","), "/db-backups/2026/a.sql.gz"; got != want { + t.Errorf("fetched = %q, want only the pattern match", got) + } +} + +// In bulk mode an existing file is a resumption point, not a failure: a restore +// that was interrupted is finished by re-running the same command. +func TestRunDownloadRecursiveSkipsExistingFiles(t *testing.T) { + var fetched []string + client := newMockClient(t, listingHandler(t, + `a.sql.gz4`+ + `b.sql.gz4`, &fetched)) + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.sql.gz"), []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(fetched, ","), "/db-backups/b.sql.gz"; got != want { + t.Errorf("fetched = %q, want only the missing object", got) + } + if !strings.Contains(buf.String(), "Skipped (exists)") { + t.Errorf("output %q does not report the skip", buf.String()) + } + // The existing file must be untouched, not truncated. + if body, _ := os.ReadFile(filepath.Join(dir, "a.sql.gz")); string(body) != "old" { + t.Errorf("the existing file was overwritten: %q", body) + } +} + +// An object key is server-supplied data: "../../etc/x" is a legal S3 key, and a +// restore that wrote it where it says would let whoever can write to the bucket +// write anywhere koc can. +func TestRunDownloadRecursiveRefusesEscapingKeys(t *testing.T) { + client := newMockClient(t, listingHandler(t, + `../escaped.sh4`, nil)) + + dir := filepath.Join(t.TempDir(), "restore") + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "", dest: dir, flags: f}, &buf) + if err == nil { + t.Fatal("a key escaping the destination was accepted") + } + if !strings.Contains(err.Error(), "outside") { + t.Errorf("error %q does not say what was refused", err) + } + if _, statErr := os.Stat(filepath.Join(filepath.Dir(dir), "escaped.sh")); statErr == nil { + t.Fatal("the escaping key was written") + } +} + +// A "directory marker" — a zero-length key ending in "/" — must not become a +// file, or it would shadow the directory its siblings need. +func TestRunDownloadRecursiveSkipsDirectoryMarkers(t *testing.T) { + var fetched []string + client := newMockClient(t, listingHandler(t, + `2026/0`+ + `2026/a.sql.gz4`, &fetched)) + + dir := t.TempDir() + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(fetched, ","), "/db-backups/2026/a.sql.gz"; got != want { + t.Errorf("fetched = %q, want the marker skipped", got) + } +} + +func TestRunDownloadRecursiveNoMatches(t *testing.T) { + client := newMockClient(t, listingHandler(t, "", nil)) + + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "gone/", dest: t.TempDir(), flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "No objects under db-backups/gone/\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// The combinations that cannot mean anything must be refused when the arguments +// are parsed, before a client is even built. +func TestNewDownloadRequestValidation(t *testing.T) { + for _, tc := range []struct { + name string + args []string + flags downloadFlags + want string + }{ + {"recursive to stdout", []string{"b/k", "-"}, downloadFlags{recursive: true}, "not stdout"}, + {"recursive with a version", []string{"b/k", "d"}, downloadFlags{recursive: true, versionID: "v"}, "--recursive"}, + {"no key", []string{"b"}, downloadFlags{}, "names no object"}, + {"filters without recursion", []string{"b/k"}, + downloadFlags{filter: keyFilter{include: []string{"*"}}}, "--recursive"}, + {"flatten without recursion", []string{"b/k"}, downloadFlags{flatten: true}, "--flatten"}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := tc.flags + _, err := newDownloadRequest(tc.args, &flags) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// A recursive upload mirrors the local tree under the destination prefix. +func TestRunUploadRecursive(t *testing.T) { + var puts []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + puts = append(puts, r.URL.Path) + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + root := t.TempDir() + mustWrite(t, filepath.Join(root, "a.sql.gz"), "one") + mustWrite(t, filepath.Join(root, "nested", "b.sql.gz"), "two") + + var buf bytes.Buffer + f := testUploadFlags() + f.recursive = true + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: root, bucket: "db-backups", key: "2026/", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + sort.Strings(puts) + want := "/db-backups/2026/a.sql.gz,/db-backups/2026/nested/b.sql.gz" + if got := strings.Join(puts, ","); got != want { + t.Errorf("uploaded = %q, want %q", got, want) + } +} + +// --include selects files by the key they would get, so a filter reads the same +// way as it does on a listing. +func TestRunUploadRecursiveFilters(t *testing.T) { + var puts []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + puts = append(puts, r.URL.Path) + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + root := t.TempDir() + mustWrite(t, filepath.Join(root, "a.sql.gz"), "one") + mustWrite(t, filepath.Join(root, "a.sha256"), "two") + + var buf bytes.Buffer + f := testUploadFlags() + f.recursive = true + f.filter = keyFilter{include: []string{"*.sql.gz"}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: root, bucket: "db-backups", key: "", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(puts, ","), "/db-backups/a.sql.gz"; got != want { + t.Errorf("uploaded = %q, want %q", got, want) + } +} + +// --no-clobber costs one HEAD and leaves an object that is already there alone, +// which is how an interrupted bulk upload resumes. +func TestRunUploadNoClobberSkipsExisting(t *testing.T) { + var methods []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method+" "+r.URL.Path) + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "3") + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + src := filepath.Join(t.TempDir(), "dump.sql.gz") + mustWrite(t, src, "payload") + + var buf bytes.Buffer + f := testUploadFlags() + f.noClobber = true + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "dump.sql.gz", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(methods, ","), "HEAD /db-backups/dump.sql.gz"; got != want { + t.Errorf("requests = %q, want only the existence check", got) + } + if !strings.Contains(buf.String(), "Skipped (exists)") { + t.Errorf("output %q does not report the skip", buf.String()) + } +} + +// Standard input has no length, so it must take the multipart path — and it +// needs a full key, since a stream has no basename to fall back on. +func TestRunUploadStdinNeedsAFullKey(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("no request should be made without a key") + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: "-", bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf) + if err == nil || !strings.Contains(err.Error(), "full key") { + t.Errorf("err = %v, want it to ask for a full key", err) + } +} + +func TestUploadFlagValidation(t *testing.T) { + for _, tc := range []struct { + name string + req uploadRequest + flags uploadFlags + want string + }{ + {"stdin with recursive", uploadRequest{src: "-"}, uploadFlags{recursive: true}, "--recursive"}, + {"flatten without recursion", uploadRequest{src: "x"}, uploadFlags{flatten: true}, "--flatten"}, + {"filters without recursion", uploadRequest{src: "x"}, + uploadFlags{filter: keyFilter{exclude: []string{"*"}}}, "--recursive"}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := tc.flags + if err := flags.validate(tc.req); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// An object past the part size has to go out as a multipart upload, or the +// 5 GiB single-PUT ceiling would apply. +func TestRunUploadUsesMultipartPastThePartSize(t *testing.T) { + var started bool + var parts int + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case q.Has("uploads"): + started = true + _, _ = fmt.Fprint(w, `u`) + case q.Has("partNumber"): + parts++ + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("ETag", fmt.Sprintf(`"p%s"`, q.Get("partNumber"))) + w.WriteHeader(http.StatusOK) + default: + _, _ = fmt.Fprint(w, `"final"`) + } + }) + + src := filepath.Join(t.TempDir(), "big.bin") + mustWrite(t, src, strings.Repeat("x", s3.MinPartSize+1024)) + + var buf bytes.Buffer + f := testUploadFlags() + f.partSizeMiB = s3.MinPartSize >> 20 // 5 MiB, the protocol minimum + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "big.bin", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if !started { + t.Error("an object past the part size was sent as a single PUT") + } + if parts != 2 { + t.Errorf("parts = %d, want 2", parts) + } + if !strings.Contains(buf.String(), "final") { + t.Errorf("output %q does not carry the completed ETag", buf.String()) + } +} + +// --content-type overrides the extension guess for the whole run. +func TestRunUploadContentTypeOverride(t *testing.T) { + var gotType string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotType = r.Header.Get("Content-Type") + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + src := filepath.Join(t.TempDir(), "dump.txt") + mustWrite(t, src, "payload") + + var buf bytes.Buffer + f := testUploadFlags() + f.contentType = "application/x-custom" + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "b", key: "k", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if gotType != "application/x-custom" { + t.Errorf("Content-Type = %q, want the override", gotType) + } +} + +// --dry-run must name every transfer it would make and make none. +func TestRunUploadRecursiveDryRun(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("--dry-run sent %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + }) + + root := t.TempDir() + mustWrite(t, filepath.Join(root, "a.sql.gz"), "one") + + var buf bytes.Buffer + f := testUploadFlags() + f.recursive, f.dryRun = true, true + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: root, bucket: "db-backups", key: "p/", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got := sortedLines(buf.String()); len(got) != 1 || !strings.Contains(got[0], "Would upload") { + t.Errorf("output = %q, want one 'Would upload' line", buf.String()) + } +} + +func TestKeyForFile(t *testing.T) { + root := filepath.Join("tmp", "src") + for _, tc := range []struct { + name, prefix, path string + flatten bool + want string + }{ + {"mirrors the tree", "2026/", filepath.Join(root, "a", "b.gz"), false, "2026/a/b.gz"}, + {"no prefix", "", filepath.Join(root, "a", "b.gz"), false, "a/b.gz"}, + {"flattened", "2026/", filepath.Join(root, "a", "b.gz"), true, "2026/b.gz"}, + {"prefix without a slash", "2026", filepath.Join(root, "b.gz"), false, "2026/b.gz"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := keyForFile(tc.prefix, root, tc.path, tc.flatten) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("keyForFile = %q, want %q", got, tc.want) + } + }) + } +} + +func mustWrite(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/internal/cli/s3/transfer_paths_test.go b/internal/cli/s3/transfer_paths_test.go index ad0b4c3..1ebeaf5 100644 --- a/internal/cli/s3/transfer_paths_test.go +++ b/internal/cli/s3/transfer_paths_test.go @@ -23,7 +23,7 @@ func TestRunDownload_DefaultsToTheKeyBasename(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "nested/e2e-mariadb.sql.gz"}, &buf) + downloadRequest{bucket: "db-backups", key: "nested/e2e-mariadb.sql.gz", flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestRunDownload_DirectoryDestination(t *testing.T) { dir := t.TempDir() var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dir}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dir, flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -65,7 +65,7 @@ func TestRunDownload_UncreatableDestination(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(file, "dump")}, &buf) + downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(file, "dump"), flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("an uncreatable destination was reported as success") } @@ -82,7 +82,7 @@ func TestRunDownload_StdoutMissingKey(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "missing", dest: "-"}, &buf) + downloadRequest{bucket: "db-backups", key: "missing", dest: "-", flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("a 404 streaming to stdout was reported as success") } @@ -101,7 +101,7 @@ func TestRunDownload_NonNotFoundErrorIsWrapped(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(t.TempDir(), "dump")}, &buf) + downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(t.TempDir(), "dump"), flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("a 403 was reported as success") } @@ -120,7 +120,7 @@ func TestRunUpload_MissingSourceFile(t *testing.T) { var buf bytes.Buffer err := runUpload(context.Background(), client, valueOpts(), - filepath.Join(t.TempDir(), "absent"), "db-backups", "key", &buf) + uploadRequest{src: filepath.Join(t.TempDir(), "absent"), bucket: "db-backups", key: "key", flags: testUploadFlags()}, &buf) if err == nil { t.Fatal("a missing source file was reported as success") } @@ -139,7 +139,8 @@ func TestRunUpload_RejectsADirectory(t *testing.T) { }) var buf bytes.Buffer - err := runUpload(context.Background(), client, valueOpts(), t.TempDir(), "db-backups", "", &buf) + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: t.TempDir(), bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf) if err == nil { t.Fatal("a directory was accepted as an upload source") } @@ -166,7 +167,8 @@ func TestRunUpload_PrefixKeyAppendsBasename(t *testing.T) { } var buf bytes.Buffer - if err := runUpload(context.Background(), client, valueOpts(), src, "db-backups", "daily/", &buf); err != nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "daily/", flags: testUploadFlags()}, &buf); err != nil { t.Fatal(err) } if gotPath != "/db-backups/daily/dump.sql.gz" { @@ -186,7 +188,8 @@ func TestRunUpload_APIErrorIsWrapped(t *testing.T) { } var buf bytes.Buffer - err := runUpload(context.Background(), client, valueOpts(), src, "db-backups", "key", &buf) + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "key", flags: testUploadFlags()}, &buf) if err == nil { t.Fatal("a 403 was reported as success") } diff --git a/internal/cli/s3/upload.go b/internal/cli/s3/upload.go new file mode 100644 index 0000000..5771fa8 --- /dev/null +++ b/internal/cli/s3/upload.go @@ -0,0 +1,370 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// uploadFlags holds the options accepted by "upload". +type uploadFlags struct { + recursive bool + dryRun bool + flatten bool + noClobber bool + contentType string + partSizeMiB int + concurrency int + filter keyFilter +} + +// uploadRequest is one "upload" invocation. +type uploadRequest struct { + // src is a file, a directory (with --recursive), or "-" for stdin. + src string + bucket string + key string + flags *uploadFlags +} + +const uploadLong = `Upload a file, a directory, or standard input. + +With no key, or a key ending in "/", the file's basename is used. + +Objects larger than the part size go out as a multipart upload, so there is no +5 GiB single-PUT ceiling: a dump of any size uploads, in --part-size chunks, +--concurrency of them at a time. An upload that fails is aborted, so a +half-written object never becomes visible. + +SRC "-" reads standard input, which is what makes a dump streamable without +staging it on disk first: + + mysqldump ... | gzip | koc s3 upload - db-backups/dump-$(date +%F).sql.gz + +A stream's length is not known ahead of time, so it is always multipart; memory +stays at --part-size × --concurrency however long the stream turns out to be. + +--recursive uploads a directory tree, mirroring each file's path under the key +as a prefix; --flatten puts every file at the top of that prefix instead. +--include/--exclude select which files are taken, matched against the *key* the +file would get. --no-clobber skips a key that already exists, so an interrupted +bulk upload resumes by re-running the same command. + +Content types are guessed from the file extension, which is what makes a +".sha256" sibling read as text in a browser rather than download; --content-type +overrides that for every object in the run. Standard input has no extension, so +it gets the S3 default unless --content-type says otherwise.` + +const uploadExample = ` # One file + koc s3 upload ./dump.sql.gz db-backups/ + + # Straight from a pipe, no staging on disk + mysqldump --all-databases | gzip | koc s3 upload - db-backups/nightly.sql.gz + + # A tree, skipping what is already there + koc s3 upload ./restore db-backups/2026/ --recursive --no-clobber + + # Bigger parts for a fat link + koc s3 upload ./huge.tar db-backups/ --part-size 128 --concurrency 8` + +func newUploadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + uf := &uploadFlags{} + cmd := &cobra.Command{ + Use: "upload [/]", + Short: "Upload a file, a directory, or standard input", + Long: uploadLong, + Example: uploadExample, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, key, err := parseRef(args[1]) + if err != nil { + return err + } + req := uploadRequest{src: args[0], bucket: bucket, key: key, flags: uf} + if err := uf.validate(req); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runUpload(ctx, client, o, req, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVarP(&uf.recursive, "recursive", "r", false, "upload a directory tree") + fl.BoolVar(&uf.dryRun, "dry-run", false, "print what would be uploaded and transfer nothing") + fl.BoolVar(&uf.flatten, "flatten", false, + "put every file at the top of the destination prefix instead of mirroring its path") + fl.BoolVar(&uf.noClobber, "no-clobber", false, "skip a key that already exists") + fl.StringVar(&uf.contentType, "content-type", "", "Content-Type for every object in this run") + fl.IntVar(&uf.partSizeMiB, "part-size", s3.DefaultPartSize>>20, + "multipart part size in MiB (minimum 5)") + fl.IntVar(&uf.concurrency, "concurrency", s3.DefaultConcurrency, + "parts — or, under --recursive, files — to upload at once") + uf.filter.addTo(fl) + return cmd +} + +// validate rejects flag combinations that cannot mean anything. +func (f *uploadFlags) validate(r uploadRequest) error { + switch { + case f.recursive && r.src == "-": + return errors.New("standard input is one object; it cannot be combined with --recursive") + case !f.recursive && f.flatten: + return errors.New("--flatten only applies to a recursive upload") + case !f.recursive && f.filter.active(): + return errors.New("--include/--exclude select files from a tree; pass --recursive") + } + return f.filter.compile() +} + +// uploadOptions renders the flags into the client's options. +func (f *uploadFlags) uploadOptions(size int64, contentType string) s3.UploadOptions { + return s3.UploadOptions{ + ContentType: contentType, + PartSize: int64(f.partSizeMiB) << 20, + Concurrency: f.concurrency, + Size: size, + } +} + +// runUpload is the test seam for "upload". +func runUpload(ctx context.Context, client *s3.Client, o *output.Options, + r uploadRequest, w io.Writer) error { + switch { + case r.flags.recursive: + return runUploadRecursive(ctx, client, r, w) + case r.src == "-": + return runUploadStdin(ctx, client, o, r, w) + default: + return runUploadFile(ctx, client, o, r, w) + } +} + +// runUploadStdin uploads standard input under the key given, which must be a +// full key: a stream has no basename to fall back on. +func runUploadStdin(ctx context.Context, client *s3.Client, o *output.Options, + r uploadRequest, w io.Writer) error { + if r.key == "" || strings.HasSuffix(r.key, "/") { + return fmt.Errorf("uploading standard input needs a full key, got %q", r.bucket+"/"+r.key) + } + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would upload standard input to %s/%s\n", r.bucket, r.key) + return err + } + if skip, err := checkClobber(ctx, client, r.bucket, r.key, r.flags, w); skip || err != nil { + return err + } + + // Size -1 says "unknown", which forces the multipart path: a pipe cannot be + // rewound, and a single signed PUT has to hash its body before sending it. + opts := r.flags.uploadOptions(-1, r.flags.contentType) + obj, err := client.PutObjectStream(ctx, r.bucket, r.key, os.Stdin, opts) + if err != nil { + return fmt.Errorf("uploading standard input to %s/%s: %w", r.bucket, r.key, err) + } + return o.WriteSingle(w, + []string{"Bucket", "Key", "File", "Size", "ETag"}, + []any{r.bucket, r.key, "-", obj.Size, obj.ETag}) +} + +// runUploadFile uploads one local file. +func runUploadFile(ctx context.Context, client *s3.Client, o *output.Options, + r uploadRequest, w io.Writer) error { + key := r.key + if key == "" || strings.HasSuffix(key, "/") { + key += filepath.Base(r.src) + } + + src, err := os.Open(r.src) + if err != nil { + return fmt.Errorf("opening %q: %w", r.src, err) + } + defer func() { _ = src.Close() }() + + info, err := src.Stat() + if err != nil { + return fmt.Errorf("stat %q: %w", r.src, err) + } + if info.IsDir() { + return fmt.Errorf("%q is a directory; pass --recursive to upload a tree", r.src) + } + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would upload %s to %s/%s\n", r.src, r.bucket, key) + return err + } + if skip, err := checkClobber(ctx, client, r.bucket, key, r.flags, w); skip || err != nil { + return err + } + + opts := r.flags.uploadOptions(info.Size(), r.contentTypeFor(r.src)) + obj, err := client.PutObjectStream(ctx, r.bucket, key, src, opts) + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", r.src, r.bucket, key, err) + } + return o.WriteSingle(w, + []string{"Bucket", "Key", "File", "Size", "ETag"}, + []any{r.bucket, key, r.src, obj.Size, obj.ETag}) +} + +// runUploadRecursive uploads every file under a directory. +func runUploadRecursive(ctx context.Context, client *s3.Client, r uploadRequest, w io.Writer) error { + root, err := os.Stat(r.src) + if err != nil { + return fmt.Errorf("reading %q: %w", r.src, err) + } + if !root.IsDir() { + return fmt.Errorf("%q is not a directory; drop --recursive to upload one file", r.src) + } + + out := &syncWriter{w: w} + // Under --recursive the workers are files, and each file's own multipart + // then goes out a part at a time: the product of the two would put + // concurrency² parts in flight and that many part buffers in memory. + fileFlags := *r.flags + fileFlags.concurrency = 1 + p, workCtx := newPool(ctx, r.flags.concurrency) + + taken := 0 + err = filepath.WalkDir(r.src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !d.Type().IsRegular() { + // Symlinks and devices are skipped rather than followed: a tree + // walked from a backup directory should not chase a link out of it. + return nil + } + key, err := keyForFile(r.key, r.src, path, r.flags.flatten) + if err != nil { + return err + } + if !r.flags.filter.match(key) { + return nil + } + taken++ + if !p.run(workCtx, func() error { + return uploadOne(workCtx, client, path, r.bucket, key, &fileFlags, out) + }) { + return filepath.SkipAll + } + return nil + }) + if waitErr := p.wait(); err == nil { + err = waitErr + } + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", r.src, r.bucket, r.key, err) + } + if taken == 0 { + _, err = fmt.Fprintf(w, "No files under %s\n", r.src) + return err + } + return nil +} + +// uploadOne uploads one file of a recursive run. +func uploadOne(ctx context.Context, client *s3.Client, path, bucket, key string, + f *uploadFlags, w io.Writer) error { + if f.dryRun { + _, err := fmt.Fprintf(w, "Would upload %s to %s/%s\n", path, bucket, key) + return err + } + if skip, err := checkClobber(ctx, client, bucket, key, f, w); skip || err != nil { + return err + } + + src, err := os.Open(path) //nolint:gosec // G304: path from the operator-named tree + if err != nil { + return fmt.Errorf("opening %q: %w", path, err) + } + defer func() { _ = src.Close() }() + + info, err := src.Stat() + if err != nil { + return fmt.Errorf("stat %q: %w", path, err) + } + + contentType := f.contentType + if contentType == "" { + contentType = contentTypeFor(path) + } + obj, err := client.PutObjectStream(ctx, bucket, key, src, f.uploadOptions(info.Size(), contentType)) + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", path, bucket, key, err) + } + _, err = fmt.Fprintf(w, "Uploaded: %s -> %s/%s (%d bytes)\n", path, bucket, key, obj.Size) + return err +} + +// checkClobber implements --no-clobber: one HEAD before the transfer, and the +// object is left alone if it is already there. It reports whether the caller +// should skip this object. +func checkClobber(ctx context.Context, client *s3.Client, bucket, key string, + f *uploadFlags, w io.Writer) (skip bool, err error) { + if !f.noClobber { + return false, nil + } + if _, err := client.HeadObject(ctx, bucket, key, ""); err != nil { + if s3.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("checking %s/%s: %w", bucket, key, err) + } + _, err = fmt.Fprintf(w, "Skipped (exists): %s/%s\n", bucket, key) + return true, err +} + +// keyForFile maps one local path to its object key, mirroring the tree under +// the destination prefix unless flattened. +func keyForFile(prefix, root, path string, flatten bool) (string, error) { + rel := filepath.Base(path) + if !flatten { + var err error + if rel, err = filepath.Rel(root, path); err != nil { + return "", fmt.Errorf("resolving %q under %q: %w", path, root, err) + } + } + // Object keys are slash-separated whatever the local separator is, so a + // tree uploaded from Windows lands under the same keys as from Linux. + rel = filepath.ToSlash(rel) + + if prefix == "" { + return rel, nil + } + return strings.TrimSuffix(prefix, "/") + "/" + rel, nil +} + +// contentTypeFor guesses a Content-Type from the file extension, falling back to +// the S3 default. Getting this right matters for the objects koc uploads next to +// a backup — a .sha256 sibling should read as text in a browser, not download. +func contentTypeFor(file string) string { + if ct := mime.TypeByExtension(filepath.Ext(file)); ct != "" { + return ct + } + return "application/octet-stream" +} + +// contentTypeFor on the request prefers an explicit --content-type. +func (r uploadRequest) contentTypeFor(file string) string { + if r.flags.contentType != "" { + return r.flags.contentType + } + return contentTypeFor(file) +} diff --git a/internal/output/output.go b/internal/output/output.go index 3dfdd5c..3ab0696 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -968,3 +968,40 @@ func cellRaw(v any) string { return string(b) } } + +// byteUnits are the binary (IEC) multiples HumanBytes renders. Binary rather +// than decimal because that is what df, ls -h and every S3 console show, so a +// "14.2 GiB" from koc lines up with what an operator sees elsewhere. +var byteUnits = []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} + +// HumanBytes renders a byte count for a reader: exact bytes below 1 KiB, then +// three significant figures with a binary unit. +// +// It is opt-in everywhere it appears (a --human flag), never the default: koc's +// tables are meant to survive a pipe into awk, and a rounded "14.2 GiB" is not +// a number a script can add up. +func HumanBytes(n int64) string { + if n < 0 { + return "-" + HumanBytes(-n) + } + if n < 1024 { + return strconv.FormatInt(n, 10) + " B" + } + + value, unit := float64(n)/1024, byteUnits[0] + for _, next := range byteUnits[1:] { + if value < 1024 { + break + } + value, unit = value/1024, next + } + + switch { + case value < 10: + return strconv.FormatFloat(value, 'f', 2, 64) + " " + unit + case value < 100: + return strconv.FormatFloat(value, 'f', 1, 64) + " " + unit + default: + return strconv.FormatFloat(value, 'f', 0, 64) + " " + unit + } +} diff --git a/internal/s3/copy.go b/internal/s3/copy.go new file mode 100644 index 0000000..0a7dec4 --- /dev/null +++ b/internal/s3/copy.go @@ -0,0 +1,82 @@ +package s3 + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" +) + +// CopyObject copies an object inside the store, without the bytes travelling +// through koc. The source is named in a header rather than a body, so a 100 GiB +// copy costs one small request. +// +// versionID selects a specific version of the source, or "" for the current one. +// contentType, when empty, is carried over from the source along with the rest +// of its metadata (S3's default COPY directive). +func (c *Client) CopyObject(ctx context.Context, src, dst ObjectRef, versionID, contentType string) (*ObjectInfo, error) { + // The copy source is a path, so its slashes must survive encoding while + // everything else in the key is escaped — the same rule as a request URI. + source := "/" + src.Bucket + "/" + strings.TrimPrefix(src.Key, "/") + if versionID != "" { + source += "?versionId=" + uriEncode(versionID, false) + } + + hdr := map[string]string{"x-amz-copy-source": uriEncode(source, true)} + if contentType != "" { + hdr["Content-Type"] = contentType + // Without this S3 keeps the source's metadata and ignores the header. + hdr["x-amz-metadata-directive"] = "REPLACE" + } + + var result struct { + ETag string `xml:"ETag"` + LastModified string `xml:"LastModified"` + } + req := request{ + method: http.MethodPut, + url: c.url(dst.Bucket, dst.Key, nil), + payloadHash: emptySHA256, + header: hdr, + } + // Like a multipart completion, a copy can fail inside a 200: the server + // holds the connection open while it copies and then writes an . + err := c.do(ctx, req, func(resp *http.Response) error { + payload, readErr := io.ReadAll(io.LimitReader(resp.Body, errorBodyLimit)) + if readErr != nil { + return fmt.Errorf("reading copy result: %w", readErr) + } + if code := errorCodeInBody(payload); code != "" { + return &APIError{ + StatusCode: resp.StatusCode, + Code: code, + Message: "copy was refused after the request was accepted", + Method: http.MethodPut, + Path: req.url.Path, + } + } + return xml.Unmarshal(payload, &result) + }) + if err != nil { + return nil, fmt.Errorf("copying %s to %s: %w", src, dst, err) + } + + return &ObjectInfo{ + Bucket: dst.Bucket, + Key: dst.Key, + ETag: strings.Trim(result.ETag, `"`), + LastModified: parseS3Time(result.LastModified), + ContentType: contentType, + }, nil +} + +// ObjectRef is a bucket/key pair. It exists so CopyObject's four string +// arguments cannot be transposed into a copy in the wrong direction. +type ObjectRef struct { + Bucket string + Key string +} + +func (r ObjectRef) String() string { return r.Bucket + "/" + r.Key } diff --git a/internal/s3/copy_delete_test.go b/internal/s3/copy_delete_test.go new file mode 100644 index 0000000..88a2061 --- /dev/null +++ b/internal/s3/copy_delete_test.go @@ -0,0 +1,206 @@ +package s3 + +import ( + "context" + "crypto/md5" + "encoding/base64" + "fmt" + "io" + "net/http" + "strings" + "testing" +) + +func TestCopyObject(t *testing.T) { + var gotMethod, gotPath, gotSource, gotDirective string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + gotMethod, gotPath = r.Method, r.URL.Path + gotSource = r.Header.Get("x-amz-copy-source") + gotDirective = r.Header.Get("x-amz-metadata-directive") + _, _ = fmt.Fprint(w, `"copied"`+ + `2026-09-14T10:00:00.000Z`) + }) + + src := ObjectRef{Bucket: "db-backups", Key: "nightly.sql.gz"} + dst := ObjectRef{Bucket: "archive", Key: "2026/nightly.sql.gz"} + obj, err := c.CopyObject(context.Background(), src, dst, "", "") + if err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut || gotPath != "/archive/2026/nightly.sql.gz" { + t.Errorf("request = %s %s, want PUT the destination", gotMethod, gotPath) + } + if gotSource != "/db-backups/nightly.sql.gz" { + t.Errorf("x-amz-copy-source = %q", gotSource) + } + // Without --content-type the source's metadata is carried over, which is + // S3's default and must not be overridden. + if gotDirective != "" { + t.Errorf("x-amz-metadata-directive = %q, want it unset", gotDirective) + } + if obj.ETag != "copied" { + t.Errorf("ETag = %q, want copied", obj.ETag) + } + if obj.LastModified.IsZero() { + t.Error("LastModified was not parsed") + } +} + +// A key with a space or a "+" has to reach the header percent-encoded, with its +// slashes intact — the copy source is a path, and the signature covers it. +func TestCopyObjectEncodesTheSource(t *testing.T) { + var gotSource string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotSource = r.Header.Get("x-amz-copy-source") + _, _ = fmt.Fprint(w, `"e"`) + }) + + src := ObjectRef{Bucket: "b", Key: "dir/a b+c.txt"} + if _, err := c.CopyObject(context.Background(), src, ObjectRef{Bucket: "b", Key: "d"}, "", ""); err != nil { + t.Fatal(err) + } + if gotSource != "/b/dir/a%20b%2Bc.txt" { + t.Errorf("x-amz-copy-source = %q, want the key encoded and the slashes kept", gotSource) + } +} + +// Replacing the content type needs the REPLACE directive, or S3 keeps the +// source's and ignores the header. +func TestCopyObjectReplacesMetadataForANewContentType(t *testing.T) { + var gotType, gotDirective string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotType, gotDirective = r.Header.Get("Content-Type"), r.Header.Get("x-amz-metadata-directive") + _, _ = fmt.Fprint(w, `"e"`) + }) + + _, err := c.CopyObject(context.Background(), + ObjectRef{Bucket: "b", Key: "k"}, ObjectRef{Bucket: "b", Key: "k2"}, "", "text/plain") + if err != nil { + t.Fatal(err) + } + if gotType != "text/plain" || gotDirective != "REPLACE" { + t.Errorf("Content-Type = %q, directive = %q; want text/plain and REPLACE", gotType, gotDirective) + } +} + +// A version ID travels in the copy-source header as a query string, not as a +// request parameter. +func TestCopyObjectCarriesTheSourceVersion(t *testing.T) { + var gotSource string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotSource = r.Header.Get("x-amz-copy-source") + _, _ = fmt.Fprint(w, `"e"`) + }) + + _, err := c.CopyObject(context.Background(), + ObjectRef{Bucket: "b", Key: "k"}, ObjectRef{Bucket: "b", Key: "k2"}, "v7", "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(gotSource, "versionId%3Dv7") && !strings.Contains(gotSource, "versionId=v7") { + t.Errorf("x-amz-copy-source = %q, want it to name version v7", gotSource) + } +} + +// Like a multipart completion, a copy can be refused inside a 200. +func TestCopyObjectDetectsAnErrorInsideA200(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, `SlowDowntry later`) + }) + + _, err := c.CopyObject(context.Background(), + ObjectRef{Bucket: "b", Key: "k"}, ObjectRef{Bucket: "b", Key: "k2"}, "", "") + if err == nil { + t.Fatal("an inside a 200 was reported as a successful copy") + } + if got := ErrorCode(err); got != "SlowDown" { + t.Errorf("ErrorCode = %q, want SlowDown", got) + } +} + +func TestDeleteObjectsBatch(t *testing.T) { + var gotMethod, gotBody, gotMD5, gotType string + var hasDeleteParam bool + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + body, _ := io.ReadAll(r.Body) + gotMethod, gotBody = r.Method, string(body) + gotMD5, gotType = r.Header.Get("Content-MD5"), r.Header.Get("Content-Type") + hasDeleteParam = r.URL.Query().Has("delete") + _, _ = fmt.Fprint(w, ``) + }) + + targets := []DeleteTarget{{Key: "a"}, {Key: "b", VersionID: "v1"}} + failures, err := c.DeleteObjects(context.Background(), "db-backups", targets) + if err != nil { + t.Fatal(err) + } + if len(failures) != 0 { + t.Errorf("failures = %v, want none", failures) + } + if gotMethod != http.MethodPost || !hasDeleteParam { + t.Errorf("request = %s (delete param: %v), want POST ?delete", gotMethod, hasDeleteParam) + } + if gotType != "application/xml" { + t.Errorf("Content-Type = %q", gotType) + } + // Quiet mode: only failures come back, which is all this call reports. + for _, want := range []string{"true", "a", "b", "v1"} { + if !strings.Contains(gotBody, want) { + t.Errorf("body %s missing %q", gotBody, want) + } + } + // AWS refuses a batch delete without the Content-MD5 of the body. + sum := md5.Sum([]byte(gotBody)) + if want := base64.StdEncoding.EncodeToString(sum[:]); gotMD5 != want { + t.Errorf("Content-MD5 = %q, want %q", gotMD5, want) + } +} + +// S3 reports per-key refusals inside a 200, so they are returned rather than +// raised: the rest of the batch did happen. +func TestDeleteObjectsReportsPerKeyFailures(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, ` + lockedAccessDeniedno + `) + }) + + failures, err := c.DeleteObjects(context.Background(), "b", []DeleteTarget{{Key: "ok"}, {Key: "locked"}}) + if err != nil { + t.Fatalf("a partial failure must not fail the call: %v", err) + } + if len(failures) != 1 || failures[0].Key != "locked" || failures[0].Code != "AccessDenied" { + t.Fatalf("failures = %+v, want the one refused key", failures) + } + if got := failures[0].Error(); !strings.Contains(got, "locked") || !strings.Contains(got, "AccessDenied") { + t.Errorf("failure message %q does not name the key and the code", got) + } +} + +func TestDeleteObjectsRejectsAnOversizedBatch(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an oversized batch must be refused before any request") + w.WriteHeader(http.StatusOK) + }) + + targets := make([]DeleteTarget, MaxDeleteBatch+1) + if _, err := c.DeleteObjects(context.Background(), "b", targets); err == nil { + t.Fatal("an oversized batch was accepted") + } +} + +// An empty batch is a no-op, not a malformed request. +func TestDeleteObjectsOnAnEmptyBatch(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an empty batch must make no request") + w.WriteHeader(http.StatusOK) + }) + + failures, err := c.DeleteObjects(context.Background(), "b", nil) + if err != nil || failures != nil { + t.Errorf("DeleteObjects(nil) = %v, %v; want no-op", failures, err) + } +} diff --git a/internal/s3/delete.go b/internal/s3/delete.go new file mode 100644 index 0000000..fbede31 --- /dev/null +++ b/internal/s3/delete.go @@ -0,0 +1,118 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" +) + +// MaxDeleteBatch is the protocol's limit on keys per DeleteObjects call. +const MaxDeleteBatch = 1000 + +// DeleteTarget is one key, optionally one specific version of it, for a batch +// delete. +type DeleteTarget struct { + Key string + VersionID string +} + +// DeleteError is one key a batch delete refused. S3 reports these inside a +// 200 response, so they are returned rather than raised: the rest of the batch +// did happen. +type DeleteError struct { + Key string + VersionID string + Code string + Message string +} + +func (f DeleteError) Error() string { + if f.Message != "" { + return fmt.Sprintf("%s: %s (%s)", f.Key, f.Message, f.Code) + } + return fmt.Sprintf("%s: %s", f.Key, f.Code) +} + +// DeleteObjects removes up to MaxDeleteBatch keys in one request, which is what +// makes emptying a bucket of a million objects finish: it is a thousandth of the +// requests one-DELETE-per-key costs. +// +// The returned slice names the keys the server refused. A nil error with a +// non-empty slice is the normal partial-failure case, not a contradiction — +// S3 answers 200 and itemises what it would not do. +func (c *Client) DeleteObjects(ctx context.Context, bucket string, targets []DeleteTarget) ([]DeleteError, error) { + if len(targets) == 0 { + return nil, nil + } + if len(targets) > MaxDeleteBatch { + return nil, fmt.Errorf("a batch delete takes at most %d keys, got %d", MaxDeleteBatch, len(targets)) + } + + // Field-for-field a DeleteTarget, so one converts to the other; the tags + // are what this copy exists for. + type object struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + } + doc := struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet"` + Objects []object `xml:"Object"` + }{ + // Quiet suppresses a success entry per key; only the failures come + // back, which is all this call reports. + Quiet: true, + Objects: make([]object, len(targets)), + } + for i, t := range targets { + doc.Objects[i] = object(t) + } + + body, err := xml.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("encoding batch delete: %w", err) + } + + var result struct { + Errors []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` + } + req := request{ + method: http.MethodPost, + url: c.url(bucket, "", url.Values{"delete": {""}}), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + header: map[string]string{ + "Content-Type": "application/xml", + // AWS refuses a batch delete without it; see contentMD5. + "Content-MD5": contentMD5(body), + }, + } + err = c.do(ctx, req, func(resp *http.Response) error { + payload, readErr := io.ReadAll(io.LimitReader(resp.Body, errorBodyLimit)) + if readErr != nil { + return fmt.Errorf("reading batch delete result: %w", readErr) + } + return xml.Unmarshal(payload, &result) + }) + if err != nil { + return nil, fmt.Errorf("deleting %d objects from %s: %w", len(targets), bucket, err) + } + + var failures []DeleteError + for _, e := range result.Errors { + failures = append(failures, DeleteError{ + Key: e.Key, VersionID: e.VersionID, Code: e.Code, Message: e.Message, + }) + } + return failures, nil +} diff --git a/internal/s3/errors_test.go b/internal/s3/errors_test.go index e135e16..2913b4e 100644 --- a/internal/s3/errors_test.go +++ b/internal/s3/errors_test.go @@ -76,7 +76,7 @@ func TestNewAPIErrorSynthesisesNoSuchKey(t *testing.T) { w.WriteHeader(http.StatusNotFound) }) - _, err := c.HeadObject(context.Background(), "db-backups", "missing") + _, err := c.HeadObject(context.Background(), "db-backups", "missing", "") if !IsNotFound(err) { t.Fatalf("a bodiless 404 was not recognised as not-found: %v", err) } @@ -189,7 +189,7 @@ func TestGetObjectCopyError(t *testing.T) { _, _ = w.Write([]byte("some bytes")) }) - _, err := c.GetObject(context.Background(), "db-backups", "key", failWriter{}) + _, err := c.GetObject(context.Background(), "db-backups", "key", "", failWriter{}) if err == nil { t.Fatal("a failing writer was reported as success") } @@ -225,7 +225,7 @@ func TestDoTransportError(t *testing.T) { t.Fatal(err) } - _, err = c.HeadObject(context.Background(), "db-backups", "key") + _, err = c.HeadObject(context.Background(), "db-backups", "key", "") if err == nil { t.Fatal("an unreachable endpoint was reported as success") } @@ -244,7 +244,7 @@ func TestHeadObjectUsesContentLengthHeader(t *testing.T) { w.WriteHeader(http.StatusOK) }) - info, err := c.HeadObject(context.Background(), "db-backups", "key") + info, err := c.HeadObject(context.Background(), "db-backups", "key", "") if err != nil { t.Fatal(err) } diff --git a/internal/s3/integrity.go b/internal/s3/integrity.go new file mode 100644 index 0000000..a40139e --- /dev/null +++ b/internal/s3/integrity.go @@ -0,0 +1,20 @@ +package s3 + +import ( + "crypto/md5" //nolint:gosec // G501: not a security choice — see contentMD5 + "encoding/base64" +) + +// contentMD5 renders the Content-MD5 header S3 requires on the request bodies +// that carry a list of operations (DeleteObjects) and accepts on the small +// configuration PUTs (PutBucketVersioning). +// +// MD5 is not a security decision here and no alternative is available: the +// algorithm is fixed by the S3 protocol, the header exists so the server can +// reject a body corrupted in transit, and the request is already authenticated +// by a SHA-256 SigV4 signature over the same bytes. Omitting the header makes +// AWS refuse a batch delete outright. +func contentMD5(body []byte) string { + sum := md5.Sum(body) //nolint:gosec // G401: protocol-mandated integrity check, not authentication + return base64.StdEncoding.EncodeToString(sum[:]) +} diff --git a/internal/s3/lifecycle_test.go b/internal/s3/lifecycle_test.go index 330b317..1169df2 100644 --- a/internal/s3/lifecycle_test.go +++ b/internal/s3/lifecycle_test.go @@ -111,7 +111,7 @@ func TestDeleteObject(t *testing.T) { w.WriteHeader(http.StatusNoContent) }) - if err := client.DeleteObject(context.Background(), "db-backups", "dump.sql.gz"); err != nil { + if err := client.DeleteObject(context.Background(), "db-backups", "dump.sql.gz", ""); err != nil { t.Fatal(err) } if gotMethod != http.MethodDelete || gotPath != "/db-backups/dump.sql.gz" { @@ -150,7 +150,7 @@ func TestListObjectsFuncPagesAndCapsLimit(t *testing.T) { var keys []string var sum int64 - err := client.ListObjectsFunc(context.Background(), "b", "", 3, func(o Object) error { + err := client.ListObjectsFunc(context.Background(), "b", ListOptions{Limit: 3}, func(o Object) error { keys = append(keys, o.Key) sum += o.Size return nil @@ -183,7 +183,7 @@ func TestListObjectsFuncPropagatesCallbackError(t *testing.T) { }) seen := 0 - err := client.ListObjectsFunc(context.Background(), "b", "", 0, func(Object) error { + err := client.ListObjectsFunc(context.Background(), "b", ListOptions{}, func(Object) error { seen++ return io.ErrUnexpectedEOF }) diff --git a/internal/s3/multipart.go b/internal/s3/multipart.go new file mode 100644 index 0000000..f9a9579 --- /dev/null +++ b/internal/s3/multipart.go @@ -0,0 +1,389 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" +) + +// Multipart upload. Two things force it, and PutObject can do neither: +// +// - the single-part ceiling. A signed PUT carries the whole object in one +// request body, and every S3 implementation caps that at 5 GiB, so a +// database dump past that size simply could not be uploaded. +// - an unseekable source. SigV4 signs a hash of the payload, so PutObject has +// to read its body twice and therefore demands an io.ReadSeeker. A pipe +// ("mysqldump | koc s3 upload -") cannot provide one. Multipart hashes each +// part separately, so the source only ever has to be read forwards. +const ( + // MinPartSize is the floor S3 puts on every part but the last. + MinPartSize = 5 << 20 + + // DefaultPartSize is the part size koc uses when none is given. Memory cost + // is PartSize × Concurrency, so this pairs with DefaultConcurrency for a + // 64 MiB working set — a deliberate choice for the air-gapped cluster nodes + // koc runs on, where s5cmd's 50 MiB × 5 would be a quarter of a gigabyte. + DefaultPartSize = 16 << 20 + + // DefaultConcurrency is how many parts are uploaded at once by default. + DefaultConcurrency = 4 + + // maxParts is the protocol's limit on parts per upload. With the default + // part size that caps one object at 160 GiB; --part-size raises it. + maxParts = 10000 +) + +// UploadOptions configures PutObjectStream. +type UploadOptions struct { + ContentType string + + // PartSize is the size of each part but the last (0 = DefaultPartSize). + // Values below MinPartSize are rejected rather than silently raised, so a + // "--part-size 1" does not quietly become something else. + PartSize int64 + + // Concurrency is how many parts travel at once (0 = DefaultConcurrency). + Concurrency int + + // Size is the source's length when it is known, or -1 when it is not (a + // pipe). It only selects the strategy: a known size at or below PartSize + // goes out as a single PUT, which is one request instead of three. + Size int64 +} + +// withDefaults validates the options and fills the zero values in. +func (o UploadOptions) withDefaults() (UploadOptions, error) { + if o.PartSize == 0 { + o.PartSize = DefaultPartSize + } + if o.PartSize < MinPartSize { + return o, fmt.Errorf("part size must be at least %d bytes (S3's minimum), got %d", MinPartSize, o.PartSize) + } + if o.Concurrency == 0 { + o.Concurrency = DefaultConcurrency + } + if o.Concurrency < 1 { + return o, fmt.Errorf("concurrency must be at least 1, got %d", o.Concurrency) + } + if o.Size == 0 { + // A zero-length object is legitimate and must not be mistaken for + // "unknown"; callers say unknown with a negative value. + o.Size = 0 + } + return o, nil +} + +// PutObjectStream uploads from r, choosing the single-PUT or multipart path by +// what opts.Size says about the source. +// +// A multipart upload that fails is aborted, so a half-written object never +// becomes visible and the parts already stored stop accruing cost. If the abort +// itself fails the returned error says so — the upload ID is then the operator's +// to clean up. +func (c *Client) PutObjectStream(ctx context.Context, bucket, key string, r io.Reader, + opts UploadOptions) (*ObjectInfo, error) { + opts, err := opts.withDefaults() + if err != nil { + return nil, err + } + + // A source whose length is known and fits one part is cheaper as a plain + // PUT, but only if it is seekable — SigV4 has to hash it before sending. + if seeker, ok := r.(io.ReadSeeker); ok && opts.Size >= 0 && opts.Size <= opts.PartSize { + return c.PutObject(ctx, bucket, key, seeker, opts.Size, opts.ContentType) + } + + uploadID, err := c.createMultipartUpload(ctx, bucket, key, opts.ContentType) + if err != nil { + return nil, err + } + + parts, err := c.uploadParts(ctx, bucket, key, uploadID, r, opts) + if err != nil { + if abortErr := c.abortMultipartUpload(ctx, bucket, key, uploadID); abortErr != nil { + return nil, fmt.Errorf("%w (and aborting upload %s failed: %w)", err, uploadID, abortErr) + } + return nil, err + } + return c.completeMultipartUpload(ctx, bucket, key, uploadID, parts, opts.ContentType) +} + +// completedPart is one finished part: S3 wants the number and the ETag back in +// the completion body, in ascending order. +type completedPart struct { + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + size int64 +} + +// createMultipartUpload opens an upload and returns its ID. +func (c *Client) createMultipartUpload(ctx context.Context, bucket, key, contentType string) (string, error) { + hdr := map[string]string{} + if contentType != "" { + hdr["Content-Type"] = contentType + } + + var result struct { + UploadID string `xml:"UploadId"` + } + req := request{ + method: http.MethodPost, + url: c.url(bucket, key, url.Values{"uploads": {""}}), + payloadHash: emptySHA256, + header: hdr, + } + err := c.do(ctx, req, func(resp *http.Response) error { + return xml.NewDecoder(resp.Body).Decode(&result) + }) + if err != nil { + return "", fmt.Errorf("starting multipart upload of %s/%s: %w", bucket, key, err) + } + if result.UploadID == "" { + return "", fmt.Errorf("starting multipart upload of %s/%s: server returned no upload ID", bucket, key) + } + return result.UploadID, nil +} + +// partUploader carries the state uploadParts shares with its workers. It is a +// struct rather than a pile of closure variables because the read loop, the +// worker body and the failure path all touch the same four fields, and a +// mutex-guarded field is easier to reason about when its lock is next to it. +type partUploader struct { + client *Client + bucket string + key string + upload string + + // slots bounds how many parts are in flight, and with them how many + // part-sized buffers are live: the working set stays PartSize × + // Concurrency however large the object is. + slots chan struct{} + wg sync.WaitGroup + cancel context.CancelFunc + + mu sync.Mutex + parts []completedPart + err error +} + +// fail records the first error and stops the other workers: their parts are +// about to be aborted anyway, and a failing upload should not keep pushing +// bytes at the server. +func (u *partUploader) fail(err error) { + u.mu.Lock() + defer u.mu.Unlock() + if u.err == nil { + u.err = err + u.cancel() + } +} + +// send uploads one part in the background, blocking first until a slot frees +// up. It reports false if the upload has already failed, which ends the read +// loop. +func (u *partUploader) send(ctx context.Context, number int, body []byte) bool { + select { + case u.slots <- struct{}{}: + case <-ctx.Done(): + u.fail(ctx.Err()) + return false + } + + u.wg.Add(1) + go func() { + defer u.wg.Done() + defer func() { <-u.slots }() + + etag, err := u.client.uploadPart(ctx, u.bucket, u.key, u.upload, number, body) + if err != nil { + u.fail(err) + return + } + u.mu.Lock() + u.parts = append(u.parts, completedPart{PartNumber: number, ETag: etag, size: int64(len(body))}) + u.mu.Unlock() + }() + return true +} + +// uploadParts reads r into part-sized buffers and uploads them with up to +// opts.Concurrency workers. +// +// Reading is sequential and single-threaded because the source may be a pipe +// and a pipe can only be read forwards; only the uploads overlap. +func (c *Client) uploadParts(ctx context.Context, bucket, key, uploadID string, + r io.Reader, opts UploadOptions) ([]completedPart, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + u := &partUploader{ + client: c, bucket: bucket, key: key, upload: uploadID, + slots: make(chan struct{}, opts.Concurrency), + cancel: cancel, + } + + for number := 1; ; number++ { + if number > maxParts { + u.fail(fmt.Errorf("object needs more than %d parts at a part size of %d bytes; raise --part-size", + maxParts, opts.PartSize)) + break + } + + buf := make([]byte, opts.PartSize) + n, readErr := io.ReadFull(r, buf) + if n > 0 && !u.send(ctx, number, buf[:n]) { + break + } + if readErr != nil { + // ErrUnexpectedEOF is the short final part, not a failure. + if !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { + u.fail(fmt.Errorf("reading upload source: %w", readErr)) + } + break + } + } + + u.wg.Wait() + if u.err != nil { + return nil, u.err + } + sort.Slice(u.parts, func(i, j int) bool { return u.parts[i].PartNumber < u.parts[j].PartNumber }) + return u.parts, nil +} + +// uploadPart sends one part and returns its ETag. +func (c *Client) uploadPart(ctx context.Context, bucket, key, uploadID string, + number int, body []byte) (string, error) { + q := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {uploadID}} + var etag string + req := request{ + method: http.MethodPut, + url: c.url(bucket, key, q), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + } + err := c.do(ctx, req, func(resp *http.Response) error { + etag = resp.Header.Get("ETag") + if etag == "" { + return fmt.Errorf("part %d: server returned no ETag", number) + } + return drainBody(resp) + }) + if err != nil { + return "", fmt.Errorf("uploading part %d of %s/%s: %w", number, bucket, key, err) + } + return etag, nil +} + +// completeMultipartUpload assembles the parts into the finished object. +func (c *Client) completeMultipartUpload(ctx context.Context, bucket, key, uploadID string, + parts []completedPart, contentType string) (*ObjectInfo, error) { + // An upload with no parts at all cannot be completed — S3 rejects an empty + // part list — so a zero-length source becomes a zero-length single PUT. + if len(parts) == 0 { + if err := c.abortMultipartUpload(ctx, bucket, key, uploadID); err != nil { + return nil, err + } + return c.PutObject(ctx, bucket, key, bytes.NewReader(nil), 0, contentType) + } + + body, err := xml.Marshal(struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []completedPart `xml:"Part"` + }{Parts: parts}) + if err != nil { + return nil, fmt.Errorf("encoding multipart completion: %w", err) + } + + var total int64 + for _, p := range parts { + total += p.size + } + + var result struct { + ETag string `xml:"ETag"` + } + req := request{ + method: http.MethodPost, + url: c.url(bucket, key, url.Values{"uploadId": {uploadID}}), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + header: map[string]string{"Content-Type": "application/xml"}, + } + // A completion can fail *inside* a 200 response: S3 keeps the connection + // open while it assembles the object and then writes an document. + // Decoding into a struct that only knows about ETag would swallow it. + err = c.do(ctx, req, func(resp *http.Response) error { + payload, readErr := io.ReadAll(io.LimitReader(resp.Body, errorBodyLimit)) + if readErr != nil { + return fmt.Errorf("reading multipart completion: %w", readErr) + } + if code := errorCodeInBody(payload); code != "" { + return &APIError{ + StatusCode: resp.StatusCode, + Code: code, + Message: "multipart completion was refused after the request was accepted", + Method: http.MethodPost, + Path: req.url.Path, + } + } + return xml.Unmarshal(payload, &result) + }) + if err != nil { + return nil, fmt.Errorf("completing multipart upload of %s/%s: %w", bucket, key, err) + } + + return &ObjectInfo{ + Bucket: bucket, + Key: key, + Size: total, + ETag: strings.Trim(result.ETag, `"`), + ContentType: contentType, + }, nil +} + +// abortMultipartUpload discards an upload and the parts already stored. +func (c *Client) abortMultipartUpload(ctx context.Context, bucket, key, uploadID string) error { + // The parent context may already be cancelled — that is the usual reason + // this is being called — so the abort gets a context of its own, or it + // would fail without being sent and leak the parts. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortTimeout) + defer cancel() + + req := request{ + method: http.MethodDelete, + url: c.url(bucket, key, url.Values{"uploadId": {uploadID}}), + payloadHash: emptySHA256, + } + if err := c.do(ctx, req, drainBody); err != nil { + return fmt.Errorf("aborting multipart upload of %s/%s: %w", bucket, key, err) + } + return nil +} + +// errorCodeInBody reports the of an S3 document, or "" if the +// payload is not one. +func errorCodeInBody(payload []byte) string { + var doc struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + } + if xml.Unmarshal(payload, &doc) != nil { + return "" + } + if doc.Code == "" && doc.XMLName.Local == "Error" { + return "InternalError" + } + return doc.Code +} diff --git a/internal/s3/multipart_test.go b/internal/s3/multipart_test.go new file mode 100644 index 0000000..22c71d0 --- /dev/null +++ b/internal/s3/multipart_test.go @@ -0,0 +1,275 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "testing" +) + +// multipartRecorder is a mock S3 that speaks the three-call multipart protocol +// and records what it was sent. +type multipartRecorder struct { + mu sync.Mutex + + created int + aborted int + complete string // the completion body + parts map[int]string + failPart int // a part number to refuse, 0 for none +} + +func newMultipartRecorder() *multipartRecorder { + return &multipartRecorder{parts: map[int]string{}} +} + +func (m *multipartRecorder) handler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + body, _ := io.ReadAll(r.Body) + + m.mu.Lock() + defer m.mu.Unlock() + switch { + case r.Method == http.MethodPost && q.Has("uploads"): + m.created++ + _, _ = fmt.Fprint(w, `up-1`) + case r.Method == http.MethodPut && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + if n == m.failPart { + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `AccessDenied`) + return + } + m.parts[n] = string(body) + w.Header().Set("ETag", fmt.Sprintf(`"etag-%d"`, n)) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && q.Has("uploadId"): + m.complete = string(body) + _, _ = fmt.Fprint(w, `"final-etag"`) + case r.Method == http.MethodDelete && q.Has("uploadId"): + m.aborted++ + w.WriteHeader(http.StatusNoContent) + default: + // A plain PUT: the single-part path. + m.parts[0] = string(body) + w.Header().Set("ETag", `"single"`) + w.WriteHeader(http.StatusOK) + } + } +} + +// unseekable hides the io.Seeker a bytes.Reader would otherwise offer, which is +// what a pipe looks like to the client. +type unseekable struct{ r io.Reader } + +func (u unseekable) Read(p []byte) (int, error) { return u.r.Read(p) } + +// A source whose length is unknown — stdin — must take the multipart path even +// when it is tiny, because a single signed PUT has to hash its body first and a +// pipe cannot be rewound. +func TestPutObjectStreamUsesMultipartForAnUnknownSize(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + obj, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader("small")}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if rec.created != 1 { + t.Errorf("multipart uploads created = %d, want 1", rec.created) + } + if got := rec.parts[1]; got != "small" { + t.Errorf("part 1 = %q, want small", got) + } + if obj.ETag != "final-etag" { + t.Errorf("ETag = %q, want final-etag", obj.ETag) + } + if obj.Size != 5 { + t.Errorf("Size = %d, want 5", obj.Size) + } +} + +// A known, small, seekable source is one request instead of three. +func TestPutObjectStreamUsesASinglePutWhenItFits(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + obj, err := c.PutObjectStream(context.Background(), "b", "k", + bytes.NewReader([]byte("payload")), + UploadOptions{Size: 7, PartSize: MinPartSize, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if rec.created != 0 { + t.Errorf("a multipart upload was started for a %d-byte object", 7) + } + if obj.ETag != "single" { + t.Errorf("ETag = %q, want single", obj.ETag) + } +} + +// Several parts must arrive whole, in the right pieces, and be listed in +// ascending order in the completion body — S3 rejects any other order. +func TestPutObjectStreamSplitsAndOrdersParts(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + // Three parts: two full and a short last one. + payload := strings.Repeat("a", MinPartSize) + strings.Repeat("b", MinPartSize) + "tail" + obj, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader(payload)}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 3}) + if err != nil { + t.Fatal(err) + } + if got := len(rec.parts); got != 3 { + t.Fatalf("parts = %d, want 3", got) + } + if len(rec.parts[1]) != MinPartSize || len(rec.parts[2]) != MinPartSize || rec.parts[3] != "tail" { + t.Errorf("part sizes = %d/%d/%q, want two full parts and a short tail", + len(rec.parts[1]), len(rec.parts[2]), rec.parts[3]) + } + if obj.Size != int64(len(payload)) { + t.Errorf("Size = %d, want %d", obj.Size, len(payload)) + } + // Reassembled, the parts are the original stream. + var nums []int + for n := range rec.parts { + nums = append(nums, n) + } + sort.Ints(nums) + var joined strings.Builder + for _, n := range nums { + joined.WriteString(rec.parts[n]) + } + if joined.String() != payload { + t.Error("the parts do not reassemble into the source stream") + } + // The completion body must name the parts in ascending order. + if !strings.Contains(rec.complete, "1") || + strings.Index(rec.complete, "1") > + strings.Index(rec.complete, "3") { + t.Errorf("completion body is not in ascending part order:\n%s", rec.complete) + } +} + +// A failed part must abort the upload: a half-written object must never become +// visible, and the stored parts must stop costing money. +func TestPutObjectStreamAbortsOnFailure(t *testing.T) { + rec := newMultipartRecorder() + rec.failPart = 2 + c := newTestClient(t, rec.handler(t)) + + payload := strings.Repeat("a", MinPartSize) + strings.Repeat("b", MinPartSize) + _, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader(payload)}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err == nil { + t.Fatal("a refused part was reported as success") + } + if !strings.Contains(err.Error(), "part 2") { + t.Errorf("error %q does not name the failing part", err) + } + if rec.aborted != 1 { + t.Errorf("aborts = %d, want 1 — the parts already stored would otherwise leak", rec.aborted) + } + if rec.complete != "" { + t.Error("the upload was completed despite a failed part") + } +} + +// S3 can refuse a completion inside a 200 response: it holds the connection +// open while it assembles the object, then writes an document. Decoding +// only the fields we want would swallow it and report a successful upload. +func TestPutObjectStreamDetectsAnErrorInsideA200(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case q.Has("uploads"): + _, _ = fmt.Fprint(w, `up-1`) + case q.Has("partNumber"): + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, `InternalErrorassembly failed`) + } + }) + + _, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader("payload")}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err == nil { + t.Fatal("an inside a 200 was reported as a successful upload") + } + if got := ErrorCode(err); got != "InternalError" { + t.Errorf("ErrorCode = %q, want InternalError", got) + } +} + +// A zero-length stream cannot be completed as a multipart upload — S3 rejects +// an empty part list — so it has to fall back to a plain PUT. +func TestPutObjectStreamHandlesAnEmptyStream(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + obj, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader("")}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if obj.Size != 0 { + t.Errorf("Size = %d, want 0", obj.Size) + } + if rec.aborted != 1 { + t.Errorf("aborts = %d, want the empty upload discarded", rec.aborted) + } + if rec.complete != "" { + t.Error("an empty part list was sent to CompleteMultipartUpload") + } +} + +// A part size the protocol forbids must be refused before anything is uploaded, +// not silently raised to the minimum. +func TestUploadOptionsRejectATooSmallPartSize(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + _, err := c.PutObjectStream(context.Background(), "b", "k", strings.NewReader("x"), + UploadOptions{Size: -1, PartSize: 1024}) + if err == nil || !strings.Contains(err.Error(), "at least") { + t.Errorf("err = %v, want it to name S3's minimum part size", err) + } + if rec.created != 0 { + t.Error("an upload was started despite invalid options") + } +} + +func TestUploadOptionsDefaults(t *testing.T) { + got, err := UploadOptions{Size: -1}.withDefaults() + if err != nil { + t.Fatal(err) + } + if got.PartSize != DefaultPartSize { + t.Errorf("PartSize = %d, want %d", got.PartSize, DefaultPartSize) + } + if got.Concurrency != DefaultConcurrency { + t.Errorf("Concurrency = %d, want %d", got.Concurrency, DefaultConcurrency) + } + if _, err := (UploadOptions{Concurrency: -1}).withDefaults(); err == nil { + t.Error("a negative concurrency was accepted") + } +} diff --git a/internal/s3/presign.go b/internal/s3/presign.go new file mode 100644 index 0000000..201d416 --- /dev/null +++ b/internal/s3/presign.go @@ -0,0 +1,97 @@ +package s3 + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// Presigned URLs. The credential moves out of the Authorization header and into +// the query string, which makes the whole request a URL anyone can fetch until +// it expires — the way to hand a colleague one backup without handing them a +// key. No request is made: this is pure local computation, so it works offline +// and leaves no trace on the store. +const ( + // MaxPresignExpiry is SigV4's ceiling on a presigned URL's lifetime. + MaxPresignExpiry = 7 * 24 * time.Hour + + // unsignedPayload replaces the body hash: the signature cannot cover a body + // the signer never sees, and for a GET there is none. + unsignedPayload = "UNSIGNED-PAYLOAD" +) + +// PresignGetObject returns a URL that fetches the object, valid for expiry. +// versionID selects a specific version, or "" for the current one. +// +// Anyone holding the URL can read the object until it expires, so treat one +// like a password: it is a bearer credential scoped to a single key. +func (c *Client) PresignGetObject(bucket, key, versionID string, expiry time.Duration) (string, error) { + return c.presign(http.MethodGet, bucket, key, versionQuery(versionID), expiry) +} + +// PresignPutObject returns a URL that uploads to the key, valid for expiry. The +// holder can write that one key and nothing else — which is how a machine with +// no S3 credentials at all delivers a dump. +func (c *Client) PresignPutObject(bucket, key string, expiry time.Duration) (string, error) { + return c.presign(http.MethodPut, bucket, key, nil, expiry) +} + +// presign builds a query-signed URL for one method/key pair. +func (c *Client) presign(method, bucket, key string, extra url.Values, expiry time.Duration) (string, error) { + if c.cfg.Anonymous { + return "", errors.New("cannot presign without credentials (--s3-anonymous is set)") + } + if key == "" { + return "", errors.New("presigning needs an object key") + } + if expiry <= 0 { + return "", fmt.Errorf("expiry must be positive, got %s", expiry) + } + if expiry > MaxPresignExpiry { + return "", fmt.Errorf("expiry must be at most %s (SigV4's limit), got %s", MaxPresignExpiry, expiry) + } + + utc := c.now().UTC() + amzDate, dateStamp := utc.Format(amzDateFormat), utc.Format(dateStampFormat) + scope := strings.Join([]string{dateStamp, c.cfg.Region, service, terminacc}, "/") + + q := url.Values{} + for k, vs := range extra { + q[k] = vs + } + q.Set("X-Amz-Algorithm", algorithm) + q.Set("X-Amz-Credential", c.cfg.AccessKey+"/"+scope) + q.Set("X-Amz-Date", amzDate) + q.Set("X-Amz-Expires", strconv.Itoa(int(expiry.Seconds()))) + // Only host is signed: a browser or curl sends nothing else predictable, + // and an unsigned header is one the holder cannot be forced to reproduce. + q.Set("X-Amz-SignedHeaders", "host") + + u := c.url(bucket, key, q) + + canonicalRequest := strings.Join([]string{ + method, + canonicalURI(u), + u.RawQuery, // already canonical: c.url renders it with canonicalQuery + "host:" + u.Host + "\n", + "host", + unsignedPayload, + }, "\n") + + stringToSign := strings.Join([]string{ + algorithm, + amzDate, + scope, + hexSHA256([]byte(canonicalRequest)), + }, "\n") + + sig := hmacSHA256(signingKey(c.cfg.SecretKey, dateStamp, c.cfg.Region), stringToSign) + // Appended rather than added to q before signing: the signature is not part + // of what it covers, and re-rendering the query would reorder the pairs. + u.RawQuery += "&X-Amz-Signature=" + hexEncode(sig) + return u.String(), nil +} diff --git a/internal/s3/presign_test.go b/internal/s3/presign_test.go new file mode 100644 index 0000000..3e1c5e7 --- /dev/null +++ b/internal/s3/presign_test.go @@ -0,0 +1,125 @@ +package s3 + +import ( + "strings" + "testing" + "time" +) + +// presignClient builds a client with fixed credentials and a frozen clock, so a +// presigned URL is reproducible and can be compared against a golden value. +func presignClient(t *testing.T) *Client { + t.Helper() + c, err := New(Config{ + Endpoint: "https://s3.example.com", + Region: "garage", + AccessKey: "GKtest", + SecretKey: "secret", + PathStyle: true, + }) + if err != nil { + t.Fatal(err) + } + c.now = func() time.Time { return time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) } + return c +} + +// The golden signatures come from an independent SigV4 query-signing +// implementation written from the AWS specification ("Authenticating Requests: +// Using Query Parameters"), not from this package. A URL koc computes is only +// useful if a *server* accepts it, so agreeing with a second implementation is +// the property worth asserting — a test that re-derived the value with these +// same functions would pass however wrong they both were. +func TestPresignGetObjectMatchesTheSpec(t *testing.T) { + c := presignClient(t) + + // A space in the key is the interesting case: net/url would encode it as + // "+" in a query and leave it alone in a path, either of which breaks the + // signature. + got, err := c.PresignGetObject("db-backups", "dump 1.sql.gz", "", time.Hour) + if err != nil { + t.Fatal(err) + } + const want = "https://s3.example.com/db-backups/dump%201.sql.gz?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Credential=GKtest%2F20260914%2Fgarage%2Fs3%2Faws4_request" + + "&X-Amz-Date=20260914T120000Z" + + "&X-Amz-Expires=3600" + + "&X-Amz-SignedHeaders=host" + + "&X-Amz-Signature=8190674d81d0695616b83fa8f54b73ba0d8fa202bc6b505ce52ba8b516771afc" + if got != want { + t.Errorf("presigned URL\n got %s\nwant %s", got, want) + } +} + +func TestPresignPutObjectMatchesTheSpec(t *testing.T) { + c := presignClient(t) + + got, err := c.PresignPutObject("db-backups", "incoming.bin", 15*time.Minute) + if err != nil { + t.Fatal(err) + } + const wantSig = "X-Amz-Signature=1b17213576b7451158c6054284e65dfdea8ea1c046a947af8d8993896feedf7d" + if !strings.HasSuffix(got, wantSig) { + t.Errorf("presigned PUT URL %s does not end with the expected signature", got) + } + if !strings.Contains(got, "X-Amz-Expires=900") { + t.Errorf("presigned PUT URL %s does not carry the 15m expiry", got) + } +} + +// A version ID is an ordinary query parameter, so it has to be signed with the +// rest — in canonical (sorted) order, which puts it after the X-Amz-* names. +func TestPresignGetObjectWithVersion(t *testing.T) { + c := presignClient(t) + + got, err := c.PresignGetObject("db-backups", "dump.sql.gz", "v2", time.Hour) + if err != nil { + t.Fatal(err) + } + const wantTail = "&X-Amz-SignedHeaders=host&versionId=v2" + + "&X-Amz-Signature=5d68330c6f95ebef61e3e373add6affbd8c3919a94a83b895c2360e6ee1630cd" + if !strings.HasSuffix(got, wantTail) { + t.Errorf("presigned URL %s does not end with %s", got, wantTail) + } +} + +func TestPresignRejectsBadExpiry(t *testing.T) { + c := presignClient(t) + + for _, tc := range []struct { + name string + expiry time.Duration + want string + }{ + {"zero", 0, "must be positive"}, + {"negative", -time.Second, "must be positive"}, + {"past the ceiling", MaxPresignExpiry + time.Second, "at most"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := c.PresignGetObject("b", "k", "", tc.expiry); err == nil || + !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// Anonymous mode has no secret to sign with, so presigning must say so rather +// than emit a URL signed with an empty key. +func TestPresignRefusedWhenAnonymous(t *testing.T) { + c, err := New(Config{Endpoint: "https://s3.example.com", Anonymous: true}) + if err != nil { + t.Fatal(err) + } + if _, err := c.PresignGetObject("b", "k", "", time.Hour); err == nil || + !strings.Contains(err.Error(), "anonymous") { + t.Errorf("err = %v, want it to name --s3-anonymous", err) + } +} + +func TestPresignNeedsAKey(t *testing.T) { + if _, err := presignClient(t).PresignGetObject("b", "", "", time.Hour); err == nil { + t.Error("presigning a bucket with no key was accepted") + } +} diff --git a/internal/s3/retry.go b/internal/s3/retry.go new file mode 100644 index 0000000..6d82b2a --- /dev/null +++ b/internal/s3/retry.go @@ -0,0 +1,91 @@ +package s3 + +import ( + "context" + "errors" + "net/http" + "time" +) + +// Retry policy. A koc S3 call can be a multi-gigabyte transfer over a cluster +// network, where a reset connection or a momentary 503 from one gateway node is +// routine rather than exceptional — s5cmd retries ten times by default for the +// same reason. Without this a transient blip failed the whole command and, on an +// upload, left the object half-written. +const ( + // retryBaseDelay is the first backoff interval; each further attempt doubles + // it up to retryMaxDelay. + retryBaseDelay = 200 * time.Millisecond + retryMaxDelay = 10 * time.Second +) + +// retryableCodes are the S3 error codes that mean "ask again", as opposed to +// "you asked wrong". SlowDown and RequestTimeout are the two a healthy store +// still emits under load. RequestTimeTooSkewed is deliberately absent: it is a +// clock problem, and repeating the request cannot fix it. +var retryableCodes = map[string]bool{ + "InternalError": true, + "ServiceUnavailable": true, + "SlowDown": true, + "RequestTimeout": true, +} + +// nonRetryableError marks an error raised after the response headers were accepted, +// i.e. by the sink while it consumed the body. Those are never replayed: the +// sink has already written some of the object to a file or to stdout, and a +// second attempt would append a duplicate prefix rather than resume. +type nonRetryableError struct{ err error } + +func (e *nonRetryableError) Error() string { return e.err.Error() } +func (e *nonRetryableError) Unwrap() error { return e.err } + +// retryable reports whether err is worth another attempt. +func retryable(err error) bool { + var sink *nonRetryableError + if errors.As(err, &sink) { + return false + } + + var ae *APIError + if !errors.As(err, &ae) { + // Not an answer from S3 at all: a dial failure, a reset, or EOF before + // the headers arrived. Nothing was applied, so it is safe to repeat. + return true + } + if retryableCodes[ae.Code] { + return true + } + switch ae.StatusCode { + case http.StatusRequestTimeout, http.StatusTooManyRequests: + return true + } + return ae.StatusCode >= http.StatusInternalServerError +} + +// backoff waits before attempt n (1 = the first retry), or returns the +// context's error if the caller gave up first. The delay carries a little +// jitter so a pool of --concurrency workers that all hit the same 503 do not +// all come back in the same millisecond; it is read off the clock rather than +// from math/rand because spreading requests is the only property needed and an +// unseeded PRNG in a short-lived CLI gives no better one. +func (c *Client) backoff(ctx context.Context, n int) error { + d := c.retryBase + for range n - 1 { + if d *= 2; d >= retryMaxDelay { + d = retryMaxDelay + break + } + } + if spread := int64(d / 4); spread > 0 { + d += time.Duration(c.now().UnixNano() % spread) + } + + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/s3/retry_test.go b/internal/s3/retry_test.go new file mode 100644 index 0000000..f1538e3 --- /dev/null +++ b/internal/s3/retry_test.go @@ -0,0 +1,234 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +// retryClient is newTestClient with retries on and a backoff short enough that +// a test does not spend real time asleep. +func retryClient(t *testing.T, retries int, h http.HandlerFunc) *Client { + t.Helper() + c := newTestClient(t, h) + c.cfg.MaxRetries = retries + c.retryBase = time.Microsecond + return c +} + +func TestRetryableClassification(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"transport failure", errors.New("dial tcp: connection refused"), true}, + {"500", &APIError{StatusCode: 500}, true}, + {"503 SlowDown", &APIError{StatusCode: 503, Code: "SlowDown"}, true}, + {"429", &APIError{StatusCode: 429}, true}, + {"408", &APIError{StatusCode: 408}, true}, + {"403", &APIError{StatusCode: 403, Code: "AccessDenied"}, false}, + {"404", &APIError{StatusCode: 404, Code: "NoSuchKey"}, false}, + {"clock skew is not retryable", &APIError{StatusCode: 403, Code: "RequestTimeTooSkewed"}, false}, + {"a sink failure is never replayed", &nonRetryableError{err: errors.New("short write")}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := retryable(tc.err); got != tc.want { + t.Errorf("retryable(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +func TestDoRetriesA503ThenSucceeds(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 3, func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = fmt.Fprint(w, `SlowDown`) + return + } + _, _ = fmt.Fprint(w, ``) + }) + + if _, err := c.ListBuckets(context.Background()); err != nil { + t.Fatalf("a retryable failure was not retried: %v", err) + } + if got := calls.Load(); got != 3 { + t.Errorf("attempts = %d, want 3", got) + } +} + +// The retry budget is finite: past it the last error reaches the caller. +func TestDoGivesUpAfterMaxRetries(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 2, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusInternalServerError) + }) + + if _, err := c.ListBuckets(context.Background()); err == nil { + t.Fatal("expected the failure to be reported once the budget ran out") + } + if got := calls.Load(); got != 3 { + t.Errorf("attempts = %d, want 1 try + 2 retries", got) + } +} + +func TestDoDoesNotRetryA403(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 5, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `AccessDenied`) + }) + + if _, err := c.ListBuckets(context.Background()); err == nil { + t.Fatal("expected an error") + } + if got := calls.Load(); got != 1 { + t.Errorf("attempts = %d, want 1 — a 403 is not worth repeating", got) + } +} + +// A retried PUT has to send the same bytes again, which is the whole reason +// request.body is an io.ReadSeeker. +func TestDoRewindsTheBodyOnRetry(t *testing.T) { + var bodies []string + var calls atomic.Int32 + c := retryClient(t, 1, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(body)) + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set("ETag", `"abc"`) + w.WriteHeader(http.StatusOK) + }) + + payload := "the whole dump" + obj, err := c.PutObject(context.Background(), "b", "k", strings.NewReader(payload), + int64(len(payload)), "application/octet-stream") + if err != nil { + t.Fatal(err) + } + if obj.ETag != "abc" { + t.Errorf("ETag = %q, want abc", obj.ETag) + } + if len(bodies) != 2 || bodies[0] != payload || bodies[1] != payload { + t.Errorf("bodies = %q, want the same payload twice", bodies) + } +} + +// A body read from the caller's current offset must rewind to *that* offset, not +// to byte zero: an upload of one slice of a file would otherwise re-send the +// wrong bytes. +func TestDoRewindsToTheCallerOffset(t *testing.T) { + var bodies []string + var calls atomic.Int32 + c := retryClient(t, 1, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(body)) + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + }) + + src := bytes.NewReader([]byte("SKIPMEpayload")) + if _, err := src.Seek(6, io.SeekStart); err != nil { + t.Fatal(err) + } + if _, err := c.PutObject(context.Background(), "b", "k", src, 7, ""); err != nil { + t.Fatal(err) + } + for i, got := range bodies { + if got != "payload" { + t.Errorf("attempt %d sent %q, want payload", i+1, got) + } + } +} + +// A failure raised while the sink was consuming the body must not be replayed: +// the bytes it already wrote to a file or to stdout cannot be taken back. +func TestDoDoesNotRetryASinkFailure(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 5, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + _, _ = w.Write([]byte("payload")) + }) + + _, err := c.GetObject(context.Background(), "b", "k", "", failWriter{}) + if err == nil { + t.Fatal("expected the sink failure to be reported") + } + if got := calls.Load(); got != 1 { + t.Errorf("attempts = %d, want 1 — a partly written destination cannot be replayed", got) + } + // The caller must see its own error, not the internal marker. + var marker *nonRetryableError + if errors.As(err, &marker) { + t.Errorf("error %v still carries the internal retry marker", err) + } +} + +// A cancelled context ends the loop instead of sleeping out the budget. +func TestDoStopsOnContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var calls atomic.Int32 + c := retryClient(t, 5, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + cancel() + w.WriteHeader(http.StatusServiceUnavailable) + }) + + if _, err := c.ListBuckets(ctx); err == nil { + t.Fatal("expected an error") + } + if got := calls.Load(); got != 1 { + t.Errorf("attempts = %d, want 1 once the caller gave up", got) + } +} + +// Anonymous mode must send no credential at all — that is the point of it. +func TestAnonymousSendsNoSignature(t *testing.T) { + var auth, sha string + srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + auth, sha = r.Header.Get("Authorization"), r.Header.Get("X-Amz-Content-Sha256") + _, _ = fmt.Fprint(w, `false`) + }) + + c, err := New(Config{Endpoint: srv.Endpoint(), Anonymous: true, PathStyle: true}) + if err != nil { + t.Fatal(err) + } + if _, err := c.ListObjects(context.Background(), "public", ListOptions{}); err != nil { + t.Fatal(err) + } + if auth != "" { + t.Errorf("Authorization = %q, want none", auth) + } + if sha != "" { + t.Errorf("X-Amz-Content-Sha256 = %q, want none", sha) + } +} + +// Anonymous mode is the only one that needs no credentials; every other mode +// must still insist on them. +func TestNewRequiresCredentialsUnlessAnonymous(t *testing.T) { + if _, err := New(Config{Endpoint: "https://s3.example.com"}); err == nil { + t.Error("a client with no credentials was accepted") + } + if _, err := New(Config{Endpoint: "https://s3.example.com", Anonymous: true}); err != nil { + t.Errorf("anonymous client rejected: %v", err) + } +} diff --git a/internal/s3/s3.go b/internal/s3/s3.go index fae92f9..9e92832 100644 --- a/internal/s3/s3.go +++ b/internal/s3/s3.go @@ -70,6 +70,14 @@ type Config struct { // cap. A wedged endpoint is still caught by responseHeaderTimeout. Timeout time.Duration + // MaxRetries is how many further attempts a retryable failure gets (0 = + // none). See retry.go for what counts as retryable. + MaxRetries int + + // Anonymous sends requests unsigned, for a bucket granted to everyone. It + // is the only mode in which no credentials are required. + Anonymous bool + Debug bool } @@ -81,6 +89,9 @@ type Client struct { // now is the signing clock, overridden in tests. now func() time.Time + // retryBase is the first backoff interval, overridden in tests so a retry + // case does not spend real time asleep. + retryBase time.Duration } // New validates the config and builds the client. It performs no network I/O: @@ -89,8 +100,8 @@ func New(cfg Config) (*Client, error) { if cfg.Endpoint == "" { return nil, errors.New("S3 endpoint is required (--s3-endpoint / AWS_ENDPOINT_URL)") } - if cfg.AccessKey == "" || cfg.SecretKey == "" { - return nil, errors.New("S3 credentials are required (--s3-access-key/--s3-secret-key, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY, or --s3-creds-from-ns)") + if !cfg.Anonymous && (cfg.AccessKey == "" || cfg.SecretKey == "") { + return nil, errors.New("S3 credentials are required (--s3-access-key/--s3-secret-key, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY, or --s3-creds-from-ns); pass --s3-anonymous for a publicly readable bucket") } if cfg.Region == "" { cfg.Region = DefaultRegion @@ -118,10 +129,11 @@ func New(cfg Config) (*Client, error) { } return &Client{ - cfg: cfg, - hc: newHTTPClient(tlsCfg, cfg.Timeout), - base: base, - now: time.Now, + cfg: cfg, + hc: newHTTPClient(tlsCfg, cfg.Timeout), + base: base, + now: time.Now, + retryBase: retryBaseDelay, }, nil } @@ -216,6 +228,39 @@ type Object struct { LastModified time.Time ETag string StorageClass string + + // IsPrefix marks a CommonPrefixes entry rather than a key: what a + // delimited listing reports in place of everything below it. Only Key is + // meaningful on one. + IsPrefix bool + + // VersionID, IsLatest and DeleteMarker are filled only by a versioned + // listing (ListOptions.Versions). A store without versioning — Garage has + // none — never sets them. + VersionID string + IsLatest bool + DeleteMarker bool +} + +// ListOptions narrows a listing. The zero value lists every current object in +// the bucket. +type ListOptions struct { + // Prefix restricts the listing to keys starting with it. + Prefix string + + // Delimiter collapses everything after the next occurrence of it into a + // single CommonPrefixes entry, which is how "/" turns a flat keyspace into + // one directory level. Such entries arrive as Objects with IsPrefix set. + Delimiter string + + // Limit caps the number of entries reported (0 = no cap). It is a hard + // result cap, not merely a page size — the same rule the rest of koc's + // --limit flags follow. + Limit int + + // Versions lists every version and delete marker of each key instead of + // only the current object, via the ?versions endpoint. + Versions bool } // ObjectInfo describes a single object, as returned by HeadObject (and by @@ -227,6 +272,7 @@ type ObjectInfo struct { LastModified time.Time ETag string ContentType string + VersionID string // set only by a store with versioning on Metadata map[string]string // x-amz-meta-*, with the prefix stripped } @@ -314,15 +360,18 @@ type listObjectsPage struct { ETag string `xml:"ETag"` StorageClass string `xml:"StorageClass"` } `xml:"Contents"` + CommonPrefixes []struct { + Prefix string `xml:"Prefix"` + } `xml:"CommonPrefixes"` } // ListObjects lists objects in a bucket, following continuation tokens until the // server says there are no more. limit caps the number of objects returned (0 = // no cap) and is applied as a hard result cap, not merely as a page size — the // same rule the rest of koc's --limit flags follow. -func (c *Client) ListObjects(ctx context.Context, bucket, prefix string, limit int) ([]Object, error) { +func (c *Client) ListObjects(ctx context.Context, bucket string, opts ListOptions) ([]Object, error) { var out []Object - err := c.ListObjectsFunc(ctx, bucket, prefix, limit, func(o Object) error { + err := c.ListObjectsFunc(ctx, bucket, opts, func(o Object) error { out = append(out, o) return nil }) @@ -337,35 +386,22 @@ func (c *Client) ListObjects(ctx context.Context, bucket, prefix string, limit i // returned as-is. Callers that only fold over the listing (summing sizes, // deleting as they go) use this so a bucket with a million keys costs one page // of memory instead of all of them. -func (c *Client) ListObjectsFunc(ctx context.Context, bucket, prefix string, limit int, fn func(Object) error) error { +func (c *Client) ListObjectsFunc(ctx context.Context, bucket string, opts ListOptions, fn func(Object) error) error { + if opts.Versions { + return c.listObjectVersions(ctx, bucket, opts, fn) + } + token, seen := "", 0 for { - q := url.Values{"list-type": {"2"}, "max-keys": {strconv.Itoa(pageSize(limit, seen))}} - if prefix != "" { - q.Set("prefix", prefix) - } - if token != "" { - q.Set("continuation-token", token) - } - var page listObjectsPage - if err := c.getXML(ctx, c.url(bucket, "", q), &page); err != nil { + u := c.url(bucket, "", listObjectsQuery(opts, token, seen)) + if err := c.getXML(ctx, u, &page); err != nil { return err } - for _, o := range page.Contents { - err := fn(Object{ - Key: o.Key, - Size: o.Size, - LastModified: parseS3Time(o.LastModified), - ETag: strings.Trim(o.ETag, `"`), - StorageClass: o.StorageClass, - }) - if err != nil { - return err - } - if seen++; limit > 0 && seen >= limit { - return nil - } + + done, err := emitPage(&page, opts.Limit, &seen, fn) + if err != nil || done { + return err } if !page.IsTruncated || page.NextContinuationToken == "" { return nil @@ -374,6 +410,54 @@ func (c *Client) ListObjectsFunc(ctx context.Context, bucket, prefix string, lim } } +// listObjectsQuery renders one ListObjectsV2 request's parameters. +func listObjectsQuery(opts ListOptions, token string, seen int) url.Values { + q := url.Values{"list-type": {"2"}, "max-keys": {strconv.Itoa(pageSize(opts.Limit, seen))}} + if opts.Prefix != "" { + q.Set("prefix", opts.Prefix) + } + if opts.Delimiter != "" { + q.Set("delimiter", opts.Delimiter) + } + if token != "" { + q.Set("continuation-token", token) + } + return q +} + +// emitPage hands one page's entries to fn, counting them against limit. It +// reports done once the limit is reached, so the caller stops paging. +// +// CommonPrefixes come first so a delimited listing reads like a directory: the +// subtrees, then the keys at this level. +func emitPage(page *listObjectsPage, limit int, seen *int, fn func(Object) error) (done bool, err error) { + for _, p := range page.CommonPrefixes { + if err := fn(Object{Key: p.Prefix, IsPrefix: true}); err != nil { + return false, err + } + if *seen++; limit > 0 && *seen >= limit { + return true, nil + } + } + for _, o := range page.Contents { + err := fn(Object{ + Key: o.Key, + Size: o.Size, + LastModified: parseS3Time(o.LastModified), + ETag: strings.Trim(o.ETag, `"`), + StorageClass: o.StorageClass, + IsLatest: true, + }) + if err != nil { + return false, err + } + if *seen++; limit > 0 && *seen >= limit { + return true, nil + } + } + return false, nil +} + // pageSize asks for a full page unless a limit means fewer keys are wanted. func pageSize(limit, have int) int { if limit <= 0 || limit-have > maxKeysPerPage { @@ -382,12 +466,30 @@ func pageSize(limit, have int) int { return limit - have } -// HeadObject fetches an object's metadata without its body. -func (c *Client) HeadObject(ctx context.Context, bucket, key string) (*ObjectInfo, error) { +// versionQuery addresses one specific version of a key, or the current one when +// versionID is empty. +func versionQuery(versionID string) url.Values { + if versionID == "" { + return nil + } + return url.Values{"versionId": {versionID}} +} + +// HeadBucket reports whether the bucket exists and the credentials may reach +// it, without listing anything. It is the cheapest possible probe: a bodiless +// request that costs the server no listing work. +func (c *Client) HeadBucket(ctx context.Context, bucket string) error { + return c.do(ctx, request{method: http.MethodHead, url: c.url(bucket, "", nil), payloadHash: emptySHA256}, drainBody) +} + +// HeadObject fetches an object's metadata without its body. versionID selects a +// specific version, or "" for the current one. +func (c *Client) HeadObject(ctx context.Context, bucket, key, versionID string) (*ObjectInfo, error) { var info *ObjectInfo - err := c.do(ctx, request{method: http.MethodHead, url: c.url(bucket, key, nil), payloadHash: emptySHA256}, + err := c.do(ctx, request{method: http.MethodHead, url: c.url(bucket, key, versionQuery(versionID)), payloadHash: emptySHA256}, func(resp *http.Response) error { info = objectInfoFromHeader(bucket, key, resp) + info.VersionID = resp.Header.Get("x-amz-version-id") return nil }) if err != nil { @@ -398,9 +500,9 @@ func (c *Client) HeadObject(ctx context.Context, bucket, key string) (*ObjectInf // GetObject streams an object's body to w and returns the number of bytes // written. -func (c *Client) GetObject(ctx context.Context, bucket, key string, w io.Writer) (int64, error) { +func (c *Client) GetObject(ctx context.Context, bucket, key, versionID string, w io.Writer) (int64, error) { var n int64 - err := c.do(ctx, request{method: http.MethodGet, url: c.url(bucket, key, nil), payloadHash: emptySHA256}, + err := c.do(ctx, request{method: http.MethodGet, url: c.url(bucket, key, versionQuery(versionID)), payloadHash: emptySHA256}, func(resp *http.Response) error { var cerr error n, cerr = io.Copy(w, resp.Body) @@ -415,16 +517,18 @@ func (c *Client) GetObject(ctx context.Context, bucket, key string, w io.Writer) // DeleteObject removes one object. S3 answers 204 for a key that was never // there, so a delete is idempotent and "no such key" is not reported — the // caller asked for the key to be gone, and it is. -func (c *Client) DeleteObject(ctx context.Context, bucket, key string) error { - return c.do(ctx, request{method: http.MethodDelete, url: c.url(bucket, key, nil), payloadHash: emptySHA256}, drainBody) +func (c *Client) DeleteObject(ctx context.Context, bucket, key, versionID string) error { + return c.do(ctx, request{method: http.MethodDelete, url: c.url(bucket, key, versionQuery(versionID)), payloadHash: emptySHA256}, drainBody) } -// PutObject uploads body as a single part. The reader must be seekable because -// SigV4 signs a hash of the payload: the body is read once to hash it, then -// rewound and sent. That rules out streaming from a pipe, and is the reason -// there is no multipart support here — for the backup-sized objects koc moves, -// a single signed PUT is enough, and an S3 server's own single-part ceiling -// (5 GiB) applies. +// PutObject uploads body as a single request. The reader must be seekable +// because SigV4 signs a hash of the payload: the body is read once to hash it, +// then rewound and sent — which is also what lets a retry replay it. +// +// That rules out a pipe, and the server's single-part ceiling (5 GiB +// everywhere) applies. Callers that cannot promise either use +// PutObjectStream, which picks this path when the source is seekable and fits +// one part and goes multipart otherwise. func (c *Client) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker, size int64, contentType string) (*ObjectInfo, error) { hash, err := hashSeeker(body) if err != nil { @@ -555,22 +659,89 @@ func (c *Client) getXML(ctx context.Context, u *url.URL, out any) error { type request struct { method string url *url.URL - body io.Reader // nil for GET/HEAD + // body is seekable rather than a plain io.Reader so a retried attempt can + // replay it — SigV4 already requires the payload to be readable twice, so + // this costs nothing extra. + body io.ReadSeeker // nil for GET/HEAD // payloadHash is the SigV4 hash of body; emptySHA256 when there is none. payloadHash string size int64 // Content-Length; only read when body != nil header map[string]string // extra request headers, e.g. Content-Type } -// do signs and performs one request, then hands the still-open response to sink. -// Owning the response lifecycle here (rather than returning it) keeps every body -// closed on every path, including the error ones. +// do performs a request, retrying a retryable failure up to cfg.MaxRetries +// times with exponential backoff. An error raised by sink is never retried — +// see nonRetryableError. // // Bodies are never logged, even under --debug: a request body is object data and // a response body can be too, while the headers carry the signature. Method, // path and status are enough to debug a 403. func (c *Client) do(ctx context.Context, r request, sink func(*http.Response) error) error { - req, err := http.NewRequestWithContext(ctx, r.method, r.url.String(), r.body) + start, err := bodyOffset(r.body) + if err != nil { + return err + } + + for attempt := 0; ; attempt++ { + if attempt > 0 { + if err := c.backoff(ctx, attempt); err != nil { + return err + } + if r.body != nil { + if _, err := r.body.Seek(start, io.SeekStart); err != nil { + return fmt.Errorf("rewinding request body for retry: %w", err) + } + } + } + + err := c.attempt(ctx, r, sink) + if err == nil { + return nil + } + if attempt >= c.cfg.MaxRetries || ctx.Err() != nil || !retryable(err) { + return unwrapSink(err) + } + if c.cfg.Debug { + fmt.Fprintf(os.Stderr, "s3: %s %s failed (%v); retrying\n", r.method, r.url.RequestURI(), err) + } + } +} + +// bodyOffset records where a replayable body starts, so a retry rewinds to the +// caller's position rather than to byte zero of, say, an open file. +func bodyOffset(body io.ReadSeeker) (int64, error) { + if body == nil { + return 0, nil + } + at, err := body.Seek(0, io.SeekCurrent) + if err != nil { + return 0, fmt.Errorf("seeking request body: %w", err) + } + return at, nil +} + +// unwrapSink strips the marker retryable() keys off, so callers see the error +// the sink actually returned. +func unwrapSink(err error) error { + var sink *nonRetryableError + if errors.As(err, &sink) { + return sink.err + } + return err +} + +// attempt performs one signed request, then hands the still-open response to +// sink. Owning the response lifecycle here (rather than returning it) keeps +// every body closed on every path, including the error ones. +func (c *Client) attempt(ctx context.Context, r request, sink func(*http.Response) error) error { + // A nil io.ReadSeeker in an interface-typed argument is not a nil + // io.Reader, and net/http treats the difference as "body of unknown + // length" — so the nil case has to be passed explicitly. + var rc io.Reader + if r.body != nil { + rc = r.body + } + req, err := http.NewRequestWithContext(ctx, r.method, r.url.String(), rc) if err != nil { return err } @@ -580,7 +751,9 @@ func (c *Client) do(ctx context.Context, r request, sink func(*http.Response) er if r.body != nil { req.ContentLength = r.size } - c.sign(req, r.payloadHash, c.now()) + if !c.cfg.Anonymous { + c.sign(req, r.payloadHash, c.now()) + } resp, err := c.hc.Do(req) if err != nil { @@ -594,7 +767,10 @@ func (c *Client) do(ctx context.Context, r request, sink func(*http.Response) er if resp.StatusCode < 200 || resp.StatusCode >= 300 { return newAPIError(r.method, r.url.Path, resp) } - return sink(resp) + if err := sink(resp); err != nil { + return &nonRetryableError{err: err} + } + return nil } // newAPIError builds an APIError from a failed response, parsing S3's XML error diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index 4f5a33e..1ecb422 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -121,7 +121,7 @@ func TestListObjectsPaging(t *testing.T) { `) }) - objs, err := c.ListObjects(context.Background(), "db-backups", "e2e-", 0) + objs, err := c.ListObjects(context.Background(), "db-backups", ListOptions{Prefix: "e2e-"}) if err != nil { t.Fatal(err) } @@ -159,7 +159,7 @@ func TestListObjectsLimit(t *testing.T) { `) }) - objs, err := c.ListObjects(context.Background(), "b", "", 1) + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Limit: 1}) if err != nil { t.Fatal(err) } @@ -188,7 +188,7 @@ func TestHeadObject(t *testing.T) { w.WriteHeader(http.StatusOK) }) - info, err := c.HeadObject(context.Background(), "db-backups", "e2e-a.sql.gz") + info, err := c.HeadObject(context.Background(), "db-backups", "e2e-a.sql.gz", "") if err != nil { t.Fatal(err) } @@ -216,7 +216,7 @@ func TestGetObject(t *testing.T) { }) var buf bytes.Buffer - n, err := c.GetObject(context.Background(), "db-backups", "dir/a b.txt", &buf) + n, err := c.GetObject(context.Background(), "db-backups", "dir/a b.txt", "", &buf) if err != nil { t.Fatal(err) } @@ -296,7 +296,7 @@ func TestAPIError(t *testing.T) { `/db-backups/garage`)) }) - _, err := c.ListObjects(context.Background(), "db-backups", "", 0) + _, err := c.ListObjects(context.Background(), "db-backups", ListOptions{}) var apiErr *APIError if !asAPIError(err, &apiErr) { t.Fatalf("err = %v (%T), want *APIError", err, err) @@ -311,7 +311,7 @@ func TestAPIError(t *testing.T) { t.Error("AccessDenied must not read as not-found") } - _, err = c.HeadObject(context.Background(), "db-backups", "missing") + _, err = c.HeadObject(context.Background(), "db-backups", "missing", "") if !IsNotFound(err) { t.Errorf("bodiless 404 = %v, want IsNotFound", err) } diff --git a/internal/s3/sign.go b/internal/s3/sign.go index ed471ee..45316da 100644 --- a/internal/s3/sign.go +++ b/internal/s3/sign.go @@ -204,3 +204,7 @@ func hexSHA256(b []byte) string { sum := sha256.Sum256(b) return hex.EncodeToString(sum[:]) } + +// hexEncode names encoding/hex at the one call site outside this file, so +// presign.go does not import it only to render a signature. +func hexEncode(b []byte) string { return hex.EncodeToString(b) } diff --git a/internal/s3/transport.go b/internal/s3/transport.go index 8d5e6d1..4ed3972 100644 --- a/internal/s3/transport.go +++ b/internal/s3/transport.go @@ -19,6 +19,12 @@ const ( // maxRedirects caps a redirect chain. See sameHostRedirect for what travels // with one. maxRedirects = 5 + + // abortTimeout bounds the cleanup request a failed multipart upload sends. + // It runs on a fresh context — the caller's is usually already cancelled — + // so it needs a deadline of its own or a wedged endpoint would hang the + // error path. + abortTimeout = 30 * time.Second ) // credentialHeaders are dropped when a redirect leaves the origin host. Go's own diff --git a/internal/s3/versioning.go b/internal/s3/versioning.go new file mode 100644 index 0000000..1a757f8 --- /dev/null +++ b/internal/s3/versioning.go @@ -0,0 +1,185 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" +) + +// Object versioning. Note that Garage — koc's primary target — does not +// implement it: its GetBucketVersioning is a stub that always answers "not +// enabled", and enabling it fails. These calls are here for the other stores +// koc can be pointed at (AWS, Ceph RGW, MinIO), and for the operator who needs +// to confirm which of the two they are talking to. +const ( + // VersioningEnabled and VersioningSuspended are the two states S3 defines. + // A bucket that was never configured reports neither, which this package + // renders as VersioningUnversioned. + VersioningEnabled = "Enabled" + VersioningSuspended = "Suspended" + VersioningUnversioned = "Unversioned" +) + +// versioningConfiguration is the body of both versioning calls. +type versioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status,omitempty"` +} + +// GetBucketVersioning reports the bucket's versioning state, normalising the +// "never configured" empty answer to VersioningUnversioned so a caller never +// has to special-case the empty string. +func (c *Client) GetBucketVersioning(ctx context.Context, bucket string) (string, error) { + var cfg versioningConfiguration + if err := c.getXML(ctx, c.url(bucket, "", url.Values{"versioning": {""}}), &cfg); err != nil { + return "", err + } + if cfg.Status == "" { + return VersioningUnversioned, nil + } + return cfg.Status, nil +} + +// SetBucketVersioning enables or suspends versioning. status must be +// VersioningEnabled or VersioningSuspended: S3 has no call that returns a +// bucket to never-versioned, which is why VersioningUnversioned is not accepted +// here. +func (c *Client) SetBucketVersioning(ctx context.Context, bucket, status string) error { + switch status { + case VersioningEnabled, VersioningSuspended: + default: + return fmt.Errorf("versioning status must be %q or %q, not %q", + VersioningEnabled, VersioningSuspended, status) + } + + body, err := xml.Marshal(versioningConfiguration{Status: status}) + if err != nil { + return fmt.Errorf("encoding versioning body: %w", err) + } + return c.do(ctx, request{ + method: http.MethodPut, + url: c.url(bucket, "", url.Values{"versioning": {""}}), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + header: map[string]string{ + "Content-Type": "application/xml", + "Content-MD5": contentMD5(body), + }, + }, drainBody) +} + +// listVersionsPage is one ListObjectVersions response. +type listVersionsPage struct { + IsTruncated bool `xml:"IsTruncated"` + NextKeyMarker string `xml:"NextKeyMarker"` + NextVersionIDMarker string `xml:"NextVersionIdMarker"` + Versions []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + Size int64 `xml:"Size"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + StorageClass string `xml:"StorageClass"` + } `xml:"Version"` + DeleteMarkers []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified string `xml:"LastModified"` + } `xml:"DeleteMarker"` + CommonPrefixes []struct { + Prefix string `xml:"Prefix"` + } `xml:"CommonPrefixes"` +} + +// listObjectVersions walks the ?versions endpoint, which pages on a key marker +// plus a version-id marker rather than a continuation token. +// +// A page's and elements interleave on the wire, and +// decoding them into two slices loses that order, so each page is re-sorted by +// key and then newest-first. The result is the presentation a reader wants +// anyway, and it is deterministic, which the wire order across a page boundary +// is not. +func (c *Client) listObjectVersions(ctx context.Context, bucket string, opts ListOptions, + fn func(Object) error) error { + keyMarker, versionMarker, seen := "", "", 0 + for { + q := url.Values{"versions": {""}, "max-keys": {strconv.Itoa(pageSize(opts.Limit, seen))}} + if opts.Prefix != "" { + q.Set("prefix", opts.Prefix) + } + if opts.Delimiter != "" { + q.Set("delimiter", opts.Delimiter) + } + if keyMarker != "" { + q.Set("key-marker", keyMarker) + } + if versionMarker != "" { + q.Set("version-id-marker", versionMarker) + } + + var page listVersionsPage + if err := c.getXML(ctx, c.url(bucket, "", q), &page); err != nil { + return err + } + for _, obj := range versionEntries(&page) { + if err := fn(obj); err != nil { + return err + } + if seen++; opts.Limit > 0 && seen >= opts.Limit { + return nil + } + } + if !page.IsTruncated || (page.NextKeyMarker == "" && page.NextVersionIDMarker == "") { + return nil + } + keyMarker, versionMarker = page.NextKeyMarker, page.NextVersionIDMarker + } +} + +// versionEntries flattens one page into the order described on +// listObjectVersions. +func versionEntries(page *listVersionsPage) []Object { + out := make([]Object, 0, len(page.Versions)+len(page.DeleteMarkers)+len(page.CommonPrefixes)) + for _, p := range page.CommonPrefixes { + out = append(out, Object{Key: p.Prefix, IsPrefix: true}) + } + for _, v := range page.Versions { + out = append(out, Object{ + Key: v.Key, + Size: v.Size, + LastModified: parseS3Time(v.LastModified), + ETag: strings.Trim(v.ETag, `"`), + StorageClass: v.StorageClass, + VersionID: v.VersionID, + IsLatest: v.IsLatest, + }) + } + for _, d := range page.DeleteMarkers { + out = append(out, Object{ + Key: d.Key, + LastModified: parseS3Time(d.LastModified), + VersionID: d.VersionID, + IsLatest: d.IsLatest, + DeleteMarker: true, + }) + } + + prefixes := len(page.CommonPrefixes) + sort.SliceStable(out[prefixes:], func(i, j int) bool { + a, b := out[prefixes+i], out[prefixes+j] + if a.Key != b.Key { + return a.Key < b.Key + } + return a.LastModified.After(b.LastModified) + }) + return out +} diff --git a/internal/s3/versioning_test.go b/internal/s3/versioning_test.go new file mode 100644 index 0000000..8366e96 --- /dev/null +++ b/internal/s3/versioning_test.go @@ -0,0 +1,260 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" +) + +// A bucket that was never configured answers an empty document, which must be +// normalised rather than handed to the caller as an empty string. +func TestGetBucketVersioning(t *testing.T) { + for _, tc := range []struct { + name, body, want string + }{ + {"enabled", `Enabled`, VersioningEnabled}, + {"suspended", `Suspended`, VersioningSuspended}, + {"never configured", ``, VersioningUnversioned}, + } { + t.Run(tc.name, func(t *testing.T) { + var hasParam bool + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + hasParam = r.URL.Query().Has("versioning") + _, _ = fmt.Fprint(w, tc.body) + }) + + got, err := c.GetBucketVersioning(context.Background(), "b") + if err != nil { + t.Fatal(err) + } + if !hasParam { + t.Error("the request did not carry ?versioning") + } + if got != tc.want { + t.Errorf("status = %q, want %q", got, tc.want) + } + }) + } +} + +func TestSetBucketVersioning(t *testing.T) { + var gotMethod, gotBody, gotMD5 string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + body, _ := io.ReadAll(r.Body) + gotMethod, gotBody, gotMD5 = r.Method, string(body), r.Header.Get("Content-MD5") + w.WriteHeader(http.StatusOK) + }) + + if err := c.SetBucketVersioning(context.Background(), "b", VersioningEnabled); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut { + t.Errorf("method = %q, want PUT", gotMethod) + } + want := "Enabled" + if gotBody != want { + t.Errorf("body = %q, want %q", gotBody, want) + } + if gotMD5 == "" { + t.Error("Content-MD5 was not sent") + } +} + +// S3 has no call that returns a bucket to never-versioned, so the constant that +// means it must not be accepted here. +func TestSetBucketVersioningRejectsAnImpossibleState(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an invalid status must be refused before any request") + w.WriteHeader(http.StatusOK) + }) + + if err := c.SetBucketVersioning(context.Background(), "b", VersioningUnversioned); err == nil { + t.Fatal("Unversioned was accepted as a settable status") + } +} + +// A versioned listing pages on a key marker plus a version-id marker, and its +// and elements have to be presented together per key, +// newest first, rather than as two separate runs. +func TestListObjectVersionsPagesAndOrders(t *testing.T) { + var queries []string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + queries = append(queries, q.Get("key-marker")+"/"+q.Get("version-id-marker")) + if !q.Has("versions") { + t.Error("the request did not carry ?versions") + } + if q.Get("key-marker") == "" { + _, _ = fmt.Fprint(w, `true + dumpv1 + dumpv2true + 202026-09-14T10:00:00.000Z + dumpv3 + 2026-09-14T11:00:00.000Z + `) + return + } + _, _ = fmt.Fprint(w, `false + dumpv110 + 2026-09-14T09:00:00.000Z + `) + }) + + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Versions: true}) + if err != nil { + t.Fatal(err) + } + if len(queries) != 2 || queries[1] != "dump/v1" { + t.Errorf("requests = %v, want the second to carry both markers", queries) + } + if len(objs) != 3 { + t.Fatalf("entries = %d, want 3", len(objs)) + } + // Within a page: same key, newest first — the delete marker (11:00) before + // the version it hides (10:00). + if !objs[0].DeleteMarker || objs[0].VersionID != "v3" { + t.Errorf("first entry = %+v, want the newest delete marker", objs[0]) + } + if objs[1].VersionID != "v2" || !objs[1].IsLatest { + t.Errorf("second entry = %+v, want version v2", objs[1]) + } + if objs[2].VersionID != "v1" || objs[2].Size != 10 { + t.Errorf("third entry = %+v, want version v1 from the second page", objs[2]) + } +} + +// A delimited listing reports subtrees as CommonPrefixes, which is what turns a +// flat keyspace into one directory level. +func TestListObjectsWithADelimiter(t *testing.T) { + var gotDelimiter string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotDelimiter = r.URL.Query().Get("delimiter") + _, _ = fmt.Fprint(w, `false + 2026/ + 2025/ + latest.sql.gz7 + `) + }) + + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Delimiter: "/"}) + if err != nil { + t.Fatal(err) + } + if gotDelimiter != "/" { + t.Errorf("delimiter = %q, want /", gotDelimiter) + } + if len(objs) != 3 { + t.Fatalf("entries = %d, want 2 prefixes and 1 key", len(objs)) + } + // Subtrees first, so a delimited listing reads like a directory. + if !objs[0].IsPrefix || objs[0].Key != "2026/" { + t.Errorf("first entry = %+v, want the 2026/ prefix", objs[0]) + } + if !objs[1].IsPrefix { + t.Errorf("second entry = %+v, want a prefix", objs[1]) + } + if objs[2].IsPrefix || objs[2].Key != "latest.sql.gz" { + t.Errorf("third entry = %+v, want the key at this level", objs[2]) + } +} + +// --limit counts prefixes as well as keys: they are both entries the caller +// asked to see at most N of. +func TestListObjectsLimitCountsPrefixes(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + a/ + b/ + c + `) + }) + + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Delimiter: "/", Limit: 2}) + if err != nil { + t.Fatal(err) + } + if len(objs) != 2 { + t.Errorf("entries = %d, want the limit honoured", len(objs)) + } +} + +func TestHeadBucket(t *testing.T) { + var gotMethod, gotPath string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusOK) + }) + + if err := c.HeadBucket(context.Background(), "db-backups"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodHead || gotPath != "/db-backups" { + t.Errorf("request = %s %s, want HEAD /db-backups", gotMethod, gotPath) + } +} + +// A HEAD reply has no body, so a missing bucket is a bare 404 and IsNotFound +// has to recognise it from the status alone. +func TestHeadBucketNotFound(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + err := c.HeadBucket(context.Background(), "absent") + if err == nil || !IsNotFound(err) { + t.Errorf("err = %v, want a recognised not-found", err) + } +} + +// A version ID must reach the request as a query parameter on the object's URL. +func TestVersionedObjectAddressing(t *testing.T) { + var queries []string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.Query().Get("versionId")) + w.Header().Set("x-amz-version-id", "v9") + w.WriteHeader(http.StatusOK) + }) + + info, err := c.HeadObject(context.Background(), "b", "k", "v9") + if err != nil { + t.Fatal(err) + } + if info.VersionID != "v9" { + t.Errorf("VersionID = %q, want v9 from the response header", info.VersionID) + } + if err := c.DeleteObject(context.Background(), "b", "k", "v9"); err != nil { + t.Fatal(err) + } + if _, err := c.GetObject(context.Background(), "b", "k", "v9", io.Discard); err != nil { + t.Fatal(err) + } + for i, got := range queries { + if got != "v9" { + t.Errorf("request %d carried versionId=%q, want v9", i, got) + } + } + if len(queries) != 3 { + t.Errorf("requests = %d, want head/delete/get", len(queries)) + } +} + +// The current object must not carry a versionId parameter at all: some stores +// reject an empty one. +func TestUnversionedObjectAddressingSendsNoParameter(t *testing.T) { + var raw string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + raw = r.URL.RawQuery + w.WriteHeader(http.StatusOK) + }) + + if _, err := c.HeadObject(context.Background(), "b", "k", ""); err != nil { + t.Fatal(err) + } + if strings.Contains(raw, "versionId") { + t.Errorf("query = %q, want no versionId", raw) + } +} From fc73c5c5a61401363a741267accc249af67d4ff3 Mon Sep 17 00:00:00 2001 From: Fedor Tarasenko Date: Mon, 14 Sep 2026 12:20:43 +0000 Subject: [PATCH 3/3] feat(s3): add sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last s5cmd capability koc did not have. It needed no new dependency and no new import: sync is a comparison layer over the verbs the previous commit landed, so it cost +40,960 bytes of binary (+0.245%) and the module graph is still 9. koc s3 sync ./restore s3://db-backups/2026/ # local -> S3 koc s3 sync s3://db-backups/2026/ ./restore # S3 -> local koc s3 sync s3://db-backups/ s3://archive/ # S3 -> S3, server-side The decisions, which are the actual content here: - The s3:// prefix is mandatory on this command alone. Every other verb guesses nothing because it has one subject; sync has two, and mistaking which side is local is how a --delete removes the wrong one. A bare path is always local. - An object moves when it is absent at the destination, when the sizes differ, or when the source is newer; --size-only drops the timestamp test, which is the right rule for content that never changes in place. - The S3-side timestamp is Last-Modified. S3 keeps no record of the source file's mtime, and — decisively — a listing would not return one if it did, so using x-amz-meta-mtime would cost a HEAD per object and defeat the point. Last-Modified is correct in the steady state: an upload lands after the file was written, a download after the object was, so neither direction re-transfers what it just moved. The one seam is a round trip, where a restored tree carries the restore time and syncing it back re-uploads once before settling. Documented, with --size-only as the way out. Timestamps are compared truncated to the second, because S3 reports Last-Modified at that resolution and a nanosecond local mtime would otherwise make every file look newer than its own copy. - --delete is refused when the source turned out to be empty unless --force is also given. A mistyped source that lists nothing would erase the destination, and that is not recoverable by re-running the command. Entries --include/--exclude leave out are outside the sync and are never deleted. Local deletion removes files, not the directories behind them. - The destination is the side held in memory, not the source: a lookup per source entry decides each transfer, and --delete then wants exactly the entries nothing looked up. The two sides cannot be streamed against each other because S3 lists keys lexicographically while a directory walk descends depth-first, so "a.txt" and "a/b" come out in opposite orders. - A local side must be a directory. A single file as the source would otherwise walk to the relative path "." and upload itself under that key. Reuses downloadToFile (atomic, removes a partial file), PutObjectStream (multipart), CopyObject, the batched DeleteObjects, the worker pool and the glob filter. Uploads run one part at a time inside the pool so --concurrency is not squared into that many part buffers. A download destination goes through the same traversal check as the recursive one, since an object key is server-supplied data. Exercised end to end against a mock endpoint that verifies SigV4 itself and records real per-object Last-Modified: a tree synced up, then reported already in sync; one edited and one added file moving alone; all three directions; --delete in both directions and its empty-source refusal; --dry-run writing nothing; --include scoping both sides; and the documented round-trip seam re-uploading once and then settling. Every request in that run verified. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TQ2Xt7Epc3n8XAXiAgPXiC --- AGENTS.md | 2 +- README.md | 15 + docs/coverage.md | 16 +- internal/cli/s3/s3.go | 1 + internal/cli/s3/sync.go | 547 +++++++++++++++++++++++++++++++++++ internal/cli/s3/sync_test.go | 482 ++++++++++++++++++++++++++++++ 6 files changed, 1054 insertions(+), 9 deletions(-) create mode 100644 internal/cli/s3/sync.go create mode 100644 internal/cli/s3/sync_test.go diff --git a/AGENTS.md b/AGENTS.md index 4e51ebd..7f49471 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -204,7 +204,7 @@ internal/output/ -f/--format {table,json,yaml,value,csv} and -c/--colu internal/cli/keyvrm/ KeyVRM (in-house catalog service); typed request layer (types.go/requests.go) + cobra verbs internal/cli/vault/ "koc vault kv" list/get/copy/export/decrypt (package vaultcli); Vault creds only internal/cli/s3/ "koc s3" bucket/object lifecycle, du, download/upload (multipart, stdin, - recursive), server-side copy/move, presign (package s3cli); S3 creds only + recursive), server-side copy/move, presign, sync (package s3cli); S3 creds only internal/cli/quota/ "koc quota show|set" — the one cross-service noun (nova+cinder+neutron) internal/cli/ root.go wires every service's command group onto the root internal/cli/resolve/ cross-service name→ID (image→glance, network→neutron, project→keystone) diff --git a/README.md b/README.md index e8e0405..ed7b614 100644 --- a/README.md +++ b/README.md @@ -538,6 +538,7 @@ koc s3 upload ./restore db-backups/2026/ -r koc s3 copy db-backups/ db-backups/latest.mbs.gz.enc koc s3 move db-backups/ archive/ koc s3 presign db-backups/ --expire 1h +koc s3 sync ./restore s3://db-backups/2026/ --delete ``` Every ref also accepts the `s3:///` spelling, so a path copied from @@ -569,6 +570,20 @@ bytes never travel through koc and a 100 GiB object is one small request. A move is the copy and then the delete, in that order — a failure leaves the source intact rather than losing the object. +**`sync`** makes a destination match a source, transferring only what differs — +local→S3, S3→local, or S3→S3 (server-side). It is the one command in the group +that *requires* the `s3://` prefix to mark a remote side: it has two sides, and +mistaking which is local is how a `--delete` removes the wrong one. An object +moves when it is absent, when the sizes differ, or when the source is newer; +`--size-only` drops the timestamp test, which is the right rule for content that +never changes in place (a dated backup). The S3-side timestamp is the object's +Last-Modified, because S3 keeps no record of the source file's mtime and a +listing would not return one if it did — correct in the steady state, with one +seam: a tree restored by `download` carries the restore time, so syncing it +*back* re-uploads once and then settles. `--delete` turns the sync into a mirror +and is refused when the source turned out to be empty unless `--force` is also +given, so a mistyped source cannot erase the destination. + `object delete --recursive` is the one destructive shape in the group, so it is never inferred from a trailing slash — and `--dry-run` prints exactly the keys it would remove without touching any of them. It is also how a bucket is emptied diff --git a/docs/coverage.md b/docs/coverage.md index fa78fc1..841299c 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -3,7 +3,7 @@ How much of the upstream OpenStack CLI surface `koc` implements, measured against primary sources rather than documentation. -**Snapshot:** 2026-09-14 · `koc` @ this commit (base `c532e4c`) · 564 leaf +**Snapshot:** 2026-09-14 · `koc` @ this commit (base `82afd0a`) · 565 leaf commands (visible tree; 2 more are hidden duplicates). **Keep this file current** — see "Updating this document" below. Any commit that @@ -28,8 +28,8 @@ PyPI is the source of record. ## Headline -**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 564 leaf commands, -515 are upstream-equivalent and 49 are koc-native. +**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 565 leaf commands, +515 are upstream-equivalent and 50 are koc-native. The denominator grew by 13 against the 2026-08-07 snapshot without a single command changing: `python-ironic-inspector-client` is now a **baseline** rather @@ -433,14 +433,14 @@ limitations"; the fix is to make the other resolvers match `server`'s behaviour. ## koc-native commands -No upstream equivalent, by design — **49 leaves**, itemised so the total -reconciles with the headline (564 = 515 + 49): +No upstream equivalent, by design — **50 leaves**, itemised so the total +reconciles with the headline (565 = 515 + 50): | Count | Commands | Why it has no upstream equivalent | | --- | --- | --- | | 18 | `koc keyvrm …` — `app-config` ×2, `availability-zone` ×2, `event` ×4, `host-aggregate-config` ×5, `recommendation` ×5 | in-house KeyVRM catalog service; no gophercloud package and no OSC plugin | | 5 | `koc vault kv list/get/copy/export/decrypt` | Vault is not an OpenStack service; `copy` fills a gap in the Vault CLI itself | -| 14 | `koc s3 bucket list/create/delete/show/set`, `koc s3 object list/show/delete`, `koc s3 du`, `koc s3 download/upload`, `koc s3 copy/move`, `koc s3 presign` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | +| 15 | `koc s3 bucket list/create/delete/show/set`, `koc s3 object list/show/delete`, `koc s3 du`, `koc s3 download/upload`, `koc s3 copy/move`, `koc s3 presign`, `koc s3 sync` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | | 2 | `koc dns pool list/show` | designate's API and its Python SDK both expose `/v2/pools`, but `python-designateclient` registers no `openstack` command for it. Reads only — pool *writes* are a `designate-manage`/config operation on the servers | | 2 | `koc server add/remove server-group` | KeyStack dynamic server groups | | 2 | `koc network trunk subport add`/`remove` | upstream folds these into `network trunk set`/`unset --subport` flags rather than giving them verbs (`network subport list` does exist and is counted — see "Naming deviations") | @@ -481,7 +481,7 @@ The tables are derived, not hand-maintained. To re-derive after a version bump or a batch of new commands: ```sh -# 1. koc's own command tree (564 leaf commands at the snapshot above) +# 1. koc's own command tree (565 leaf commands at the snapshot above) make build # Walk `--help` recursively. Count a command when it is *runnable*, not merely when # it is childless: `koc image import ` is a verb that also parents `koc image @@ -511,7 +511,7 @@ Then **check the arithmetic**, because that is the only thing that makes these tables worth reading. Three identities must hold at every snapshot: 1. every raw row numerator summed = the headline numerator (515); -2. leaf commands = headline numerator + koc-native (564 = 515 + 49); +2. leaf commands = headline numerator + koc-native (565 = 515 + 50); 3. every raw row denominator summed = 901, and minus the two not-targeted rows (swift 17 + manila 40) = the in-scope denominator (844). diff --git a/internal/cli/s3/s3.go b/internal/cli/s3/s3.go index 6f7e418..2e313ae 100644 --- a/internal/cli/s3/s3.go +++ b/internal/cli/s3/s3.go @@ -109,5 +109,6 @@ func NewCommand(a *auth.Options, o *output.Options) *cobra.Command { cmd.AddCommand(newCopyCommand(a, o, f)) cmd.AddCommand(newMoveCommand(a, o, f)) cmd.AddCommand(newPresignCommand(a, o, f)) + cmd.AddCommand(newSyncCommand(a, o, f)) return cmd } diff --git a/internal/cli/s3/sync.go b/internal/cli/s3/sync.go new file mode 100644 index 0000000..3feebd1 --- /dev/null +++ b/internal/cli/s3/sync.go @@ -0,0 +1,547 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// syncFlags holds the options accepted by "sync". +type syncFlags struct { + del bool + sizeOnly bool + dryRun bool + force bool + concurrency int + partSizeMiB int + filter keyFilter +} + +const syncLong = `Make a destination match a source, transferring only what differs. + +Both sides are named with the s3:// prefix for an S3 location and a plain path +for a local one: + + koc s3 sync ./restore s3://db-backups/2026/ # local -> S3 + koc s3 sync s3://db-backups/2026/ ./restore # S3 -> local + koc s3 sync s3://db-backups/ s3://archive/ # S3 -> S3 (server-side) + +sync is the one command in this group that *requires* the s3:// prefix. Every +other verb accepts a bare "/" because it has a single subject; sync +has two, and mistaking which one is local is how a --delete removes the wrong +side. A bare path is always local here. + +An S3 side is a prefix, treated as a directory: a trailing "/" is added if it is +missing, and each object's key below it is what the two sides are matched on. A +local side is a directory tree; symlinks and devices are skipped rather than +followed. + +An object is transferred when it is absent at the destination, when the sizes +differ, or when the source is newer. --size-only drops the timestamp test, which +is what you want for content that never changes in place (a dated backup) and +the cheapest correct rule there is. + +The timestamp on the S3 side is the object's Last-Modified, i.e. when it was +uploaded — S3 keeps no record of the source file's own mtime, and a listing +would not return one if it did. That is correct in the steady state, because an +upload always lands after the file was written and a download always lands after +the object was: neither direction re-transfers what it just moved. The one seam +is a round trip — files restored with "download" carry the restore time, so +syncing that tree *back* re-uploads it once, after which it settles. Pass +--size-only to avoid even that. + +--delete removes destination entries the source does not have, which is what +makes this a mirror rather than an overlay. It is refused when the source turned +out to be empty unless --force is also given: a mistyped source that lists +nothing would otherwise erase the destination. Entries excluded by +--include/--exclude are outside the sync and are never deleted. Local deletion +removes files, not the directories left behind. + +--dry-run prints exactly the transfers and deletions it would perform.` + +const syncExample = ` # Mirror a restore tree up, removing what is no longer local + koc s3 sync ./restore s3://db-backups/2026/ --delete + + # Pull a month down, eight objects at a time + koc s3 sync s3://db-backups/2026/08/ ./restore --concurrency 8 + + # Dated dumps never change in place, so size is the whole test + koc s3 sync s3://db-backups/ s3://archive/ --size-only + + # See what would move, before it moves + koc s3 sync ./restore s3://db-backups/2026/ --delete --dry-run + + # Only the compressed dumps + koc s3 sync s3://db-backups/ ./restore --include "*.sql.gz"` + +func newSyncCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + sf := &syncFlags{} + cmd := &cobra.Command{ + Use: "sync ", + Short: "Make a destination match a source, transferring only what differs", + Long: syncLong, + Example: syncExample, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + src, dst, err := parseSyncEndpoints(args[0], args[1]) + if err != nil { + return err + } + if err := sf.filter.compile(); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runSync(ctx, client, src, dst, sf, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVar(&sf.del, "delete", false, + "remove destination entries the source does not have") + fl.BoolVar(&sf.sizeOnly, "size-only", false, + "compare only sizes, ignoring timestamps") + fl.BoolVar(&sf.dryRun, "dry-run", false, + "print the transfers and deletions that would happen, and perform none") + fl.BoolVar(&sf.force, "force", false, + "allow --delete even when the source turned out to be empty") + fl.IntVar(&sf.concurrency, "concurrency", s3.DefaultConcurrency, + "objects to transfer at once") + fl.IntVar(&sf.partSizeMiB, "part-size", s3.DefaultPartSize>>20, + "multipart part size in MiB for uploads (minimum 5)") + sf.filter.addTo(fl) + return cmd +} + +// syncEndpoint is one end of a sync: a local directory, or a prefix in a +// bucket. Exactly one of dir / bucket is set. +type syncEndpoint struct { + dir string + + bucket string + // prefix always ends in "/" when non-empty, so a key below it trims + // cleanly to the relative path the two sides are matched on. + prefix string +} + +func (e syncEndpoint) isLocal() bool { return e.bucket == "" } + +func (e syncEndpoint) String() string { + if e.isLocal() { + return e.dir + } + return "s3://" + e.bucket + "/" + e.prefix +} + +// join renders the full address of one relative key on this side. +func (e syncEndpoint) join(key string) string { + if e.isLocal() { + return filepath.Join(e.dir, filepath.FromSlash(key)) + } + return e.prefix + key +} + +// parseSyncEndpoints resolves both arguments and rejects the pairs that are not +// a sync. +func parseSyncEndpoints(srcRef, dstRef string) (src, dst syncEndpoint, err error) { + if src, err = parseSyncEndpoint(srcRef); err != nil { + return src, dst, err + } + if dst, err = parseSyncEndpoint(dstRef); err != nil { + return src, dst, err + } + if src.isLocal() && dst.isLocal() { + return src, dst, errors.New("at least one side must be an s3:// location; " + + "two local paths are a job for cp or rsync") + } + if src == dst { + return src, dst, fmt.Errorf("source and destination are the same location (%s)", src) + } + return src, dst, nil +} + +// parseSyncEndpoint reads one side. The s3:// prefix is mandatory for an S3 +// location here — see the command's Long for why sync alone insists on it. +func parseSyncEndpoint(ref string) (syncEndpoint, error) { + rest, ok := strings.CutPrefix(ref, "s3://") + if !ok { + if ref == "" { + return syncEndpoint{}, errors.New("a sync side cannot be empty") + } + return syncEndpoint{dir: ref}, nil + } + + bucket, prefix, _ := strings.Cut(strings.TrimPrefix(rest, "/"), "/") + if bucket == "" { + return syncEndpoint{}, fmt.Errorf("%q names no bucket: expected s3://[/]", ref) + } + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + return syncEndpoint{bucket: bucket, prefix: prefix}, nil +} + +// checkLocalSides rejects a local side that is not a directory tree. A source +// that is a single file would otherwise walk to the relative path "." and +// upload itself under that key; a destination that is a file would fail later, +// halfway through the transfers. +func checkLocalSides(src, dst syncEndpoint) error { + if src.isLocal() { + info, err := os.Stat(src.dir) + if err != nil { + return fmt.Errorf("reading source %q: %w", src.dir, err) + } + if !info.IsDir() { + return fmt.Errorf("source %q is not a directory; sync mirrors trees, "+ + "use \"koc s3 upload\" for one file", src.dir) + } + } + // A destination that does not exist yet is about to be created, so only an + // existing non-directory is a problem. + if dst.isLocal() { + if info, err := os.Stat(dst.dir); err == nil && !info.IsDir() { + return fmt.Errorf("destination %q is not a directory", dst.dir) + } + } + return nil +} + +// syncEntry is one item on either side, identified by its path relative to that +// side's root — which is what makes a local file and an object comparable. +type syncEntry struct { + key string + size int64 + mtime time.Time + // path is the local file's path; empty for an object. + path string +} + +// runSync is the test seam for "sync". +func runSync(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + f *syncFlags, w io.Writer) error { + if err := checkLocalSides(src, dst); err != nil { + return err + } + + // The destination is indexed rather than the source: a lookup per source + // entry is what decides a transfer, and --delete then needs exactly the + // entries nothing looked up. Memory is therefore the destination's size — + // the two sides cannot be streamed against each other because S3 lists + // keys lexicographically while a directory walk descends depth-first, so + // "a.txt" and "a/b" come out in opposite orders. + index, err := indexEndpoint(ctx, client, dst, f) + if err != nil { + return err + } + + seen := make(map[string]bool, len(index)) + out := &syncWriter{w: w} + p, workCtx := newPool(ctx, f.concurrency) + source, transferred := 0, 0 + + err = walkEndpoint(ctx, client, src, f, func(e syncEntry) error { + source++ + seen[e.key] = true + if at, ok := index[e.key]; ok && !f.needsTransfer(e, at) { + return nil + } + transferred++ + if !p.run(workCtx, func() error { return syncOne(workCtx, client, src, dst, e, f, out) }) { + return errStopListing + } + return nil + }) + if waitErr := p.wait(); err == nil || isStopListing(err) { + err = waitErr + } + if err != nil && !isStopListing(err) { + return fmt.Errorf("syncing %s to %s: %w", src, dst, err) + } + + removed, err := syncDelete(ctx, client, dst, index, seen, source, f, out) + if err != nil { + return err + } + if transferred == 0 && removed == 0 { + _, err = fmt.Fprintf(w, "Already in sync: %s and %s (%d objects)\n", src, dst, source) + return err + } + return nil +} + +// needsTransfer is the comparison rule. Timestamps are truncated to the second +// because S3 reports Last-Modified at that resolution while a local mtime +// carries nanoseconds, and the difference alone would make every file look +// newer than its own copy. +func (f *syncFlags) needsTransfer(src, dst syncEntry) bool { + if src.size != dst.size { + return true + } + if f.sizeOnly { + return false + } + return src.mtime.Truncate(time.Second).After(dst.mtime.Truncate(time.Second)) +} + +// indexEndpoint reads one side into a map keyed by relative path. A local +// destination that does not exist yet is an empty index, not an error — it is +// about to be created. +func indexEndpoint(ctx context.Context, client *s3.Client, e syncEndpoint, + f *syncFlags) (map[string]syncEntry, error) { + index := map[string]syncEntry{} + err := walkEndpoint(ctx, client, e, f, func(entry syncEntry) error { + index[entry.key] = entry + return nil + }) + if err != nil && e.isLocal() && errors.Is(err, os.ErrNotExist) { + return index, nil + } + if err != nil { + return nil, fmt.Errorf("reading %s: %w", e, err) + } + return index, nil +} + +// walkEndpoint enumerates one side, applying --include/--exclude so that a +// filtered-out entry is invisible to both the comparison and --delete. +func walkEndpoint(ctx context.Context, client *s3.Client, e syncEndpoint, + f *syncFlags, fn func(syncEntry) error) error { + keep := func(entry syncEntry) error { + if !f.filter.match(entry.key) { + return nil + } + return fn(entry) + } + if e.isLocal() { + return walkLocalTree(e.dir, keep) + } + return walkRemotePrefix(ctx, client, e, keep) +} + +// walkLocalTree reports every regular file under root. +func walkLocalTree(root string, fn func(syncEntry) error) error { + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + // Symlinks and devices are skipped rather than followed: a tree walked + // from a backup directory should not chase a link out of it. + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return fmt.Errorf("resolving %q under %q: %w", path, root, err) + } + info, err := d.Info() + if err != nil { + return fmt.Errorf("stat %q: %w", path, err) + } + return fn(syncEntry{ + key: filepath.ToSlash(rel), + size: info.Size(), + mtime: info.ModTime(), + path: path, + }) + }) +} + +// walkRemotePrefix reports every object below the endpoint's prefix. +func walkRemotePrefix(ctx context.Context, client *s3.Client, e syncEndpoint, + fn func(syncEntry) error) error { + return client.ListObjectsFunc(ctx, e.bucket, s3.ListOptions{Prefix: e.prefix}, + func(obj s3.Object) error { + // A "directory marker" — a zero-length key ending in "/" that some + // tools create — has no counterpart on a filesystem. + if strings.HasSuffix(obj.Key, "/") { + return nil + } + return fn(syncEntry{ + key: strings.TrimPrefix(obj.Key, e.prefix), + size: obj.Size, + mtime: obj.LastModified, + }) + }) +} + +// syncOne transfers one entry from src to dst, by whichever of the three routes +// the pair of endpoints implies. A local-to-local pair was rejected at parse. +func syncOne(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + switch { + case src.isLocal(): + return syncUpload(ctx, client, dst, e, f, w) + case dst.isLocal(): + return syncDownload(ctx, client, src, dst, e, f, w) + default: + return syncCopy(ctx, client, src, dst, e, f, w) + } +} + +// syncUpload sends one local file to the destination prefix. +func syncUpload(ctx context.Context, client *s3.Client, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + key := dst.join(e.key) + if f.dryRun { + _, err := fmt.Fprintf(w, "Would upload %s to %s/%s\n", e.path, dst.bucket, key) + return err + } + + src, err := os.Open(e.path) + if err != nil { + return fmt.Errorf("opening %q: %w", e.path, err) + } + defer func() { _ = src.Close() }() + + // One part at a time: the pool's workers are already objects, and the + // product of the two would put concurrency² part buffers in memory. + opts := s3.UploadOptions{ + ContentType: contentTypeFor(e.path), + PartSize: int64(f.partSizeMiB) << 20, + Concurrency: 1, + Size: e.size, + } + obj, err := client.PutObjectStream(ctx, dst.bucket, key, src, opts) + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", e.path, dst.bucket, key, err) + } + _, err = fmt.Fprintf(w, "Uploaded: %s -> %s/%s (%d bytes)\n", e.path, dst.bucket, key, obj.Size) + return err +} + +// syncDownload fetches one object into the destination tree. +func syncDownload(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + key := src.join(e.key) + // A key is server-supplied data, and "../../etc/cron.d/x" is a legal one, + // so the destination is checked to be inside the tree before anything is + // created. + path, err := destForKey(dst.dir, "", e.key, false) + if err != nil { + return err + } + if f.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to %s\n", src.bucket, key, path) + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + + // force: replacing a stale copy is the whole point of a sync, unlike the + // single-object "download" where an existing file means a mistyped key. + n, err := downloadToFile(ctx, client, src.bucket, key, "", path, true) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "Downloaded: %s/%s -> %s (%d bytes)\n", src.bucket, key, path, n) + return err +} + +// syncCopy moves one object between two S3 locations, server-side. +func syncCopy(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + from := s3.ObjectRef{Bucket: src.bucket, Key: src.join(e.key)} + to := s3.ObjectRef{Bucket: dst.bucket, Key: dst.join(e.key)} + if f.dryRun { + return sayCopy(w, from, to, false, true) + } + if _, err := client.CopyObject(ctx, from, to, "", ""); err != nil { + return err + } + return sayCopy(w, from, to, false, false) +} + +// syncDelete removes the destination entries the source did not have, and +// reports how many it took. +func syncDelete(ctx context.Context, client *s3.Client, dst syncEndpoint, + index map[string]syncEntry, seen map[string]bool, source int, + f *syncFlags, w io.Writer) (int, error) { + if !f.del { + return 0, nil + } + // A mistyped source that listed nothing would otherwise erase the + // destination, and that is not a mistake anyone recovers from by re-running + // the command. + if source == 0 && !f.force { + return 0, fmt.Errorf("the source is empty, so --delete would remove all %d "+ + "entries under %s; pass --force if that is what you meant", len(index), dst) + } + + extra := make([]string, 0, len(index)) + for key := range index { + if !seen[key] { + extra = append(extra, key) + } + } + // Sorted so a run is reproducible and its output diffs against the next. + sort.Strings(extra) + if len(extra) == 0 { + return 0, nil + } + + if dst.isLocal() { + return len(extra), deleteLocalExtras(index, extra, f, w) + } + return len(extra), deleteRemoteExtras(ctx, client, dst, extra, f, w) +} + +// deleteLocalExtras removes local files the source no longer has. The +// directories they leave behind are not removed — an empty directory is not +// something the source can be said to have deleted. +func deleteLocalExtras(index map[string]syncEntry, extra []string, f *syncFlags, w io.Writer) error { + for _, key := range extra { + path := index[key].path + if f.dryRun { + if _, err := fmt.Fprintf(w, "Would delete file: %s\n", path); err != nil { + return err + } + continue + } + if err := os.Remove(path); err != nil { + return fmt.Errorf("deleting %q: %w", path, err) + } + if _, err := fmt.Fprintf(w, "Deleted file: %s\n", path); err != nil { + return err + } + } + return nil +} + +// deleteRemoteExtras removes objects the source no longer has, batched so a +// large mirror does not cost one request per key. +func deleteRemoteExtras(ctx context.Context, client *s3.Client, dst syncEndpoint, + extra []string, f *syncFlags, w io.Writer) error { + targets := make([]s3.DeleteTarget, 0, len(extra)) + for _, key := range extra { + targets = append(targets, s3.DeleteTarget{Key: dst.join(key)}) + } + for len(targets) > 0 { + batch := targets + if len(batch) > s3.MaxDeleteBatch { + batch = batch[:s3.MaxDeleteBatch] + } + if err := deleteBatch(ctx, client, dst.bucket, batch, f.dryRun, w); err != nil { + return err + } + targets = targets[len(batch):] + } + return nil +} diff --git a/internal/cli/s3/sync_test.go b/internal/cli/s3/sync_test.go new file mode 100644 index 0000000..0270988 --- /dev/null +++ b/internal/cli/s3/sync_test.go @@ -0,0 +1,482 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +func testSyncFlags() *syncFlags { + return &syncFlags{concurrency: 1, partSizeMiB: s3.DefaultPartSize >> 20} +} + +// syncMock serves a listing whose objects carry a size and a Last-Modified, and +// records every write it is sent. +type syncMock struct { + objects map[string]syncObject + puts []string + deletes []string + copies []string + gets []string +} + +type syncObject struct { + size int64 + mtime string +} + +func (m *syncMock) handler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case q.Has("list-type"): + prefix := q.Get("prefix") + var rows string + for key, o := range m.objects { + if !strings.HasPrefix(key, prefix) { + continue + } + rows += fmt.Sprintf(`%s%d`+ + `%s"e"`, key, o.size, o.mtime) + } + _, _ = fmt.Fprintf(w, `false%s`, rows) + case r.Method == http.MethodPost && q.Has("delete"): + body := readAll(r) + m.deletes = append(m.deletes, keysInBatch(body)...) + _, _ = fmt.Fprint(w, ``) + case r.Method == http.MethodPut && r.Header.Get("x-amz-copy-source") != "": + m.copies = append(m.copies, r.Header.Get("x-amz-copy-source")+" -> "+r.URL.Path) + _, _ = fmt.Fprint(w, `"e"`) + case r.Method == http.MethodPut: + m.puts = append(m.puts, r.URL.Path) + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + default: + m.gets = append(m.gets, r.URL.Path) + _, _ = w.Write([]byte("data")) + } + } +} + +func readAll(r *http.Request) string { + var buf bytes.Buffer + _, _ = buf.ReadFrom(r.Body) + return buf.String() +} + +func localSide(dir string) syncEndpoint { return syncEndpoint{dir: dir} } +func s3Side(b, p string) syncEndpoint { return syncEndpoint{bucket: b, prefix: p} } + +// The s3:// prefix is mandatory on this command alone, because mistaking which +// side is local is how a --delete removes the wrong one. +func TestParseSyncEndpoints(t *testing.T) { + src, dst, err := parseSyncEndpoints("./restore", "s3://db-backups/2026") + if err != nil { + t.Fatal(err) + } + if !src.isLocal() || src.dir != "./restore" { + t.Errorf("source = %+v, want the local path", src) + } + // A prefix is a directory, so the trailing slash is supplied. + if dst.bucket != "db-backups" || dst.prefix != "2026/" { + t.Errorf("destination = %+v, want prefix 2026/", dst) + } + // A bare bucket has an empty prefix rather than "/". + remote, err := parseSyncEndpoint("s3://db-backups") + if err != nil { + t.Fatal(err) + } + if remote.prefix != "" { + t.Errorf("prefix = %q, want empty", remote.prefix) + } +} + +func TestParseSyncEndpointsRejectsBadPairs(t *testing.T) { + for _, tc := range []struct{ name, src, dst, want string }{ + {"two local paths", "./a", "./b", "at least one side"}, + {"same location", "s3://b/p", "s3://b/p/", "the same location"}, + {"no bucket", "s3://", "./a", "names no bucket"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, err := parseSyncEndpoints(tc.src, tc.dst); err == nil || + !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// The comparison rule: absent, a different size, or a newer source. +func TestNeedsTransfer(t *testing.T) { + old := time.Date(2026, 9, 1, 10, 0, 0, 0, time.UTC) + recent := old.Add(time.Hour) + + for _, tc := range []struct { + name string + src, dst syncEntry + sizeOnly bool + want bool + }{ + {"same size and time", syncEntry{size: 10, mtime: old}, syncEntry{size: 10, mtime: old}, false, false}, + {"different size", syncEntry{size: 11, mtime: old}, syncEntry{size: 10, mtime: old}, false, true}, + {"source newer", syncEntry{size: 10, mtime: recent}, syncEntry{size: 10, mtime: old}, false, true}, + {"destination newer", syncEntry{size: 10, mtime: old}, syncEntry{size: 10, mtime: recent}, false, false}, + {"--size-only ignores a newer source", syncEntry{size: 10, mtime: recent}, syncEntry{size: 10, mtime: old}, true, false}, + {"--size-only still sees a size change", syncEntry{size: 11, mtime: old}, syncEntry{size: 10, mtime: old}, true, true}, + // S3 reports Last-Modified to the second while a local mtime carries + // nanoseconds; without truncation every file looks newer than its copy. + {"sub-second difference is not newer", + syncEntry{size: 10, mtime: old.Add(400 * time.Millisecond)}, syncEntry{size: 10, mtime: old}, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := &syncFlags{sizeOnly: tc.sizeOnly} + if got := f.needsTransfer(tc.src, tc.dst); got != tc.want { + t.Errorf("needsTransfer = %v, want %v", got, tc.want) + } + }) + } +} + +// Only what differs moves: a matching object is left alone, so a re-run of a +// finished sync costs one listing per side and no transfers. +func TestRunSyncLocalToS3TransfersOnlyDifferences(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "same.txt"), "1234") + mustWrite(t, filepath.Join(dir, "changed.txt"), "1234567") + mustWrite(t, filepath.Join(dir, "new.txt"), "12") + // Both existing objects are newer than the files, so only the size + // difference should move. + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + m := &syncMock{objects: map[string]syncObject{ + "2026/same.txt": {size: 4, mtime: future}, + "2026/changed.txt": {size: 99, mtime: future}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(dir), s3Side("db-backups", "2026/"), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + want := "/db-backups/2026/changed.txt,/db-backups/2026/new.txt" + if got := strings.Join(sortedStrings(m.puts), ","); got != want { + t.Errorf("uploaded = %q, want %q", got, want) + } +} + +// A finished sync says so, rather than printing nothing — which would be +// indistinguishable from "the source matched nothing". +func TestRunSyncAlreadyInSync(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "a.txt"), "1234") + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + m := &syncMock{objects: map[string]syncObject{"a.txt": {size: 4, mtime: future}}} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(dir), s3Side("db-backups", ""), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + if len(m.puts) != 0 { + t.Errorf("uploaded %v, want nothing", m.puts) + } + if !strings.Contains(buf.String(), "Already in sync") { + t.Errorf("output = %q, want it to say the sides match", buf.String()) + } +} + +// S3 -> local: the tree is created and each key lands at its path below the +// prefix. +func TestRunSyncS3ToLocal(t *testing.T) { + dir := filepath.Join(t.TempDir(), "restore") + m := &syncMock{objects: map[string]syncObject{ + "2026/08/a.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + "2026/09/b.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + "2025/old.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, s3Side("db-backups", "2026/"), localSide(dir), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + for _, rel := range []string{"08/a.sql.gz", "09/b.sql.gz"} { + if _, err := os.Stat(filepath.Join(dir, rel)); err != nil { + t.Errorf("%s was not written: %v", rel, err) + } + } + // The prefix is the scope: 2025/ is outside it. + if _, err := os.Stat(filepath.Join(dir, "..", "old.sql.gz")); err == nil { + t.Error("an object outside the prefix was transferred") + } +} + +// A stale local copy is replaced, unlike the single-object "download" where an +// existing file means a mistyped key. +func TestRunSyncS3ToLocalReplacesStaleFiles(t *testing.T) { + dir := t.TempDir() + stale := filepath.Join(dir, "a.txt") + mustWrite(t, stale, "old") + m := &syncMock{objects: map[string]syncObject{ + "a.txt": {size: 4, mtime: time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z")}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + if body, _ := os.ReadFile(stale); string(body) != "data" { + t.Errorf("the stale file was not replaced: %q", body) + } +} + +// S3 -> S3 is server-side: the bytes never travel through koc. +func TestRunSyncS3ToS3IsServerSide(t *testing.T) { + m := &syncMock{objects: map[string]syncObject{ + "2026/a.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, s3Side("db-backups", "2026/"), s3Side("archive", "old/"), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + want := "/db-backups/2026/a.sql.gz -> /archive/old/a.sql.gz" + if got := strings.Join(m.copies, ","); got != want { + t.Errorf("copies = %q, want %q", got, want) + } + if len(m.gets) != 0 { + t.Errorf("objects were fetched (%v); a server-side copy transfers nothing", m.gets) + } +} + +// --delete is what makes this a mirror: a destination entry the source does not +// have goes away, batched into one request. +func TestRunSyncDeleteRemovesExtraObjects(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "keep.txt"), "1234") + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + m := &syncMock{objects: map[string]syncObject{ + "keep.txt": {size: 4, mtime: future}, + "stale.txt": {size: 4, mtime: future}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del = true + var buf bytes.Buffer + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(m.deletes, ","), "stale.txt"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } + if len(m.puts) != 0 { + t.Errorf("uploaded %v, want nothing — only the extra object changed", m.puts) + } +} + +func TestRunSyncDeleteRemovesExtraLocalFiles(t *testing.T) { + dir := t.TempDir() + stale := filepath.Join(dir, "sub", "stale.txt") + mustWrite(t, stale, "old") + m := &syncMock{objects: map[string]syncObject{}} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del, f.force = true, true // the source is empty on purpose here + var buf bytes.Buffer + if err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), f, &buf); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("the extra local file was not deleted") + } + // The directory it was in stays: an empty directory is not something the + // source can be said to have deleted. + if _, err := os.Stat(filepath.Dir(stale)); err != nil { + t.Errorf("the containing directory was removed: %v", err) + } +} + +// The sharp edge: a mistyped source that lists nothing must not erase the +// destination. +func TestRunSyncDeleteRefusesAnEmptySource(t *testing.T) { + dir := t.TempDir() // empty + m := &syncMock{objects: map[string]syncObject{ + "a.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + "b.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del = true + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf) + if err == nil { + t.Fatal("an empty source was allowed to delete the whole destination") + } + if !strings.Contains(err.Error(), "--force") || !strings.Contains(err.Error(), "2 entries") { + t.Errorf("error %q does not say what it refused and how to override", err) + } + if len(m.deletes) != 0 { + t.Errorf("deleted %v despite refusing", m.deletes) + } + + // With --force it goes ahead, because that is now an explicit instruction. + f.force = true + buf.Reset() + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(sortedStrings(m.deletes), ","), "a.txt,b.txt"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } +} + +// An entry the filter excludes is outside the sync altogether: it is neither +// transferred nor deleted. +func TestRunSyncFilterExcludesFromBothSides(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "a.sql.gz"), "1234") + mustWrite(t, filepath.Join(dir, "a.sha256"), "12") + m := &syncMock{objects: map[string]syncObject{ + "other.sha256": {size: 9, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del = true + f.filter = keyFilter{include: []string{"*.sql.gz"}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(m.puts, ","), "/b/a.sql.gz"; got != want { + t.Errorf("uploaded = %q, want only the included file", got) + } + if len(m.deletes) != 0 { + t.Errorf("deleted %v — an excluded destination entry is not the sync's to remove", m.deletes) + } +} + +// --dry-run must name every transfer and deletion it would perform, and perform +// none of them. +func TestRunSyncDryRun(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "new.txt"), "12") + m := &syncMock{objects: map[string]syncObject{ + "stale.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del, f.dryRun = true, true + var buf bytes.Buffer + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if len(m.puts)+len(m.deletes)+len(m.copies) != 0 { + t.Errorf("--dry-run wrote: puts=%v deletes=%v copies=%v", m.puts, m.deletes, m.copies) + } + out := buf.String() + if !strings.Contains(out, "Would upload") || !strings.Contains(out, "Would delete object: b/stale.txt") { + t.Errorf("output = %q, want both planned actions named", out) + } +} + +// A local side that is a single file, or an existing non-directory destination, +// is a mistake worth naming before any transfer. +func TestRunSyncChecksLocalSides(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("a request was made despite an invalid local side: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + }) + + file := filepath.Join(t.TempDir(), "one.txt") + mustWrite(t, file, "x") + + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(file), s3Side("b", ""), testSyncFlags(), &buf) + if err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Errorf("err = %v, want it to reject a file as the source", err) + } + + err = runSync(context.Background(), client, s3Side("b", ""), localSide(file), testSyncFlags(), &buf) + if err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Errorf("err = %v, want it to reject a file as the destination", err) + } + + err = runSync(context.Background(), client, localSide(filepath.Join(t.TempDir(), "absent")), + s3Side("b", ""), testSyncFlags(), &buf) + if err == nil || !strings.Contains(err.Error(), "reading source") { + t.Errorf("err = %v, want it to name the missing source", err) + } +} + +// A local destination that does not exist yet is an empty index, not an error: +// it is about to be created. +func TestRunSyncCreatesAMissingLocalDestination(t *testing.T) { + dir := filepath.Join(t.TempDir(), "new", "deep") + m := &syncMock{objects: map[string]syncObject{ + "a.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + if err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), testSyncFlags(), &buf); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "a.txt")); err != nil { + t.Errorf("the destination tree was not created: %v", err) + } +} + +// A directory marker — a zero-length key ending in "/" that some tools leave +// behind — has no counterpart on a filesystem and must not become a file. +func TestRunSyncSkipsDirectoryMarkers(t *testing.T) { + dir := t.TempDir() + m := &syncMock{objects: map[string]syncObject{ + "2026/": {size: 0, mtime: "2026-09-01T10:00:00.000Z"}, + "2026/a.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + if err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), testSyncFlags(), &buf); err != nil { + t.Fatal(err) + } + if got := strings.Join(sortedStrings(m.gets), ","); got != "/b/2026/a.txt" { + t.Errorf("fetched = %q, want the marker skipped", got) + } +} + +// sortedStrings makes an assertion independent of completion order, which with +// more than one worker is not the listing order. +func sortedStrings(in []string) []string { + out := append([]string(nil), in...) + sort.Strings(out) + return out +}