diff --git a/AGENTS.md b/AGENTS.md index a04f2bc..7f49471 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -197,11 +197,14 @@ internal/auth/ one authenticated ProviderClient per invocation + per debug.go --debug transport: redacts tokens+secrets, elides large bodies internal/kube/ minimal read-only k8s REST client (kubeconfig + secret/Ironic reads); no client-go internal/vault/ minimal Vault REST client (AppRole login / token + KV v2 read + X-Vault-Namespace) -internal/s3/ minimal S3 REST client: hand-rolled SigV4 + list/head/get/put (no aws-sdk-go-v2 / minio-go) +internal/s3/ minimal S3 REST client, no aws-sdk-go-v2 / minio-go: hand-rolled SigV4 + (header + presigned query), list/head/get/put, multipart upload, + server-side copy, batch delete, versioning, retry with backoff internal/output/ -f/--format {table,json,yaml,value,csv} and -c/--column layer internal/cli/keyvrm/ KeyVRM (in-house catalog service); typed request layer (types.go/requests.go) + cobra verbs internal/cli/vault/ "koc vault kv" list/get/copy/export/decrypt (package vaultcli); Vault creds only -internal/cli/s3/ "koc s3" bucket/object list+show, download, upload (package s3cli); S3 creds only +internal/cli/s3/ "koc s3" bucket/object lifecycle, du, download/upload (multipart, stdin, + recursive), server-side copy/move, presign, sync (package s3cli); S3 creds only internal/cli/quota/ "koc quota show|set" — the one cross-service noun (nova+cinder+neutron) internal/cli/ root.go wires every service's command group onto the root internal/cli/resolve/ cross-service name→ID (image→glance, network→neutron, project→keystone) diff --git a/README.md b/README.md index fe76103..ed7b614 100644 --- a/README.md +++ b/README.md @@ -521,17 +521,95 @@ S3 credentials alone are used, so it works on a host with no cloud credentials. ```sh koc s3 bucket list +koc s3 bucket show db-backups # exists? reachable? versioned? +koc s3 bucket create scratch +koc s3 bucket delete scratch # the bucket must be empty koc s3 object list db-backups +koc s3 object list db-backups --delimiter / # one level, like a directory koc s3 object show db-backups/ # HEAD only, no transfer +koc s3 object delete db-backups/ +koc s3 object delete db-backups/e2e- -r # every key under a prefix +koc s3 du db-backups --human --group # size, per storage class koc s3 download db-backups/ ./dump.mbs.gz.enc koc s3 download db-backups/.sha256 - # "-" streams to stdout, so it pipes +koc s3 download db-backups/2026/ ./restore -r koc s3 upload ./dump.mbs.gz.enc db-backups/ +koc s3 upload ./restore db-backups/2026/ -r +koc s3 copy db-backups/ db-backups/latest.mbs.gz.enc +koc s3 move db-backups/ archive/ +koc s3 presign db-backups/ --expire 1h +koc s3 sync ./restore s3://db-backups/2026/ --delete ``` +Every ref also accepts the `s3:///` spelling, so a path copied from +`s5cmd` or `aws s3` pastes in unchanged, and a wildcard (`"db-backups/2026/*.gz"`) +selects many where a command takes one. + `bucket list` is scoped to the **access key**, not to the store: Garage answers with the buckets that key is granted, so a key made for one bucket lists exactly that one. +**Transfers.** `upload` has no 5 GiB ceiling: anything past `--part-size` (16 MiB +by default) goes out as a multipart upload, `--concurrency` parts at a time, and +a failure aborts it so a half-written object never becomes visible. `-` as the +source reads **standard input**, which is what makes a dump streamable without +staging it on disk: + +```sh +mysqldump --all-databases | gzip | koc s3 upload - db-backups/nightly.sql.gz +``` + +`--recursive` on `download`, `upload`, `copy`, `move` and `object delete` works +on a whole prefix or tree, `--concurrency` objects at a time — which is most of +why a bulk restore finishes, since a thousand small objects are otherwise a +thousand serial round trips. `--include`/`--exclude` take globs (where `*` spans +`/`, as in s5cmd) and `--dry-run` prints exactly what would move. + +`copy` and `move` are **server-side**: the source is named in a header, so the +bytes never travel through koc and a 100 GiB object is one small request. A move +is the copy and then the delete, in that order — a failure leaves the source +intact rather than losing the object. + +**`sync`** makes a destination match a source, transferring only what differs — +local→S3, S3→local, or S3→S3 (server-side). It is the one command in the group +that *requires* the `s3://` prefix to mark a remote side: it has two sides, and +mistaking which is local is how a `--delete` removes the wrong one. An object +moves when it is absent, when the sizes differ, or when the source is newer; +`--size-only` drops the timestamp test, which is the right rule for content that +never changes in place (a dated backup). The S3-side timestamp is the object's +Last-Modified, because S3 keeps no record of the source file's mtime and a +listing would not return one if it did — correct in the steady state, with one +seam: a tree restored by `download` carries the restore time, so syncing it +*back* re-uploads once and then settles. `--delete` turns the sync into a mirror +and is refused when the source turned out to be empty unless `--force` is also +given, so a mistyped source cannot erase the destination. + +`object delete --recursive` is the one destructive shape in the group, so it is +never inferred from a trailing slash — and `--dry-run` prints exactly the keys it +would remove without touching any of them. It is also how a bucket is emptied +before `bucket delete`, which never removes objects implicitly. Keys go out in +batches of up to 1000 per request, so emptying a large bucket costs a thousandth +of the round trips. + +`du` is a listing folded into a sum, because S3 has no "how big is this" call: +no object is downloaded, but every key is walked, one request per 1000 of them. +Sizes are exact bytes everywhere unless `--human` is given — a rounded +"14.2 GiB" is not a number a script can add up. + +`presign` prints a URL that reads (or, with `--method put`, writes) one object +with no credentials attached to the recipient. It makes **no request** — pure +local signing — so it works offline, and the URL is a bearer credential for that +key until it expires: treat it like a password and keep `--expire` short. + +**Versioning** (`bucket show`, `bucket set --versioning`, `--all-versions`, +`--version-id`) is implemented but Garage has none — it reports every bucket as +unversioned — so those are for a koc pointed at AWS, Ceph RGW or MinIO. + +A failed request is retried with exponential backoff (`--s3-retries`, 5 by +default): on a cluster network a reset connection mid-transfer should not cost +the whole command. `--s3-anonymous` sends requests unsigned, for a bucket +granted to everyone. + Credentials come from flags, from the environment, or from a Kubernetes Secret: | flag | env | default | @@ -542,6 +620,8 @@ Credentials come from flags, from the environment, or from a Kubernetes Secret: | `--s3-region` | `AWS_REGION`, `AWS_DEFAULT_REGION`, `S3_REGION`, `s3_region` | `garage` | | `--s3-cacert` | `AWS_CA_BUNDLE`, `S3_CACERT` | system roots | | `--s3-creds-from-ns` | `KOC_S3_CREDS_FROM_NS` | — | +| `--s3-anonymous` | `S3_ANONYMOUS` | off (no credentials needed or used) | +| `--s3-retries` | — | `5` | | `--insecure-s3` | `S3_SKIP_VERIFY` | off (the global `--insecure` also applies) | The three env families are deliberate: `AWS_*` so an existing `aws`/`boto` @@ -590,7 +670,7 @@ internal/kube/ minimal read-only k8s REST client (no client-go) internal/vault/ minimal Vault REST client (AppRole/token + KV v2) internal/s3/ minimal S3 REST client (SigV4, no aws-sdk/minio-go) internal/cli/vault/ "koc vault kv" list/get/copy/export/decrypt, no Keystone auth -internal/cli/s3/ "koc s3" bucket/object/download/upload, no Keystone auth +internal/cli/s3/ "koc s3" bucket/object/du/transfer/copy/presign, no Keystone auth internal/output/ -f/-c formatter (table/json/yaml/value/csv) internal/cli/ root command wiring internal/cli/resolve/ cross-service name→ID resolution diff --git a/docs/coverage.md b/docs/coverage.md index 7961a6d..841299c 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -3,7 +3,7 @@ How much of the upstream OpenStack CLI surface `koc` implements, measured against primary sources rather than documentation. -**Snapshot:** 2026-09-03 · `koc` @ this commit (base `3ba95db`) · 555 leaf +**Snapshot:** 2026-09-14 · `koc` @ this commit (base `82afd0a`) · 565 leaf commands (visible tree; 2 more are hidden duplicates). **Keep this file current** — see "Updating this document" below. Any commit that @@ -28,8 +28,8 @@ PyPI is the source of record. ## Headline -**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 555 leaf commands, -515 are upstream-equivalent and 40 are koc-native. +**515 of 844 in-scope upstream commands (61%).** Of `koc`'s 565 leaf commands, +515 are upstream-equivalent and 50 are koc-native. The denominator grew by 13 against the 2026-08-07 snapshot without a single command changing: `python-ironic-inspector-client` is now a **baseline** rather @@ -433,14 +433,14 @@ limitations"; the fix is to make the other resolvers match `server`'s behaviour. ## koc-native commands -No upstream equivalent, by design — **40 leaves**, itemised so the total -reconciles with the headline (555 = 515 + 40): +No upstream equivalent, by design — **50 leaves**, itemised so the total +reconciles with the headline (565 = 515 + 50): | Count | Commands | Why it has no upstream equivalent | | --- | --- | --- | | 18 | `koc keyvrm …` — `app-config` ×2, `availability-zone` ×2, `event` ×4, `host-aggregate-config` ×5, `recommendation` ×5 | in-house KeyVRM catalog service; no gophercloud package and no OSC plugin | | 5 | `koc vault kv list/get/copy/export/decrypt` | Vault is not an OpenStack service; `copy` fills a gap in the Vault CLI itself | -| 5 | `koc s3 bucket list`, `koc s3 object list/show`, `koc s3 download/upload` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | +| 15 | `koc s3 bucket list/create/delete/show/set`, `koc s3 object list/show/delete`, `koc s3 du`, `koc s3 download/upload`, `koc s3 copy/move`, `koc s3 presign`, `koc s3 sync` | S3 is not an OpenStack service. Upstream's object-store commands speak **Swift**, which is a different API and is counted separately as not targeted (`openstack.object_store.v1`, 0/17); these talk to the LCM cluster's Garage, which holds GitLab's object storage and the `backup-db` pipeline's MariaDB dumps | | 2 | `koc dns pool list/show` | designate's API and its Python SDK both expose `/v2/pools`, but `python-designateclient` registers no `openstack` command for it. Reads only — pool *writes* are a `designate-manage`/config operation on the servers | | 2 | `koc server add/remove server-group` | KeyStack dynamic server groups | | 2 | `koc network trunk subport add`/`remove` | upstream folds these into `network trunk set`/`unset --subport` flags rather than giving them verbs (`network subport list` does exist and is counted — see "Naming deviations") | @@ -481,7 +481,7 @@ The tables are derived, not hand-maintained. To re-derive after a version bump or a batch of new commands: ```sh -# 1. koc's own command tree (553 leaf commands at the snapshot above) +# 1. koc's own command tree (565 leaf commands at the snapshot above) make build # Walk `--help` recursively. Count a command when it is *runnable*, not merely when # it is childless: `koc image import ` is a verb that also parents `koc image @@ -511,7 +511,7 @@ Then **check the arithmetic**, because that is the only thing that makes these tables worth reading. Three identities must hold at every snapshot: 1. every raw row numerator summed = the headline numerator (515); -2. leaf commands = headline numerator + koc-native (555 = 515 + 40); +2. leaf commands = headline numerator + koc-native (565 = 515 + 50); 3. every raw row denominator summed = 901, and minus the two not-targeted rows (swift 17 + manila 40) = the in-scope denominator (844). diff --git a/internal/cli/s3/bucket.go b/internal/cli/s3/bucket.go index ea84bde..a22ab60 100644 --- a/internal/cli/s3/bucket.go +++ b/internal/cli/s3/bucket.go @@ -2,12 +2,15 @@ package s3cli import ( "context" + "errors" "fmt" "io" + "strings" "github.com/spf13/cobra" "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/cli/batchdelete" "github.com/ftarasenko/go-openstackclient/internal/output" "github.com/ftarasenko/go-openstackclient/internal/s3" ) @@ -20,6 +23,10 @@ func newBucketCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.C Short: "Manage buckets", } cmd.AddCommand(newBucketListCommand(a, o, f)) + cmd.AddCommand(newBucketCreateCommand(a, o, f)) + cmd.AddCommand(newBucketDeleteCommand(a, o, f)) + cmd.AddCommand(newBucketShowCommand(a, o, f)) + cmd.AddCommand(newBucketSetCommand(a, o, f)) return cmd } @@ -63,3 +70,232 @@ func runBucketList(ctx context.Context, client *s3.Client, o *output.Options, w } return o.WriteList(w, output.Table{Columns: []string{"Name", "Created"}, Rows: rows}) } + +const bucketCreateLong = `Create a bucket. + +The request states the bucket's location as the signing region (--s3-region, +default "garage"), which is the only value a store will accept: a request signed +for one region is not served by another. So there is no separate location flag — +set --s3-region and the bucket lands there. + +Creating a bucket the credentials already own fails rather than succeeding +quietly, so a script can tell "I created this" from "it was already there". On +Garage the key that creates a bucket owns it and needs no further grant.` + +const bucketCreateExample = ` koc s3 bucket create scratch + koc s3 bucket create s3://scratch` + +func newBucketCreateCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return &cobra.Command{ + Use: "create ", + Short: "Create a bucket", + Long: bucketCreateLong, + Example: bucketCreateExample, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, err := parseBucketRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketCreate(ctx, client, o, bucket, cmd.OutOrStdout()) + }, + } +} + +// runBucketCreate is the test seam for "bucket create". +func runBucketCreate(ctx context.Context, client *s3.Client, o *output.Options, + bucket string, w io.Writer) error { + if err := client.CreateBucket(ctx, bucket); err != nil { + switch s3.ErrorCode(err) { + case "BucketAlreadyOwnedByYou": + return fmt.Errorf("bucket %q already exists and is owned by these credentials", bucket) + case "BucketAlreadyExists": + return fmt.Errorf("bucket %q already exists and is owned by someone else", bucket) + } + return fmt.Errorf("creating bucket %q on %s: %w", bucket, client.Endpoint(), err) + } + return o.WriteSingle(w, []string{"Bucket", "Region"}, []any{bucket, client.Region()}) +} + +const bucketDeleteLong = `Delete one or more empty buckets. + +A bucket that still holds objects is refused by the server — nothing is ever +removed implicitly. Empty it first: + + koc s3 object delete / --recursive + +Every bucket named is attempted even if an earlier one fails, and the command +exits non-zero naming each failure.` + +func newBucketDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return &cobra.Command{ + Use: "delete [ ...]", + Short: "Delete empty buckets", + Long: bucketDeleteLong, + Aliases: []string{"remove"}, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketDelete(ctx, client, args, cmd.OutOrStdout()) + }, + } +} + +// runBucketDelete is the test seam for "bucket delete". It follows koc's batch +// contract: attempt every ref, join the failures. +func runBucketDelete(ctx context.Context, client *s3.Client, refs []string, w io.Writer) error { + return batchdelete.Each(refs, func(ref string) error { + bucket, err := parseBucketRef(ref) + if err != nil { + return err + } + if err := client.DeleteBucket(ctx, bucket); err != nil { + if s3.ErrorCode(err) == "BucketNotEmpty" { + return fmt.Errorf("bucket %q is not empty: delete its objects first "+ + "(koc s3 object delete %s/ --recursive)", bucket, bucket) + } + return fmt.Errorf("deleting bucket %q: %w", bucket, err) + } + _, err = fmt.Fprintf(w, "Deleted bucket: %s\n", bucket) + return err + }) +} + +const bucketShowLong = `Show a bucket: that it exists, that the credentials reach it, and its +versioning state. + +The existence check is a HEAD, which costs the server no listing work — the +cheapest possible probe, and the one to use in a script that must know whether +a bucket is there before writing to it. + +Versioning is reported as Enabled, Suspended, or Unversioned for a bucket that +was never configured. Garage answers Unversioned for every bucket because it +implements no versioning at all, so this is also how to tell which kind of store +is on the other end. A store that does not answer the versioning call leaves the +field unknown rather than failing the command.` + +func newBucketShowCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return &cobra.Command{ + Use: "show ", + Short: "Show a bucket's existence and versioning state", + Long: bucketShowLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, err := parseBucketRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketShow(ctx, client, o, bucket, cmd.OutOrStdout()) + }, + } +} + +// runBucketShow is the test seam for "bucket show". +func runBucketShow(ctx context.Context, client *s3.Client, o *output.Options, + bucket string, w io.Writer) error { + if err := client.HeadBucket(ctx, bucket); err != nil { + if s3.IsNotFound(err) { + return fmt.Errorf("no bucket %q on %s (or the credentials cannot see it)", bucket, client.Endpoint()) + } + return fmt.Errorf("reading bucket %q: %w", bucket, err) + } + + // The bucket is there, which is the answer asked for; a store that does not + // implement the versioning call must not turn that into a failure. + versioning := "unknown" + if status, err := client.GetBucketVersioning(ctx, bucket); err == nil { + versioning = status + } + return o.WriteSingle(w, + []string{"Bucket", "Endpoint", "Region", "Versioning"}, + []any{bucket, client.Endpoint(), client.Region(), versioning}) +} + +const bucketSetLong = `Change a bucket's settings. Only versioning is settable. + +--versioning enabled starts keeping every version of every key; suspended stops +without discarding what is already kept. S3 has no way back to the +never-versioned state, which is why "unversioned" is not accepted here. + +Garage implements no versioning, so this fails against it — that is the store +saying so, not koc. Use "koc s3 bucket show" to see which kind of store answers.` + +func newBucketSetCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + var versioning string + cmd := &cobra.Command{ + Use: "set ", + Short: "Change a bucket's settings", + Long: bucketSetLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, err := parseBucketRef(args[0]) + if err != nil { + return err + } + if !cmd.Flags().Changed("versioning") { + return errors.New("nothing to set: pass --versioning enabled|suspended") + } + status, err := versioningStatus(versioning) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runBucketSet(ctx, client, o, bucket, status, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&versioning, "versioning", "", + "object versioning: enabled or suspended") + return cmd +} + +// versioningStatus maps the flag's lower-case spelling to the protocol's. +// Accepting both keeps "--versioning Enabled" from being a puzzling error. +func versioningStatus(v string) (string, error) { + switch strings.ToLower(v) { + case "enabled": + return s3.VersioningEnabled, nil + case "suspended": + return s3.VersioningSuspended, nil + default: + return "", fmt.Errorf("--versioning must be enabled or suspended, got %q", v) + } +} + +// runBucketSet is the test seam for "bucket set". +func runBucketSet(ctx context.Context, client *s3.Client, o *output.Options, + bucket, status string, w io.Writer) error { + if err := client.SetBucketVersioning(ctx, bucket, status); err != nil { + return fmt.Errorf("setting versioning on %q: %w", bucket, err) + } + return o.WriteSingle(w, []string{"Bucket", "Versioning"}, []any{bucket, status}) +} diff --git a/internal/cli/s3/bucket_test.go b/internal/cli/s3/bucket_test.go new file mode 100644 index 0000000..d5d963b --- /dev/null +++ b/internal/cli/s3/bucket_test.go @@ -0,0 +1,135 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" +) + +func TestRunBucketCreate(t *testing.T) { + var gotMethod, gotPath string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketCreate(context.Background(), client, valueOpts(), "scratch", &buf); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut || gotPath != "/scratch" { + t.Errorf("request = %s %s, want PUT /scratch", gotMethod, gotPath) + } + // WriteSingle in value format is one field per line. + if got, want := buf.String(), "scratch\ngarage\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// An existing bucket must produce advice, not a raw 409: the two S3 codes mean +// different things to the operator ("yours" vs "someone else's"). +func TestRunBucketCreateAlreadyExists(t *testing.T) { + for _, tc := range []struct{ code, want string }{ + {"BucketAlreadyOwnedByYou", "owned by these credentials"}, + {"BucketAlreadyExists", "owned by someone else"}, + } { + t.Run(tc.code, func(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprintf(w, `%sx`, tc.code) + }) + + var buf bytes.Buffer + err := runBucketCreate(context.Background(), client, valueOpts(), "scratch", &buf) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + if buf.Len() != 0 { + t.Errorf("wrote %q on failure, want nothing", buf.String()) + } + }) + } +} + +func TestRunBucketDelete(t *testing.T) { + var paths []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runBucketDelete(context.Background(), client, []string{"scratch", "s3://spare"}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(paths, ","), "DELETE /scratch,DELETE /spare"; got != want { + t.Errorf("requests = %q, want %q", got, want) + } + want := "Deleted bucket: scratch\nDeleted bucket: spare\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A non-empty bucket is the expected failure, and the message must name the +// command that fixes it rather than leaving the operator with "409 Conflict". +func TestRunBucketDeleteNotEmptyExplains(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprint(w, `BucketNotEmptyx`) + }) + + var buf bytes.Buffer + err := runBucketDelete(context.Background(), client, []string{"db-backups"}, &buf) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "koc s3 object delete db-backups/ --recursive") { + t.Errorf("error = %q, want it to suggest the recursive delete", err) + } +} + +// The batch contract: a failing ref must not skip the ones after it. +func TestRunBucketDeleteAttemptsEveryRef(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/locked" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runBucketDelete(context.Background(), client, []string{"locked", "scratch"}, &buf) + if err == nil { + t.Fatal("expected the failing ref to be reported") + } + if got, want := buf.String(), "Deleted bucket: scratch\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A bucket ref carrying a key is a typo, and on a create it would silently make +// a bucket the operator did not name. +func TestParseBucketRefRejectsKey(t *testing.T) { + if _, err := parseBucketRef("scratch/keys"); err == nil { + t.Error("expected scratch/keys to be rejected as a bucket ref") + } + for _, ref := range []string{"scratch", "s3://scratch", "scratch/"} { + got, err := parseBucketRef(ref) + if err != nil { + t.Errorf("parseBucketRef(%q) = %v", ref, err) + continue + } + if got != "scratch" { + t.Errorf("parseBucketRef(%q) = %q, want scratch", ref, got) + } + } +} diff --git a/internal/cli/s3/client.go b/internal/cli/s3/client.go index e423f27..7ec015d 100644 --- a/internal/cli/s3/client.go +++ b/internal/cli/s3/client.go @@ -1,5 +1,7 @@ -// Package s3cli implements the koc-specific "koc s3 ..." command group: list -// buckets and objects, and move files in and out of an S3-compatible store. +// Package s3cli implements the koc-specific "koc s3 ..." command group: the +// bucket and object lifecycle of an S3-compatible store, and moving data in and +// out of it — including multipart and streamed uploads, recursive and wildcard +// transfers, server-side copy/move, and presigned URLs. // // It has no python-openstackclient equivalent. S3 is not an OpenStack service — // upstream's object-store commands speak Swift, which KeyStack does not deploy — @@ -33,6 +35,13 @@ import ( // the group's Long help for where the other keys live). const defaultCredsSecret = "gitlab-object-storage" //nolint:gosec // G101: name of a k8s Secret object, not a credential value +// defaultRetries is --s3-retries' default. A koc S3 call can be a transfer of +// several gigabytes across a cluster network, where one reset connection should +// not cost the whole command; s5cmd defaults to ten for the same reason, and +// this is lower only because koc's backoff is not jittered across hundreds of +// workers. +const defaultRetries = 5 + // connFlags holds the "koc s3" connection and credential flags. They are // persistent flags on the group rather than global ones: every other koc command // needs Keystone, none of them needs these, and koc --help is long enough. @@ -45,6 +54,8 @@ type connFlags struct { credsFromNS string insecure bool noPathStyle bool + anonymous bool + retries int fs *pflag.FlagSet // to tell "unset" from "explicitly empty" } @@ -72,6 +83,10 @@ func (f *connFlags) addTo(fs *pflag.FlagSet) { "disable TLS verification for the S3 endpoint (env S3_SKIP_VERIFY)") fs.BoolVar(&f.noPathStyle, "no-path-style", false, "address buckets as . instead of /; needs a wildcard DNS record") + fs.BoolVar(&f.anonymous, "s3-anonymous", auth.EnvBool("S3_ANONYMOUS"), + "send requests unsigned, for a publicly readable bucket (env S3_ANONYMOUS); no credentials are needed or used") + fs.IntVar(&f.retries, "s3-retries", defaultRetries, + "retries for a failed request, with exponential backoff (0 = none)") } // client builds the S3 client. --timeout, --debug, --insecure, --kubeconfig and @@ -85,9 +100,14 @@ func (f *connFlags) client(ctx context.Context, a *auth.Options) (*s3.Client, er PathStyle: !f.noPathStyle, // The global --insecure is honoured too: a user who typed it once meant // it for the endpoint they are talking to. - Insecure: f.insecure || a.Insecure, - Timeout: a.Timeout, - Debug: a.Debug, + Insecure: f.insecure || a.Insecure, + Timeout: a.Timeout, + Debug: a.Debug, + Anonymous: f.anonymous, + MaxRetries: f.retries, + } + if f.retries < 0 { + return nil, fmt.Errorf("--s3-retries cannot be negative, got %d", f.retries) } if f.credsFromNS != "" { @@ -289,6 +309,21 @@ func parseRef(ref string) (bucket, key string, err error) { return bucket, key, nil } +// parseBucketRef is parseRef where a key is not accepted at all. "scratch/x" as +// a bucket name is almost always a mistyped object ref, and on a create that +// would make a bucket literally named "scratch" while the operator believed +// they had addressed a path. +func parseBucketRef(ref string) (string, error) { + bucket, key, err := parseRef(ref) + if err != nil { + return "", err + } + if strings.Trim(key, "/") != "" { + return "", fmt.Errorf("%q names an object, not a bucket: expected or s3://", ref) + } + return bucket, nil +} + // parseObjectRef is parseRef where the key is mandatory. func parseObjectRef(ref string) (bucket, key string, err error) { bucket, key, err = parseRef(ref) diff --git a/internal/cli/s3/copy.go b/internal/cli/s3/copy.go new file mode 100644 index 0000000..c07150b --- /dev/null +++ b/internal/cli/s3/copy.go @@ -0,0 +1,300 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// copyFlags holds the options accepted by "copy" and "move". +type copyFlags struct { + recursive bool + dryRun bool + flatten bool + versionID string + contentType string + concurrency int + filter keyFilter + // remove makes this a move: the source is deleted once the copy lands. + remove bool +} + +const copyLong = `Copy objects inside the store, without the bytes travelling through koc. + +The source is named in a header, so the store does the copying: a 100 GiB +object is one small request, at no egress cost and no local disk. That is what +makes this the way to promote, rename or archive a backup — as opposed to a +download followed by an upload, which moves every byte twice through whatever +link koc is on. + +--recursive copies every object under the source key, treated as a prefix, to +the destination key as a prefix; a wildcard source does the same without the +flag. Each object keeps its path relative to the source prefix, or goes straight +under the destination with --flatten. + +--version-id copies one specific version of one object (Garage implements no +versioning).` + +const copyExample = ` # Promote last night's dump to a stable name + koc s3 copy db-backups/nightly-2026-09-13.sql.gz db-backups/latest.sql.gz + + # Archive a month into another bucket + koc s3 copy db-backups/2026/08/ archive/2026-08/ --recursive + + # Everything matching a pattern + koc s3 copy "db-backups/*.sha256" checksums/ --recursive` + +const moveLong = `Move objects inside the store: a server-side copy, then a delete of the source. + +S3 has no atomic move, so this is the two calls in order — the source is only +removed once the copy has landed, and a failure therefore leaves the source +intact rather than losing the object. A move interrupted between the two leaves +both copies, never neither. + +Every flag behaves as for "koc s3 copy".` + +func newCopyCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return newCopyLikeCommand(a, o, f, copySpec{ + use: "copy / /", + short: "Copy objects inside the store (server-side)", + long: copyLong, + example: copyExample, + aliases: []string{"cp"}, + }) +} + +func newMoveCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + return newCopyLikeCommand(a, o, f, copySpec{ + use: "move / /", + short: "Move objects inside the store (server-side copy, then delete)", + long: moveLong, + aliases: []string{"mv"}, + remove: true, + }) +} + +// copySpec is the per-verb text of the two commands copy.go builds, which are +// the same command bar the trailing delete. +type copySpec struct { + use, short, long, example string + aliases []string + remove bool +} + +func newCopyLikeCommand(a *auth.Options, o *output.Options, f *connFlags, spec copySpec) *cobra.Command { + cf := ©Flags{remove: spec.remove} + cmd := &cobra.Command{ + Use: spec.use, + Short: spec.short, + Long: spec.long, + Example: spec.example, + Aliases: spec.aliases, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + src, dst, err := parseCopyRefs(args[0], args[1], cf) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runCopy(ctx, client, o, src, dst, cf, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVarP(&cf.recursive, "recursive", "r", false, + "treat the source key as a prefix and copy everything under it") + fl.BoolVar(&cf.dryRun, "dry-run", false, "print what would be copied and copy nothing") + fl.BoolVar(&cf.flatten, "flatten", false, + "put every object directly under the destination prefix instead of mirroring its path") + fl.StringVar(&cf.versionID, "version-id", "", "copy this version of the source instead of the current one") + fl.StringVar(&cf.contentType, "content-type", "", + "Content-Type for the destination; without it the source's metadata is carried over") + fl.IntVar(&cf.concurrency, "concurrency", s3.DefaultConcurrency, + "objects to copy at once under --recursive") + cf.filter.addTo(fl) + return cmd +} + +// parseCopyRefs validates the two references and the flag combination. +func parseCopyRefs(srcRef, dstRef string, f *copyFlags) (src, dst s3.ObjectRef, err error) { + srcBucket, srcKey, err := parseRef(srcRef) + if err != nil { + return src, dst, err + } + dstBucket, dstKey, err := parseRef(dstRef) + if err != nil { + return src, dst, err + } + src = s3.ObjectRef{Bucket: srcBucket, Key: srcKey} + dst = s3.ObjectRef{Bucket: dstBucket, Key: dstKey} + + bulk := f.recursive || hasGlob(srcKey) + switch { + case bulk && f.versionID != "": + return src, dst, errors.New("--version-id names one object; it cannot be combined with --recursive") + case !bulk && srcKey == "": + return src, dst, fmt.Errorf("%q names no object: expected /", srcRef) + case !bulk && f.filter.active(): + return src, dst, errors.New("--include/--exclude filter a listing; pass --recursive or a wildcard source") + case !bulk && f.flatten: + return src, dst, errors.New("--flatten only applies to a recursive copy") + case !bulk && src == dst: + return src, dst, fmt.Errorf("source and destination are the same object (%s)", src) + } + return src, dst, f.filter.compile() +} + +// runCopy is the test seam for "copy" and "move". +func runCopy(ctx context.Context, client *s3.Client, o *output.Options, + src, dst s3.ObjectRef, f *copyFlags, w io.Writer) error { + if f.recursive || hasGlob(src.Key) { + return runCopyRecursive(ctx, client, src, dst, f, w) + } + + // A destination ending in "/" — or naming only a bucket — takes the + // source's basename, the same rule "upload" follows. + if dst.Key == "" || strings.HasSuffix(dst.Key, "/") { + dst.Key += keyBase(src.Key) + } + if f.dryRun { + return sayCopy(w, src, dst, f.remove, true) + } + + if err := copyOne(ctx, client, src, dst, f); err != nil { + return err + } + // One object, one summary row — the per-object progress lines belong to the + // recursive form, where there is more than one to follow. + return o.WriteSingle(w, []string{"Source", "Destination"}, []any{src.String(), dst.String()}) +} + +// runCopyRecursive copies every matching object under a prefix. +func runCopyRecursive(ctx context.Context, client *s3.Client, src, dst s3.ObjectRef, + f *copyFlags, w io.Writer) error { + prefix := globPrefix(src.Key) + var glob *regexp.Regexp + if hasGlob(src.Key) { + var err error + if glob, err = compileGlob(src.Key); err != nil { + return fmt.Errorf("%s: %w", src, err) + } + } + + out := &syncWriter{w: w} + p, workCtx := newPool(ctx, f.concurrency) + taken := 0 + + err := client.ListObjectsFunc(ctx, src.Bucket, s3.ListOptions{Prefix: prefix}, func(obj s3.Object) error { + if glob != nil && !glob.MatchString(obj.Key) { + return nil + } + if !f.filter.match(obj.Key) { + return nil + } + from := s3.ObjectRef{Bucket: src.Bucket, Key: obj.Key} + to := s3.ObjectRef{Bucket: dst.Bucket, Key: copyDestKey(dst.Key, prefix, obj.Key, f.flatten)} + if from == to { + // Copying an object onto itself is refused by S3 unless metadata + // changes, and in a recursive run it is a mistake, not a request. + return nil + } + taken++ + if !p.run(workCtx, func() error { + if f.dryRun { + return sayCopy(out, from, to, f.remove, true) + } + if err := copyOne(workCtx, client, from, to, f); err != nil { + return err + } + return sayCopy(out, from, to, f.remove, false) + }) { + return errStopListing + } + return nil + }) + if waitErr := p.wait(); err == nil || isStopListing(err) { + err = waitErr + } + if err != nil && !isStopListing(err) { + return fmt.Errorf("copying %s: %w", src, err) + } + if taken == 0 { + _, err = fmt.Fprintf(w, "No objects under %s\n", src) + return err + } + return nil +} + +// copyOne performs the copy and, for a move, the delete that follows it. +func copyOne(ctx context.Context, client *s3.Client, src, dst s3.ObjectRef, f *copyFlags) error { + if _, err := client.CopyObject(ctx, src, dst, f.versionID, f.contentType); err != nil { + return err + } + if !f.remove { + return nil + } + // Only now: a delete before the copy has landed would lose the object. + if err := client.DeleteObject(ctx, src.Bucket, src.Key, f.versionID); err != nil { + return fmt.Errorf("copied %s to %s but could not delete the source: %w", src, dst, err) + } + return nil +} + +// copyDestKey maps a source key to its destination key. +func copyDestKey(dstPrefix, srcPrefix, key string, flatten bool) string { + rel := key + if flatten { + rel = keyBase(key) + } else if srcPrefix != "" { + rel = strings.TrimPrefix(strings.TrimPrefix(key, srcPrefix), "/") + } + if rel == "" { + rel = keyBase(key) + } + if dstPrefix == "" { + return rel + } + return strings.TrimSuffix(dstPrefix, "/") + "/" + rel +} + +// keyBase is path.Base for an object key: the part after the last "/". It is +// not filepath.Base, which would split on "\\" on Windows and mangle a key that +// legitimately contains one. +func keyBase(key string) string { + if i := strings.LastIndex(key, "/"); i >= 0 { + return key[i+1:] + } + return key +} + +// sayCopy prints one progress line, naming the operation the flags asked for. +func sayCopy(w io.Writer, src, dst s3.ObjectRef, remove, dryRun bool) error { + var verb string + switch { + case dryRun && remove: + verb = "Would move" + case dryRun: + verb = "Would copy" + case remove: + verb = "Moved" + default: + verb = "Copied" + } + _, err := fmt.Fprintf(w, "%s: %s -> %s\n", verb, src, dst) + return err +} diff --git a/internal/cli/s3/copy_test.go b/internal/cli/s3/copy_test.go new file mode 100644 index 0000000..e43bd2d --- /dev/null +++ b/internal/cli/s3/copy_test.go @@ -0,0 +1,292 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "sort" + "strings" + "testing" + + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +func testCopyFlags() *copyFlags { return ©Flags{concurrency: 1} } + +func TestRunCopySingleObject(t *testing.T) { + var gotPath, gotSource string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotPath, gotSource = r.URL.Path, r.Header.Get("x-amz-copy-source") + _, _ = fmt.Fprint(w, `"e"`) + }) + + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "nightly.sql.gz"} + dst := s3.ObjectRef{Bucket: "db-backups", Key: "latest.sql.gz"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, testCopyFlags(), &buf); err != nil { + t.Fatal(err) + } + if gotPath != "/db-backups/latest.sql.gz" { + t.Errorf("path = %q, want the destination", gotPath) + } + if gotSource != "/db-backups/nightly.sql.gz" { + t.Errorf("copy source = %q", gotSource) + } + if got, want := buf.String(), "db-backups/nightly.sql.gz\ndb-backups/latest.sql.gz\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A destination ending in "/" — or naming only a bucket — takes the source's +// basename, the same rule "upload" follows. +func TestRunCopyDestinationPrefixTakesTheBasename(t *testing.T) { + var gotPath string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = fmt.Fprint(w, `"e"`) + }) + + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "2026/08/a.sql.gz"} + dst := s3.ObjectRef{Bucket: "archive", Key: "restored/"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, testCopyFlags(), &buf); err != nil { + t.Fatal(err) + } + if gotPath != "/archive/restored/a.sql.gz" { + t.Errorf("path = %q, want the basename appended", gotPath) + } +} + +// A move is the copy plus a delete of the source, in that order: the source is +// only removed once the copy has landed. +func TestRunMoveCopiesThenDeletes(t *testing.T) { + var order []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + order = append(order, r.Method+" "+r.URL.Path) + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + _, _ = fmt.Fprint(w, `"e"`) + }) + + f := testCopyFlags() + f.remove = true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "a.sql.gz"} + dst := s3.ObjectRef{Bucket: "archive", Key: "a.sql.gz"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err != nil { + t.Fatal(err) + } + want := "PUT /archive/a.sql.gz,DELETE /db-backups/a.sql.gz" + if got := strings.Join(order, ","); got != want { + t.Errorf("requests = %q, want %q", got, want) + } +} + +// A failed copy must leave the source alone: losing the object is the one +// outcome a move must never have. +func TestRunMoveKeepsTheSourceWhenTheCopyFails(t *testing.T) { + var deleted bool + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + deleted = true + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `AccessDenied`) + }) + + f := testCopyFlags() + f.remove = true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "a.sql.gz"} + dst := s3.ObjectRef{Bucket: "archive", Key: "a.sql.gz"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err == nil { + t.Fatal("a refused copy was reported as a successful move") + } + if deleted { + t.Fatal("the source was deleted even though the copy failed") + } +} + +// A recursive copy mirrors each key's path below the source prefix. +func TestRunCopyRecursive(t *testing.T) { + var copies []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false + 2026/08/a.sql.gz + 2026/09/b.sql.gz + `) + return + } + copies = append(copies, r.Header.Get("x-amz-copy-source")+" -> "+r.URL.Path) + _, _ = fmt.Fprint(w, `"e"`) + }) + + f := testCopyFlags() + f.recursive = true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "2026/"} + dst := s3.ObjectRef{Bucket: "archive", Key: "old/"} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err != nil { + t.Fatal(err) + } + sort.Strings(copies) + want := []string{ + "/db-backups/2026/08/a.sql.gz -> /archive/old/08/a.sql.gz", + "/db-backups/2026/09/b.sql.gz -> /archive/old/09/b.sql.gz", + } + if got := strings.Join(copies, "|"); got != strings.Join(want, "|") { + t.Errorf("copies = %q, want %q", got, want) + } +} + +// A wildcard source is its own recursion. +func TestRunCopyWildcardSource(t *testing.T) { + var copies []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false + a.sql.gz + a.sha256 + `) + return + } + copies = append(copies, r.URL.Path) + _, _ = fmt.Fprint(w, `"e"`) + }) + + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: "*.sha256"} + dst := s3.ObjectRef{Bucket: "checksums", Key: ""} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, testCopyFlags(), &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(copies, ","), "/checksums/a.sha256"; got != want { + t.Errorf("copies = %q, want only the pattern match", got) + } +} + +// Copying an object onto itself is refused by S3 unless metadata changes, and +// in a recursive run over the same prefix it is a mistake, not a request. +func TestRunCopyRecursiveSkipsIdentityCopies(t *testing.T) { + var copies int + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false + a.sql.gz`) + return + } + copies++ + _, _ = fmt.Fprint(w, `"e"`) + }) + + f := testCopyFlags() + f.recursive = true + var buf bytes.Buffer + ref := s3.ObjectRef{Bucket: "db-backups", Key: ""} + if err := runCopy(context.Background(), client, valueOpts(), ref, ref, f, &buf); err != nil { + t.Fatal(err) + } + if copies != 0 { + t.Errorf("copies = %d, want none — every key maps onto itself", copies) + } + if !strings.Contains(buf.String(), "No objects under") { + t.Errorf("output = %q, want it to say nothing was copied", buf.String()) + } +} + +func TestRunCopyRecursiveDryRun(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if !r.URL.Query().Has("list-type") { + t.Errorf("--dry-run sent %s %s", r.Method, r.URL.Path) + } + _, _ = fmt.Fprint(w, `false + a.sql.gz`) + }) + + f := testCopyFlags() + f.recursive, f.dryRun, f.remove = true, true, true + var buf bytes.Buffer + src := s3.ObjectRef{Bucket: "db-backups", Key: ""} + dst := s3.ObjectRef{Bucket: "archive", Key: ""} + if err := runCopy(context.Background(), client, valueOpts(), src, dst, f, &buf); err != nil { + t.Fatal(err) + } + want := "Would move: db-backups/a.sql.gz -> archive/a.sql.gz\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestParseCopyRefsValidation(t *testing.T) { + for _, tc := range []struct { + name string + src, dst string + flags copyFlags + want string + }{ + {"no source key", "b", "b/k", copyFlags{}, "names no object"}, + {"same object", "b/k", "b/k", copyFlags{}, "the same object"}, + {"version with recursive", "b/p", "b/q", copyFlags{recursive: true, versionID: "v"}, "--recursive"}, + {"filters without recursion", "b/k", "b/j", + copyFlags{filter: keyFilter{include: []string{"*"}}}, "--recursive"}, + {"flatten without recursion", "b/k", "b/j", copyFlags{flatten: true}, "--flatten"}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := tc.flags + if _, _, err := parseCopyRefs(tc.src, tc.dst, &flags); err == nil || + !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } + + // The s3:// spelling is accepted on both sides. + src, dst, err := parseCopyRefs("s3://a/k", "s3://b/j", ©Flags{}) + if err != nil { + t.Fatal(err) + } + if src.Bucket != "a" || dst.Bucket != "b" { + t.Errorf("refs = %s, %s", src, dst) + } +} + +func TestCopyDestKey(t *testing.T) { + for _, tc := range []struct { + name string + dstPrefix, srcPrefix, key string + flatten bool + want string + }{ + {"mirrors below the prefix", "old/", "2026/", "2026/08/a.gz", false, "old/08/a.gz"}, + {"flattened", "old/", "2026/", "2026/08/a.gz", true, "old/a.gz"}, + {"no destination prefix", "", "2026/", "2026/a.gz", false, "a.gz"}, + {"destination without a slash", "old", "", "a.gz", false, "old/a.gz"}, + {"prefix is not a path boundary", "old/", "nightly-", "nightly-a.gz", false, "old/a.gz"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := copyDestKey(tc.dstPrefix, tc.srcPrefix, tc.key, tc.flatten); got != tc.want { + t.Errorf("copyDestKey = %q, want %q", got, tc.want) + } + }) + } +} + +// keyBase must split on "/" only: a backslash is a legal character in an S3 +// key, and filepath.Base would mangle it on Windows. +func TestKeyBase(t *testing.T) { + for _, tc := range []struct{ key, want string }{ + {"a/b/c.gz", "c.gz"}, + {"c.gz", "c.gz"}, + {`weird\name.gz`, `weird\name.gz`}, + } { + if got := keyBase(tc.key); got != tc.want { + t.Errorf("keyBase(%q) = %q, want %q", tc.key, got, tc.want) + } + } +} diff --git a/internal/cli/s3/download.go b/internal/cli/s3/download.go new file mode 100644 index 0000000..c97eb61 --- /dev/null +++ b/internal/cli/s3/download.go @@ -0,0 +1,339 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// downloadFlags holds the options accepted by "download". +type downloadFlags struct { + force bool + recursive bool + dryRun bool + flatten bool + versionID string + concurrency int + filter keyFilter +} + +// downloadRequest is one "download" invocation: the object or prefix to fetch, +// where to put it, and the flags that govern it. The positional arguments travel +// with the flags because runDownload needs all of them and a parameter list that +// long says nothing about which is which (go:S107). +type downloadRequest struct { + bucket string + // key is one object's key, or — under --recursive or with a wildcard — the + // prefix or glob selecting many. + key string + // dest is the destination path: "" means the key's basename, "-" means + // stream to stdout. Under --recursive it is a directory. + dest string + flags *downloadFlags +} + +const downloadLong = `Download an object, a prefix, or a wildcard match. + +With no FILE the object's key basename is used. FILE "-" streams the object to +stdout as raw bytes and prints nothing else, so it pipes. + +An existing file is never overwritten without --force: these are backups, and a +half-typed key that clobbers the local copy of one is not a recoverable +mistake. A transfer that fails partway removes the partial file rather than +leaving a truncated dump that looks complete. + +--recursive downloads every object under the key, treated as a prefix, into +FILE taken as a directory; a wildcard reference does the same without the flag: + + koc s3 download db-backups/2026/ ./restore --recursive + koc s3 download "db-backups/2026/*.sql.gz" ./restore + +Each object lands at its key's path relative to the prefix, so the layout is +preserved; --flatten puts every object straight in the directory instead, which +fails loudly rather than silently if two keys share a basename. In recursive +mode an existing file is *skipped* with a notice unless --force, so an +interrupted restore resumes by re-running it — unlike the single-object form, +where naming one key and doing nothing would hide the no-op. + +--concurrency downloads that many objects at once, which is most of the reason +a bulk restore finishes: a thousand small objects are a thousand round trips +serially. Progress lines then appear in completion order, not listing order. + +--include/--exclude narrow which keys are taken. --version-id fetches one +specific version of one object (Garage implements no versioning).` + +func newDownloadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + df := &downloadFlags{} + cmd := &cobra.Command{ + Use: "download / [file]", + Short: "Download an object, a prefix, or a wildcard match", + Long: downloadLong, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + req, err := newDownloadRequest(args, df) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runDownload(ctx, client, o, req, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVar(&df.force, "force", false, "overwrite the destination file if it exists") + fl.BoolVarP(&df.recursive, "recursive", "r", false, + "treat the key as a prefix and download everything under it into a directory") + fl.BoolVar(&df.dryRun, "dry-run", false, "print what would be downloaded and transfer nothing") + fl.BoolVar(&df.flatten, "flatten", false, + "put every object directly in the destination directory instead of mirroring its key path") + fl.StringVar(&df.versionID, "version-id", "", "download this version of the object instead of the current one") + fl.IntVar(&df.concurrency, "concurrency", s3.DefaultConcurrency, + "objects to download at once under --recursive") + df.filter.addTo(fl) + return cmd +} + +// newDownloadRequest parses the positional arguments and rejects the flag +// combinations that cannot mean anything. +func newDownloadRequest(args []string, f *downloadFlags) (downloadRequest, error) { + bucket, key, err := parseRef(args[0]) + if err != nil { + return downloadRequest{}, err + } + req := downloadRequest{bucket: bucket, key: key, flags: f} + if len(args) == 2 { + req.dest = args[1] + } + + bulk := f.recursive || hasGlob(key) + switch { + case bulk && req.dest == "-": + return req, errors.New("a recursive download needs a directory, not stdout") + case bulk && f.versionID != "": + return req, errors.New("--version-id names one object; it cannot be combined with --recursive") + case !bulk && key == "": + return req, fmt.Errorf("%q names no object: expected /", args[0]) + case !bulk && f.filter.active(): + return req, errors.New("--include/--exclude filter a listing; pass --recursive or a wildcard reference") + case !bulk && f.flatten: + return req, errors.New("--flatten only applies to a recursive download") + } + return req, f.filter.compile() +} + +// runDownload is the test seam for "download". w receives either the object's +// raw bytes (dest "-"), the per-object progress lines of a recursive run, or the +// summary table of a single one. +func runDownload(ctx context.Context, client *s3.Client, o *output.Options, + r downloadRequest, w io.Writer) error { + if r.flags.recursive || hasGlob(r.key) { + return runDownloadRecursive(ctx, client, r, w) + } + if r.dest == "-" { + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to stdout\n", r.bucket, r.key) + return err + } + if _, err := client.GetObject(ctx, r.bucket, r.key, r.flags.versionID, w); err != nil { + return downloadError(r.bucket, r.key, err) + } + return nil + } + + dest := r.dest + if dest == "" { + dest = filepath.Base(r.key) + } + if info, err := os.Stat(dest); err == nil && info.IsDir() { + dest = filepath.Join(dest, filepath.Base(r.key)) + } + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to %s\n", r.bucket, r.key, dest) + return err + } + + n, err := downloadToFile(ctx, client, r.bucket, r.key, r.flags.versionID, dest, r.flags.force) + if err != nil { + return err + } + return o.WriteSingle(w, + []string{"Bucket", "Key", "File", "Size"}, + []any{r.bucket, r.key, dest, n}) +} + +// runDownloadRecursive downloads every matching object under a prefix. +func runDownloadRecursive(ctx context.Context, client *s3.Client, r downloadRequest, w io.Writer) error { + dir := r.dest + if dir == "" { + dir = "." + } + prefix := globPrefix(r.key) + var glob *regexp.Regexp + if hasGlob(r.key) { + var err error + if glob, err = compileGlob(r.key); err != nil { + return fmt.Errorf("%s/%s: %w", r.bucket, r.key, err) + } + } + + out := &syncWriter{w: w} + p, workCtx := newPool(ctx, r.flags.concurrency) + taken := 0 + + err := client.ListObjectsFunc(ctx, r.bucket, s3.ListOptions{Prefix: prefix}, func(obj s3.Object) error { + if strings.HasSuffix(obj.Key, "/") { + // A "directory marker": a zero-length key ending in "/", which some + // tools create. Mirroring it as a file would shadow the directory. + return nil + } + if glob != nil && !glob.MatchString(obj.Key) { + return nil + } + if !r.flags.filter.match(obj.Key) { + return nil + } + dest, err := destForKey(dir, prefix, obj.Key, r.flags.flatten) + if err != nil { + return err + } + taken++ + if !p.run(workCtx, func() error { + return downloadOne(workCtx, client, r.bucket, obj.Key, dest, r.flags, out) + }) { + return errStopListing + } + return nil + }) + if waitErr := p.wait(); err == nil || isStopListing(err) { + err = waitErr + } + if err != nil && !isStopListing(err) { + return fmt.Errorf("downloading %s/%s: %w", r.bucket, r.key, err) + } + if taken == 0 { + _, err = fmt.Fprintf(w, "No objects under %s/%s\n", r.bucket, r.key) + return err + } + return nil +} + +// downloadOne fetches one object of a recursive run, or says what it would have +// fetched. +func downloadOne(ctx context.Context, client *s3.Client, bucket, key, dest string, + f *downloadFlags, w io.Writer) error { + if f.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to %s\n", bucket, key, dest) + return err + } + // In bulk mode an existing file is a resumption point, not a mistake: a + // restore that was interrupted is finished by re-running the same command. + if !f.force { + if _, err := os.Stat(dest); err == nil { + _, err := fmt.Fprintf(w, "Skipped (exists): %s\n", dest) + return err + } + } + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(dest), err) + } + + n, err := downloadToFile(ctx, client, bucket, key, "", dest, f.force) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "Downloaded: %s/%s -> %s (%d bytes)\n", bucket, key, dest, n) + return err +} + +// destForKey maps one key to its local path, and refuses a key that would +// escape the destination directory. +// +// A key is server-supplied data: "../../etc/cron.d/x", or an absolute one, is a +// perfectly legal S3 key, and a restore that wrote it where it says would let +// whoever can write to the bucket write anywhere koc can. +func destForKey(dir, prefix, key string, flatten bool) (string, error) { + rel := key + if flatten { + rel = filepath.Base(key) + } else if prefix != "" { + rel = strings.TrimPrefix(key, prefix) + // A prefix that is not a path boundary ("e2e-") leaves a bare file + // name, which is what should land in the directory. + rel = strings.TrimPrefix(rel, "/") + } + if rel == "" { + rel = filepath.Base(key) + } + + dest := filepath.Join(dir, filepath.FromSlash(rel)) + clean := filepath.Clean(dir) + if rel := mustRel(clean, dest); rel == "" { + return "", fmt.Errorf("object key %q would be written outside %s; refusing", key, clean) + } + return dest, nil +} + +// mustRel returns the path of target under base, or "" if it is not under it. +func mustRel(base, target string) string { + rel, err := filepath.Rel(base, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "" + } + return rel +} + +// downloadToFile streams the object into dest, leaving nothing behind if it +// fails. +func downloadToFile(ctx context.Context, client *s3.Client, bucket, key, versionID, dest string, + force bool) (n int64, err error) { + flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL + if force { + flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC + } + out, err := os.OpenFile(dest, flags, 0o600) //nolint:gosec // G304: operator-supplied output path + if err != nil { + if errors.Is(err, os.ErrExist) { + return 0, fmt.Errorf("%s already exists (pass --force to overwrite)", dest) + } + return 0, fmt.Errorf("creating %q: %w", dest, err) + } + defer func() { + closeErr := out.Close() + if err == nil && closeErr != nil { + err = fmt.Errorf("closing %q: %w", dest, closeErr) + } + if err != nil { + _ = os.Remove(dest) + } + }() + + if n, err = client.GetObject(ctx, bucket, key, versionID, out); err != nil { + return 0, downloadError(bucket, key, err) + } + return n, nil +} + +// downloadError turns a missing key into the message an operator can act on. +func downloadError(bucket, key string, err error) error { + if s3.IsNotFound(err) { + return fmt.Errorf("no object %q in bucket %q", key, bucket) + } + return fmt.Errorf("downloading %s/%s: %w", bucket, key, err) +} diff --git a/internal/cli/s3/du.go b/internal/cli/s3/du.go new file mode 100644 index 0000000..b078006 --- /dev/null +++ b/internal/cli/s3/du.go @@ -0,0 +1,162 @@ +package s3cli + +import ( + "context" + "fmt" + "io" + "sort" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// duFlags holds the options accepted by "du". +type duFlags struct { + group bool + human bool + allVersions bool + filter keyFilter +} + +const duLong = `Total the size of a bucket, or of one prefix in it. + +S3 has no "how big is this" call, so this is a full listing folded into a sum: +the objects are never downloaded, but every key is walked, which costs one +request per 1000 of them. The listing streams, so the size of the bucket does +not become the size of koc's memory. + +Sizes are exact bytes, like the rest of koc, so they stay usable in +--format value/csv and in arithmetic; --human renders them for a reader. + +--group reports one row per storage class instead of one total, which is what +tells a mixed bucket's cheap tier from its expensive one. Objects a store +reports with no storage class are grouped as STANDARD, which is what it means — +Garage reports every object that way, since it implements no storage classes. + +--all-versions counts every version and delete marker, which is the only way to +see what an old version still costs on a store that has versioning.` + +const duExample = ` # A whole bucket + koc s3 du db-backups + + # Just the end-to-end test dumps, for a reader + koc s3 du db-backups/e2e- --human + + # Per storage class + koc s3 du db-backups --group + + # Only the compressed dumps + koc s3 du db-backups --include "*.sql.gz"` + +func newDuCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + df := &duFlags{} + cmd := &cobra.Command{ + Use: "du [/]", + Short: "Total the size of a bucket or prefix", + Long: duLong, + Example: duExample, + Aliases: []string{"usage"}, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, prefix, err := parseRef(args[0]) + if err != nil { + return err + } + if err := df.filter.compile(); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runDu(ctx, client, o, bucket, prefix, df, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVar(&df.group, "group", false, "one row per storage class instead of one total") + fl.BoolVar(&df.human, "human", false, "render sizes for a reader (KiB/MiB/GiB) instead of exact bytes") + fl.BoolVar(&df.allVersions, "all-versions", false, "count every version and delete marker") + df.filter.addTo(fl) + return cmd +} + +// usage is the running total for one storage class, or for the whole listing. +type usage struct { + objects int64 + bytes int64 +} + +// runDu is the test seam for "du". +func runDu(ctx context.Context, client *s3.Client, o *output.Options, + bucket, prefix string, f *duFlags, w io.Writer) error { + var total usage + byClass := map[string]*usage{} + + opts := s3.ListOptions{Prefix: prefix, Versions: f.allVersions} + err := client.ListObjectsFunc(ctx, bucket, opts, func(obj s3.Object) error { + if !f.filter.match(obj.Key) { + return nil + } + total.objects++ + total.bytes += obj.Size + if f.group { + class := obj.StorageClass + if class == "" { + class = "STANDARD" + } + u, ok := byClass[class] + if !ok { + u = &usage{} + byClass[class] = u + } + u.objects++ + u.bytes += obj.Size + } + return nil + }) + if err != nil { + return fmt.Errorf("totalling %s/%s: %w", bucket, prefix, err) + } + + if f.group { + return writeUsageByClass(w, o, byClass, f.human) + } + return o.WriteSingle(w, + []string{"Bucket", "Prefix", "Objects", "Size"}, + []any{bucket, prefix, total.objects, renderSize(total.bytes, f.human)}) +} + +// writeUsageByClass renders --group's per-storage-class rows, ordered by class +// name so two runs of the same bucket diff cleanly. +func writeUsageByClass(w io.Writer, o *output.Options, byClass map[string]*usage, human bool) error { + classes := make([]string, 0, len(byClass)) + for class := range byClass { + classes = append(classes, class) + } + sort.Strings(classes) + + rows := make([][]any, len(classes)) + for i, class := range classes { + rows[i] = []any{class, byClass[class].objects, renderSize(byClass[class].bytes, human)} + } + return o.WriteList(w, output.Table{ + Columns: []string{"Storage Class", "Objects", "Size"}, + Rows: rows, + }) +} + +// renderSize is the one place --human is applied, so every size column in this +// group reads the same way. +func renderSize(n int64, human bool) any { + if human { + return output.HumanBytes(n) + } + return n +} diff --git a/internal/cli/s3/du_test.go b/internal/cli/s3/du_test.go new file mode 100644 index 0000000..cc575f1 --- /dev/null +++ b/internal/cli/s3/du_test.go @@ -0,0 +1,73 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/ftarasenko/go-openstackclient/internal/output" +) + +func TestRunDu(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + err := runDu(context.Background(), client, valueOpts(), "db-backups", "", &duFlags{}, &buf) + if err != nil { + t.Fatal(err) + } + // Bucket, prefix, objects, exact bytes — 100 + 20 + 3, one field per line. + if got, want := buf.String(), "db-backups\n\n3\n123\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestRunDuGroupByStorageClass(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + err := runDu(context.Background(), client, valueOpts(), "db-backups", "", &duFlags{group: true}, &buf) + if err != nil { + t.Fatal(err) + } + // Classes sort by name, and an object with no class counts as STANDARD. + want := "GLACIER\t1\t20\nSTANDARD\t2\t103\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// duListHandler serves one page of three objects across two storage classes, +// one of which the store reports with no class at all. +func duListHandler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("du issued %s, want GET only", r.Method) + } + _, _ = fmt.Fprint(w, `false + a100STANDARD + b20GLACIER + c3 + `) + } +} + +// du must be able to render as a table too, not only in --format value. +func TestRunDuTable(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + o := &output.Options{Format: output.FormatJSON} + if err := runDu(context.Background(), client, o, "db-backups", "e2e-", &duFlags{}, &buf); err != nil { + t.Fatal(err) + } + for _, want := range []string{`"Objects": 3`, `"Size": 123`, `"Prefix": "e2e-"`} { + if !strings.Contains(buf.String(), want) { + t.Errorf("JSON output %s missing %q", buf.String(), want) + } + } +} diff --git a/internal/cli/s3/filter.go b/internal/cli/s3/filter.go new file mode 100644 index 0000000..641b17b --- /dev/null +++ b/internal/cli/s3/filter.go @@ -0,0 +1,124 @@ +package s3cli + +import ( + "fmt" + "regexp" + "strings" + + "github.com/spf13/pflag" +) + +// keyFilter is the --include/--exclude pair shared by every command that walks +// a listing. Both take glob patterns and both may be repeated: +// +// - with no --include, every key is a candidate; +// - with any --include, a key must match at least one of them; +// - --exclude then removes a key whatever --include said, because "all the +// dumps except the end-to-end ones" is the shape an operator actually +// needs. +type keyFilter struct { + include []string + exclude []string + + compiled struct { + include []*regexp.Regexp + exclude []*regexp.Regexp + } +} + +// addTo registers the flags. Every command that filters a listing uses the same +// two names and the same help, so an operator learns them once. +func (f *keyFilter) addTo(fs *pflag.FlagSet) { + fs.StringArrayVar(&f.include, "include", nil, + `only keys matching this glob, repeatable (e.g. --include "*.sql.gz")`) + fs.StringArrayVar(&f.exclude, "exclude", nil, + `skip keys matching this glob, repeatable; applied after --include`) +} + +// compile turns the patterns into matchers, rejecting a malformed one before any +// request is made rather than silently matching nothing. +func (f *keyFilter) compile() error { + var err error + if f.compiled.include, err = compileGlobs(f.include, "--include"); err != nil { + return err + } + f.compiled.exclude, err = compileGlobs(f.exclude, "--exclude") + return err +} + +// active reports whether any pattern was given, so a caller can skip the whole +// filtering path (and its per-key cost) when none was. +func (f *keyFilter) active() bool { return len(f.include) > 0 || len(f.exclude) > 0 } + +// match reports whether key survives the filter. +func (f *keyFilter) match(key string) bool { + if len(f.compiled.include) > 0 && !matchesAny(f.compiled.include, key) { + return false + } + return !matchesAny(f.compiled.exclude, key) +} + +func matchesAny(pats []*regexp.Regexp, key string) bool { + for _, p := range pats { + if p.MatchString(key) { + return true + } + } + return false +} + +func compileGlobs(pats []string, flag string) ([]*regexp.Regexp, error) { + out := make([]*regexp.Regexp, 0, len(pats)) + for _, p := range pats { + re, err := compileGlob(p) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", flag, p, err) + } + out = append(out, re) + } + return out, nil +} + +// compileGlob translates a glob to an anchored regexp. +// +// path.Match is not used: its "*" stops at a "/", which is right for a +// filesystem and wrong for an S3 keyspace, where the slashes in +// "2026/08/dump.sql.gz" are ordinary characters and --include "*.sql.gz" is +// expected to reach them. This matches s5cmd's wildcards, where "*" spans +// separators and "?" is one character. +// +// Only those two are special. Everything else — a dot, a bracket, a brace — is +// quoted to a literal, so a key containing one names itself rather than turning +// into a character class the operator did not write. +func compileGlob(pattern string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for _, r := range pattern { + switch r { + case '*': + b.WriteString(".*") + case '?': + b.WriteString(".") + default: + b.WriteString(regexp.QuoteMeta(string(r))) + } + } + b.WriteString("$") + return regexp.Compile(b.String()) +} + +// hasGlob reports whether ref carries wildcard characters, i.e. whether it +// names a set of keys rather than one. +func hasGlob(ref string) bool { + return strings.ContainsAny(ref, "*?") +} + +// globPrefix is the literal part of a glob before its first wildcard, which is +// what the server can filter on: "2026/08/*.gz" lists under "2026/08/" and the +// pattern then narrows the result locally. +func globPrefix(pattern string) string { + if i := strings.IndexAny(pattern, "*?"); i >= 0 { + return pattern[:i] + } + return pattern +} diff --git a/internal/cli/s3/filter_test.go b/internal/cli/s3/filter_test.go new file mode 100644 index 0000000..84fb9a2 --- /dev/null +++ b/internal/cli/s3/filter_test.go @@ -0,0 +1,103 @@ +package s3cli + +import "testing" + +// An S3 keyspace is flat: the slashes in "2026/08/dump.sql.gz" are ordinary +// characters, so "*" has to cross them. path.Match would not, which is why this +// translates to a regexp instead. +func TestCompileGlobSpansSlashes(t *testing.T) { + for _, tc := range []struct { + pattern, key string + want bool + }{ + {"*.sql.gz", "dump.sql.gz", true}, + {"*.sql.gz", "2026/08/dump.sql.gz", true}, + {"2026/*", "2026/08/dump.sql.gz", true}, + {"2026/*", "2025/08/dump.sql.gz", false}, + {"dump-?.gz", "dump-1.gz", true}, + {"dump-?.gz", "dump-12.gz", false}, + {"dump.sql.gz", "dump.sql.gz", true}, + {"dump.sql.gz", "xdump.sql.gz", false}, + // A dot is a literal, not "any character" — a glob is not a regexp. + {"a.c", "abc", false}, + {"a.c", "a.c", true}, + } { + re, err := compileGlob(tc.pattern) + if err != nil { + t.Fatalf("compileGlob(%q) = %v", tc.pattern, err) + } + if got := re.MatchString(tc.key); got != tc.want { + t.Errorf("%q matches %q = %v, want %v", tc.pattern, tc.key, got, tc.want) + } + } +} + +func TestKeyFilter(t *testing.T) { + for _, tc := range []struct { + name string + include, exclude []string + key string + want bool + }{ + {"no patterns takes everything", nil, nil, "anything", true}, + {"include must match", []string{"*.gz"}, nil, "dump.gz", true}, + {"include that misses drops the key", []string{"*.gz"}, nil, "dump.txt", false}, + {"any include is enough", []string{"*.gz", "*.txt"}, nil, "dump.txt", true}, + {"exclude wins over include", []string{"*"}, []string{"e2e-*"}, "e2e-a.gz", false}, + {"exclude alone", nil, []string{"e2e-*"}, "nightly.gz", true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := keyFilter{include: tc.include, exclude: tc.exclude} + if err := f.compile(); err != nil { + t.Fatal(err) + } + if got := f.match(tc.key); got != tc.want { + t.Errorf("match(%q) = %v, want %v", tc.key, got, tc.want) + } + }) + } +} + +// Only "*" and "?" are special, matching s5cmd: a bracket, a brace or a dot in +// a key is a literal, so "[2026]/dump" names exactly that key rather than a +// character class over one of four digits. +func TestCompileGlobTreatsEverythingElseAsLiteral(t *testing.T) { + re, err := compileGlob("[2026]/dump.gz") + if err != nil { + t.Fatal(err) + } + if !re.MatchString("[2026]/dump.gz") { + t.Error("a bracketed key did not match itself") + } + if re.MatchString("2/dump.gz") { + t.Error("brackets were treated as a character class") + } +} + +func TestGlobPrefix(t *testing.T) { + for _, tc := range []struct{ pattern, want string }{ + {"2026/08/*.gz", "2026/08/"}, + {"*.gz", ""}, + {"dump.sql.gz", "dump.sql.gz"}, + {"dump-?.gz", "dump-"}, + } { + if got := globPrefix(tc.pattern); got != tc.want { + t.Errorf("globPrefix(%q) = %q, want %q", tc.pattern, got, tc.want) + } + } +} + +func TestHasGlob(t *testing.T) { + for _, tc := range []struct { + ref string + want bool + }{ + {"b/k", false}, + {"b/*.gz", true}, + {"b/dump-?.gz", true}, + } { + if got := hasGlob(tc.ref); got != tc.want { + t.Errorf("hasGlob(%q) = %v, want %v", tc.ref, got, tc.want) + } + } +} diff --git a/internal/cli/s3/flags_test.go b/internal/cli/s3/flags_test.go new file mode 100644 index 0000000..bd2454d --- /dev/null +++ b/internal/cli/s3/flags_test.go @@ -0,0 +1,311 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/ftarasenko/go-openstackclient/internal/output" +) + +// --delimiter turns a flat keyspace into one directory level: the subtrees come +// first, with no size, so a table never claims an empty object is there. +func TestRunObjectListDelimiter(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("delimiter"); got != "/" { + t.Errorf("delimiter = %q, want /", got) + } + _, _ = fmt.Fprint(w, `false + 2026/ + latest.sql.gz7 + 2026-09-14T10:00:00.000Z"e" + `) + }) + + var buf bytes.Buffer + err := runObjectList(context.Background(), client, valueOpts(), "db-backups", + &objectListFlags{delimiter: "/"}, &buf) + if err != nil { + t.Fatal(err) + } + want := "2026/\t\t\t\nlatest.sql.gz\t7\t2026-09-14T10:00:00Z\te\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestRunObjectListHuman(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + big.sql.gz15252880 + `) + }) + + var buf bytes.Buffer + err := runObjectList(context.Background(), client, valueOpts(), "db-backups", + &objectListFlags{human: true}, &buf) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "14.5 MiB") { + t.Errorf("output = %q, want a human-readable size", buf.String()) + } +} + +// A local filter has to be applied before --limit is counted, or a bucket whose +// first keys are all excluded would answer an empty list under "--limit 1". +func TestRunObjectListFilterThenLimit(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + // With filtering on, the cap moves to koc, so the server must not be + // asked for a short page. + if got := r.URL.Query().Get("max-keys"); got != "1000" { + t.Errorf("max-keys = %q, want a full page when filtering locally", got) + } + _, _ = fmt.Fprint(w, `false + e2e-a.gz1 + nightly-a.gz2 + nightly-b.gz3 + `) + }) + + f := &objectListFlags{limit: 1, filter: keyFilter{include: []string{"nightly-*"}}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runObjectList(context.Background(), client, valueOpts(), "db-backups", f, &buf); err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "nightly-a.gz\t2\t\t\n"; got != want { + t.Errorf("output = %q, want the first matching key only", got) + } +} + +// --all-versions adds two columns, so the extra fields are not silently dropped +// in --format value. +func TestRunObjectListAllVersions(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if !r.URL.Query().Has("versions") { + t.Error("the request did not carry ?versions") + } + _, _ = fmt.Fprint(w, `false + dump.gzv2true + 92026-09-14T10:00:00.000Z + gone.gzv1 + 2026-09-14T09:00:00.000Z + `) + }) + + var buf bytes.Buffer + err := runObjectList(context.Background(), client, valueOpts(), "b", + &objectListFlags{allVersions: true}, &buf) + if err != nil { + t.Fatal(err) + } + out := buf.String() + if !strings.Contains(out, "v2") || !strings.Contains(out, "true") { + t.Errorf("output %q does not carry the version columns", out) + } + if !strings.Contains(out, "v1 (delete marker)") { + t.Errorf("output %q does not mark the delete marker", out) + } +} + +func TestRunDuHuman(t *testing.T) { + client := newMockClient(t, duListHandler(t)) + + var buf bytes.Buffer + err := runDu(context.Background(), client, valueOpts(), "db-backups", "", &duFlags{human: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "db-backups\n\n3\n123 B\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +func TestRunDuFilter(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + a.sql.gz100 + a.sha2567 + `) + }) + + f := &duFlags{filter: keyFilter{include: []string{"*.sql.gz"}}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runDu(context.Background(), client, valueOpts(), "db-backups", "", f, &buf); err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "db-backups\n\n1\n100\n"; got != want { + t.Errorf("output = %q, want only the filtered object counted", got) + } +} + +func TestRunBucketShow(t *testing.T) { + var methods []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + if r.URL.Query().Has("versioning") { + _, _ = fmt.Fprint(w, `Enabled`) + return + } + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketShow(context.Background(), client, valueOpts(), "db-backups", &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(methods, ","), "HEAD,GET"; got != want { + t.Errorf("requests = %q, want a HEAD then the versioning read", got) + } + out := buf.String() + if !strings.Contains(out, "db-backups") || !strings.Contains(out, "Enabled") { + t.Errorf("output = %q", out) + } +} + +// The bucket being there is the answer the command was asked for; a store that +// does not implement the versioning call must not turn that into a failure. +func TestRunBucketShowToleratesNoVersioningSupport(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("versioning") { + w.WriteHeader(http.StatusNotImplemented) + return + } + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketShow(context.Background(), client, valueOpts(), "db-backups", &buf); err != nil { + t.Fatalf("a store without versioning failed the command: %v", err) + } + if !strings.Contains(buf.String(), "unknown") { + t.Errorf("output = %q, want the versioning field unknown", buf.String()) + } +} + +func TestRunBucketShowMissingBucket(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + var buf bytes.Buffer + err := runBucketShow(context.Background(), client, valueOpts(), "absent", &buf) + if err == nil { + t.Fatal("a missing bucket was reported as present") + } + if !strings.Contains(err.Error(), "absent") { + t.Errorf("error %q does not name the bucket", err) + } +} + +func TestRunBucketSet(t *testing.T) { + var gotMethod string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + if err := runBucketSet(context.Background(), client, valueOpts(), "b", "Enabled", &buf); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut { + t.Errorf("method = %q, want PUT", gotMethod) + } + if got, want := buf.String(), "b\nEnabled\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// The flag takes either spelling, so "--versioning Enabled" is not a puzzling +// error — and anything else is refused by name. +func TestVersioningStatus(t *testing.T) { + for _, in := range []string{"enabled", "Enabled", "ENABLED"} { + if got, err := versioningStatus(in); err != nil || got != "Enabled" { + t.Errorf("versioningStatus(%q) = %q, %v", in, got, err) + } + } + if got, err := versioningStatus("suspended"); err != nil || got != "Suspended" { + t.Errorf("versioningStatus(suspended) = %q, %v", got, err) + } + if _, err := versioningStatus("unversioned"); err == nil { + t.Error("unversioned was accepted as a settable state") + } +} + +// Presigning makes no request at all, which is why runPresign takes no context. +func TestRunPresign(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("presigning sent %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + err := runPresign(client, valueOpts(), "db-backups", "dump.sql.gz", "", "get", time.Hour, &buf) + if err != nil { + t.Fatal(err) + } + out := buf.String() + for _, want := range []string{"X-Amz-Signature=", "X-Amz-Expires=3600", "/db-backups/dump.sql.gz"} { + if !strings.Contains(out, want) { + t.Errorf("output %q missing %q", out, want) + } + } +} + +func TestRunPresignPut(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + var buf bytes.Buffer + if err := runPresign(client, valueOpts(), "b", "k", "", "put", time.Minute, &buf); err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "put") { + t.Errorf("output %q does not name the method", buf.String()) + } +} + +func TestRunPresignRejectsBadInput(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + var buf bytes.Buffer + if err := runPresign(client, valueOpts(), "b", "k", "", "delete", time.Hour, &buf); err == nil || + !strings.Contains(err.Error(), "get or put") { + t.Errorf("err = %v, want it to name the allowed methods", err) + } + // A version ID addresses an object that exists; presigning an upload to one + // is a contradiction. + if err := runPresign(client, valueOpts(), "b", "k", "v1", "put", time.Hour, &buf); err == nil { + t.Error("--version-id with --method put was accepted") + } +} + +func TestHumanBytes(t *testing.T) { + for _, tc := range []struct { + n int64 + want string + }{ + {0, "0 B"}, + {1023, "1023 B"}, + {1024, "1.00 KiB"}, + {1536, "1.50 KiB"}, + {15252880, "14.5 MiB"}, + {1 << 30, "1.00 GiB"}, + {200 << 30, "200 GiB"}, + {-2048, "-2.00 KiB"}, + } { + if got := output.HumanBytes(tc.n); got != tc.want { + t.Errorf("HumanBytes(%d) = %q, want %q", tc.n, got, tc.want) + } + } +} diff --git a/internal/cli/s3/object.go b/internal/cli/s3/object.go index a1be07c..472d6f9 100644 --- a/internal/cli/s3/object.go +++ b/internal/cli/s3/object.go @@ -22,28 +22,61 @@ func newObjectCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.C } cmd.AddCommand(newObjectListCommand(a, o, f)) cmd.AddCommand(newObjectShowCommand(a, o, f)) + cmd.AddCommand(newObjectDeleteCommand(a, o, f)) return cmd } // objectListFlags holds the options accepted by "object list". type objectListFlags struct { - prefix string - limit int + prefix string + delimiter string + limit int + human bool + allVersions bool + filter keyFilter } const objectListLong = `List the objects in a bucket. -Sizes are exact bytes so they stay usable in scripts and in --format value/csv. ---limit is a hard cap on the result, not a page size: listing pages until the -cap is reached and stops there.` +Sizes are exact bytes so they stay usable in scripts and in --format value/csv; +--human renders them for a reader instead. --limit is a hard cap on the result, +not a page size: listing pages until the cap is reached and stops there. + +A bucket is a flat keyspace, so a deep one lists every key at once. +--delimiter / collapses everything below each "directory" into a single entry +marked with a trailing separator, which is how to walk a big bucket one level +at a time: + + koc s3 object list db-backups --delimiter / + koc s3 object list db-backups/2026/ --delimiter / + +--include and --exclude filter the keys locally after the server has listed +them, so they cost no extra requests but do not reduce the listing itself. + +--all-versions lists every version and delete marker rather than the current +object. Garage does not implement versioning — it reports every bucket as +unversioned — so this is for a koc pointed at AWS, Ceph RGW or MinIO.` + +const objectListExample = ` # Everything, exact bytes + koc s3 object list db-backups + + # One level, like a directory listing + koc s3 object list db-backups --delimiter / + + # Just the compressed dumps, sizes for a human + koc s3 object list db-backups --include "*.sql.gz" --human + + # Everything but the end-to-end test dumps + koc s3 object list db-backups --exclude "e2e-*"` func newObjectListCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { lf := &objectListFlags{} cmd := &cobra.Command{ - Use: "list ", - Short: "List objects in a bucket", - Long: objectListLong, - Args: cobra.ExactArgs(1), + Use: "list [/]", + Short: "List objects in a bucket", + Long: objectListLong, + Example: objectListExample, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if err := o.Validate(); err != nil { return err @@ -57,6 +90,9 @@ func newObjectListCommand(a *auth.Options, o *output.Options, f *connFlags) *cob if lf.prefix == "" { lf.prefix = prefix } + if err := lf.filter.compile(); err != nil { + return err + } ctx := cmd.Context() client, err := f.client(ctx, a) if err != nil { @@ -67,36 +103,97 @@ func newObjectListCommand(a *auth.Options, o *output.Options, f *connFlags) *cob } fl := cmd.Flags() fl.StringVar(&lf.prefix, "prefix", "", "only list keys starting with this prefix") + fl.StringVar(&lf.delimiter, "delimiter", "", + `collapse keys below each occurrence of this separator into one entry (usually "/")`) fl.IntVar(&lf.limit, "limit", 0, "maximum number of objects to return (0 = no limit)") + fl.BoolVar(&lf.human, "human", false, "render sizes for a reader (KiB/MiB/GiB) instead of exact bytes") + fl.BoolVar(&lf.allVersions, "all-versions", false, + "list every version and delete marker, not just the current object") + lf.filter.addTo(fl) return cmd } // runObjectList is the test seam for "object list". func runObjectList(ctx context.Context, client *s3.Client, o *output.Options, bucket string, f *objectListFlags, w io.Writer) error { - objects, err := client.ListObjects(ctx, bucket, f.prefix, f.limit) - if err != nil { + opts := s3.ListOptions{ + Prefix: f.prefix, + Delimiter: f.delimiter, + Limit: f.limit, + Versions: f.allVersions, + } + // A local filter has to be applied before --limit is counted, or a bucket + // whose first 100 keys are all excluded would answer an empty list under + // "--limit 10". So the cap moves here whenever filtering is on. + if f.filter.active() { + opts.Limit = 0 + } + + var rows [][]any + err := client.ListObjectsFunc(ctx, bucket, opts, func(obj s3.Object) error { + // A CommonPrefixes entry has no size or timestamp; it is a subtree. + if !obj.IsPrefix && !f.filter.match(obj.Key) { + return nil + } + rows = append(rows, objectRow(obj, f)) + if f.filter.active() && f.limit > 0 && len(rows) >= f.limit { + return errStopListing + } + return nil + }) + if err != nil && !isStopListing(err) { return fmt.Errorf("listing objects in %s: %w", bucket, err) } - rows := make([][]any, len(objects)) - for i, obj := range objects { - rows[i] = []any{obj.Key, obj.Size, formatTime(obj.LastModified), obj.ETag} + return o.WriteList(w, output.Table{Columns: objectListColumns(f), Rows: rows}) +} + +// objectListColumns is the header row, which grows by two under --all-versions +// so the extra fields are not silently dropped in --format value. +func objectListColumns(f *objectListFlags) []string { + cols := []string{"Key", "Size", "Last Modified", "ETag"} + if f.allVersions { + cols = append(cols, "Version ID", "Latest") + } + return cols +} + +// objectRow renders one listing entry. +func objectRow(obj s3.Object, f *objectListFlags) []any { + key, size := obj.Key, any(obj.Size) + switch { + case obj.IsPrefix: + // A subtree has no size of its own; saying "0" would read as an empty + // object that is really there. + size = "" + case f.human: + size = output.HumanBytes(obj.Size) } - return o.WriteList(w, output.Table{ - Columns: []string{"Key", "Size", "Last Modified", "ETag"}, - Rows: rows, - }) + + row := []any{key, size, formatTime(obj.LastModified), obj.ETag} + if f.allVersions { + version := obj.VersionID + if obj.DeleteMarker { + version += " (delete marker)" + } + row = append(row, version, obj.IsLatest) + } + return row } const objectShowLong = `Show one object's metadata, without downloading it. This is the equivalent of the "s3cmd info" call the backup pipeline makes to verify an upload: it is a HEAD request, so the object's body is never -transferred.` +transferred. + +--version-id addresses one specific version on a store that has versioning +(Garage does not).` func newObjectShowCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { - return &cobra.Command{ + var versionID string + var human bool + cmd := &cobra.Command{ Use: "show /", Short: "Show an object's metadata", Long: objectShowLong, @@ -114,15 +211,18 @@ func newObjectShowCommand(a *auth.Options, o *output.Options, f *connFlags) *cob if err != nil { return err } - return runObjectShow(ctx, client, o, bucket, key, cmd.OutOrStdout()) + return runObjectShow(ctx, client, o, bucket, key, versionID, human, cmd.OutOrStdout()) }, } + cmd.Flags().StringVar(&versionID, "version-id", "", "address this version of the object instead of the current one") + cmd.Flags().BoolVar(&human, "human", false, "render the size for a reader instead of exact bytes") + return cmd } // runObjectShow is the test seam for "object show". func runObjectShow(ctx context.Context, client *s3.Client, o *output.Options, - bucket, key string, w io.Writer) error { - info, err := client.HeadObject(ctx, bucket, key) + bucket, key, versionID string, human bool, w io.Writer) error { + info, err := client.HeadObject(ctx, bucket, key, versionID) if err != nil { if s3.IsNotFound(err) { return fmt.Errorf("no object %q in bucket %q", key, bucket) @@ -130,8 +230,16 @@ func runObjectShow(ctx context.Context, client *s3.Client, o *output.Options, return fmt.Errorf("reading %s/%s: %w", bucket, key, err) } + size := any(info.Size) + if human { + size = output.HumanBytes(info.Size) + } fields := []string{"Bucket", "Key", "Size", "Last Modified", "ETag", "Content Type"} - values := []any{info.Bucket, info.Key, info.Size, formatTime(info.LastModified), info.ETag, info.ContentType} + values := []any{info.Bucket, info.Key, size, formatTime(info.LastModified), info.ETag, info.ContentType} + if info.VersionID != "" { + fields = append(fields, "Version ID") + values = append(values, info.VersionID) + } metaKeys := make([]string, 0, len(info.Metadata)) for k := range info.Metadata { diff --git a/internal/cli/s3/object_delete.go b/internal/cli/s3/object_delete.go new file mode 100644 index 0000000..6822d1e --- /dev/null +++ b/internal/cli/s3/object_delete.go @@ -0,0 +1,268 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "regexp" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/cli/batchdelete" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// objectDeleteFlags holds the options accepted by "object delete". +type objectDeleteFlags struct { + recursive bool + dryRun bool + versionID string + allVersions bool + filter keyFilter +} + +const objectDeleteLong = `Delete objects. + +Each argument is one object: "/", or "s3:///". Every +argument is attempted even if an earlier one fails, and the command exits +non-zero naming each failure. + +S3 answers a delete of a key that was never there with success, so deleting is +idempotent: a key that is already gone is reported deleted rather than as an +error. That is the protocol's behaviour, not a choice koc makes — do not use the +exit status to test whether a key existed, use "koc s3 object show". + +--recursive turns each argument's key into a prefix and deletes every object +under it, which is also how a bucket is emptied before "koc s3 bucket delete": + + koc s3 object delete db-backups/e2e- --recursive # one prefix + koc s3 object delete db-backups/ --recursive # the whole bucket + +That is the one destructive shape in this group, so it is spelled out rather +than inferred from a trailing slash, and --dry-run prints exactly the keys it +would delete without touching any of them. A recursive delete also accepts a +wildcard instead of --recursive: + + koc s3 object delete "db-backups/2026/*.sql.gz" + +A recursive delete goes out in batches of up to 1000 keys per request rather +than one request per key, so emptying a large bucket costs a thousandth of the +round trips. --include and --exclude narrow which keys it takes. + +--version-id deletes one specific version; --all-versions removes every version +and delete marker under the prefix. Garage implements no versioning, so both are +for a koc pointed at AWS, Ceph RGW or MinIO.` + +func newObjectDeleteCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + df := &objectDeleteFlags{} + cmd := &cobra.Command{ + Use: "delete / [/ ...]", + Short: "Delete objects", + Long: objectDeleteLong, + Aliases: []string{"remove"}, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + if err := df.validate(args); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runObjectDelete(ctx, client, args, df, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVarP(&df.recursive, "recursive", "r", false, + "treat each key as a prefix and delete every object under it") + fl.BoolVar(&df.dryRun, "dry-run", false, + "print the objects that would be deleted and delete nothing") + fl.StringVar(&df.versionID, "version-id", "", + "delete this version of the object instead of the current one") + fl.BoolVar(&df.allVersions, "all-versions", false, + "delete every version and delete marker, not just the current object") + df.filter.addTo(fl) + return cmd +} + +// validate rejects the flag combinations that cannot mean anything, and +// compiles the globs so a bad pattern fails before a single key is removed. +func (f *objectDeleteFlags) validate(refs []string) error { + if f.versionID != "" { + switch { + case f.recursive: + return errors.New("--version-id names one object; it cannot be combined with --recursive") + case f.allVersions: + return errors.New("--version-id and --all-versions are mutually exclusive") + case len(refs) > 1: + return errors.New("--version-id names one object; pass a single reference") + } + } + if !f.recursive && !f.allVersions && f.filter.active() { + for _, ref := range refs { + if hasGlob(ref) { + return nil + } + } + return errors.New("--include/--exclude filter a listing; pass --recursive or a wildcard reference") + } + return f.filter.compile() +} + +// runObjectDelete is the test seam for "object delete". +func runObjectDelete(ctx context.Context, client *s3.Client, refs []string, + f *objectDeleteFlags, w io.Writer) error { + return batchdelete.Each(refs, func(ref string) error { + // A wildcard reference is a recursive delete that names its own + // pattern, so "delete 'b/2026/*.gz'" needs no second flag to say so. + if f.recursive || f.allVersions || hasGlob(ref) { + // A bare bucket is a legal prefix ref here — "/" is how the + // whole bucket is named — so parseRef, not parseObjectRef. + bucket, pattern, err := parseRef(ref) + if err != nil { + return err + } + return deletePrefix(ctx, client, bucket, pattern, f, w) + } + bucket, key, err := parseObjectRef(ref) + if err != nil { + return err + } + if f.dryRun { + return sayWouldDelete(w, bucket, key, f.versionID) + } + if err := client.DeleteObject(ctx, bucket, key, f.versionID); err != nil { + return fmt.Errorf("deleting %s: %w", describeTarget(bucket, key, f.versionID), err) + } + return sayDeleted(w, bucket, key, f.versionID) + }) +} + +// deletePrefix deletes every object under a prefix (or matching a wildcard), +// batching the keys so a large bucket does not cost one round trip per object. +// +// Keys are collected a batch at a time as the listing streams rather than all +// at once, so emptying a bucket of a million objects costs one batch of memory +// rather than all of them. +func deletePrefix(ctx context.Context, client *s3.Client, bucket, pattern string, + f *objectDeleteFlags, w io.Writer) error { + var ( + pending []s3.DeleteTarget + seen int + ) + // The literal head of a wildcard is what the server can filter on; the rest + // of the pattern narrows the result here, against a matcher compiled once + // rather than per key. + prefix := globPrefix(pattern) + var glob *regexp.Regexp + if hasGlob(pattern) { + var err error + if glob, err = compileGlob(pattern); err != nil { + return fmt.Errorf("%s/%s: %w", bucket, pattern, err) + } + } + match := func(key string) bool { + if glob != nil && !glob.MatchString(key) { + return false + } + return f.filter.match(key) + } + + flush := func() error { + if len(pending) == 0 { + return nil + } + err := deleteBatch(ctx, client, bucket, pending, f.dryRun, w) + pending = pending[:0] + return err + } + + opts := s3.ListOptions{Prefix: prefix, Versions: f.allVersions} + err := client.ListObjectsFunc(ctx, bucket, opts, func(obj s3.Object) error { + if !match(obj.Key) { + return nil + } + seen++ + pending = append(pending, s3.DeleteTarget{Key: obj.Key, VersionID: obj.VersionID}) + if len(pending) < s3.MaxDeleteBatch { + return nil + } + return flush() + }) + if err != nil { + return fmt.Errorf("deleting %s/%s recursively: %w", bucket, pattern, err) + } + if err := flush(); err != nil { + return err + } + if seen == 0 { + _, err = fmt.Fprintf(w, "No objects under %s/%s\n", bucket, pattern) + return err + } + return nil +} + +// deleteBatch removes one batch of keys in a single request, or says what it +// would have removed. +func deleteBatch(ctx context.Context, client *s3.Client, bucket string, + targets []s3.DeleteTarget, dryRun bool, w io.Writer) error { + if dryRun { + for _, t := range targets { + if err := sayWouldDelete(w, bucket, t.Key, t.VersionID); err != nil { + return err + } + } + return nil + } + + failures, err := client.DeleteObjects(ctx, bucket, targets) + if err != nil { + return err + } + + // A batch delete reports per-key refusals inside a 200, so the successes + // are the targets the server did not name. + refused := make(map[string]bool, len(failures)) + for _, fail := range failures { + refused[fail.Key+"\x00"+fail.VersionID] = true + } + for _, t := range targets { + if refused[t.Key+"\x00"+t.VersionID] { + continue + } + if err := sayDeleted(w, bucket, t.Key, t.VersionID); err != nil { + return err + } + } + + errs := make([]error, 0, len(failures)) + for _, fail := range failures { + errs = append(errs, fmt.Errorf("deleting %s/%s: %w", bucket, fail.Key, fail)) + } + return errors.Join(errs...) +} + +// describeTarget names one delete target for an error message. +func describeTarget(bucket, key, versionID string) string { + if versionID == "" { + return bucket + "/" + key + } + return fmt.Sprintf("%s/%s (version %s)", bucket, key, versionID) +} + +func sayDeleted(w io.Writer, bucket, key, versionID string) error { + _, err := fmt.Fprintf(w, "Deleted object: %s\n", describeTarget(bucket, key, versionID)) + return err +} + +func sayWouldDelete(w io.Writer, bucket, key, versionID string) error { + _, err := fmt.Fprintf(w, "Would delete object: %s\n", describeTarget(bucket, key, versionID)) + return err +} diff --git a/internal/cli/s3/object_delete_test.go b/internal/cli/s3/object_delete_test.go new file mode 100644 index 0000000..7c5e820 --- /dev/null +++ b/internal/cli/s3/object_delete_test.go @@ -0,0 +1,282 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" +) + +func TestRunObjectDelete(t *testing.T) { + var paths []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + refs := []string{"db-backups/a.sql.gz", "s3://db-backups/b.sql.gz"} + if err := runObjectDelete(context.Background(), client, refs, &objectDeleteFlags{}, &buf); err != nil { + t.Fatal(err) + } + want := "DELETE /db-backups/a.sql.gz,DELETE /db-backups/b.sql.gz" + if got := strings.Join(paths, ","); got != want { + t.Errorf("requests = %q, want %q", got, want) + } + wantOut := "Deleted object: db-backups/a.sql.gz\nDeleted object: db-backups/b.sql.gz\n" + if got := buf.String(); got != wantOut { + t.Errorf("output = %q, want %q", got, wantOut) + } +} + +// Without --recursive a bare bucket is not a target: it would be a one-word +// typo away from emptying the bucket. +func TestRunObjectDeleteRequiresKey(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("no request should be made for a keyless ref") + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups"}, &objectDeleteFlags{}, &buf) + if err == nil { + t.Fatal("expected a keyless ref to be rejected") + } +} + +func TestRunObjectDeleteRecursive(t *testing.T) { + var deleted []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + // A recursive delete goes out as one batched POST ?delete, not one + // DELETE per key: that is what makes emptying a large bucket finish. + if r.Method == http.MethodPost && r.URL.Query().Has("delete") { + body, _ := io.ReadAll(r.Body) + deleted = append(deleted, keysInBatch(string(body))...) + _, _ = fmt.Fprint(w, ``) + return + } + if r.Method == http.MethodDelete { + t.Errorf("a per-key DELETE was sent for %s", r.URL.Path) + } + if got := r.URL.Query().Get("prefix"); got != "e2e-" { + t.Errorf("prefix = %q, want e2e-", got) + } + _, _ = fmt.Fprint(w, `false + e2e-a.sql.gz1 + e2e-b.sql.gz2 + `) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/e2e-"}, + &objectDeleteFlags{recursive: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(deleted, ","), "e2e-a.sql.gz,e2e-b.sql.gz"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } + want := "Deleted object: db-backups/e2e-a.sql.gz\nDeleted object: db-backups/e2e-b.sql.gz\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// A wildcard reference is itself the instruction to go recursive, and the +// pattern narrows the listing the prefix could not. +func TestRunObjectDeleteWildcardRef(t *testing.T) { + var deleted []string + client := newMockClient(t, batchDeleteHandler(t, &deleted, + `2026/a.sql.gz`+ + `2026/a.sha256`)) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/2026/*.sql.gz"}, + &objectDeleteFlags{}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(deleted, ","), "2026/a.sql.gz"; got != want { + t.Errorf("deleted = %q, want only the pattern match", got) + } +} + +// --exclude removes a key the prefix would otherwise have taken. +func TestRunObjectDeleteExclude(t *testing.T) { + var deleted []string + client := newMockClient(t, batchDeleteHandler(t, &deleted, + `a.sql.gzkeep.sql.gz`)) + + f := &objectDeleteFlags{recursive: true, filter: keyFilter{exclude: []string{"keep*"}}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runObjectDelete(context.Background(), client, []string{"db-backups/"}, f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(deleted, ","), "a.sql.gz"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } +} + +// A key the server refuses inside the 200 must be reported as a failure, while +// the rest of the batch is still reported as deleted. +func TestRunObjectDeleteReportsPerKeyFailures(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + _, _ = fmt.Fprint(w, ` + lockedAccessDeniedno + `) + return + } + _, _ = fmt.Fprint(w, `false + finelocked + `) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/"}, + &objectDeleteFlags{recursive: true}, &buf) + if err == nil { + t.Fatal("a refused key was reported as success") + } + if !strings.Contains(err.Error(), "locked") { + t.Errorf("error %q does not name the refused key", err) + } + if got, want := buf.String(), "Deleted object: db-backups/fine\n"; got != want { + t.Errorf("output = %q, want only the key that was deleted", got) + } +} + +// --dry-run must list the same keys it would remove and issue no delete at all. +func TestRunObjectDeleteDryRun(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("--dry-run issued %s %s", r.Method, r.URL.Path) + } + _, _ = fmt.Fprint(w, `false + a.sql.gz + `) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/"}, + &objectDeleteFlags{recursive: true, dryRun: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "Would delete object: db-backups/a.sql.gz\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// An empty prefix must say so rather than reporting a silent success that reads +// like "everything was already deleted". +func TestRunObjectDeleteRecursiveNoMatches(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false`) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/gone-"}, + &objectDeleteFlags{recursive: true}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "No objects under db-backups/gone-\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// --version-id names exactly one object, so every shape that means "many" is a +// mistake worth refusing before anything is removed. +func TestObjectDeleteFlagValidation(t *testing.T) { + for _, tc := range []struct { + name string + flags objectDeleteFlags + refs []string + want string + }{ + {"version with recursive", objectDeleteFlags{versionID: "v1", recursive: true}, + []string{"b/k"}, "--recursive"}, + {"version with all-versions", objectDeleteFlags{versionID: "v1", allVersions: true}, + []string{"b/k"}, "mutually exclusive"}, + {"version with two refs", objectDeleteFlags{versionID: "v1"}, + []string{"b/k", "b/j"}, "a single reference"}, + {"filters without recursion", objectDeleteFlags{filter: keyFilter{include: []string{"*"}}}, + []string{"b/k"}, "--recursive"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.flags.validate(tc.refs) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } + + // A wildcard reference is its own recursion, so filters are legal with one. + f := objectDeleteFlags{filter: keyFilter{include: []string{"*.gz"}}} + if err := f.validate([]string{"b/2026/*"}); err != nil { + t.Errorf("filters with a wildcard ref were refused: %v", err) + } +} + +// A version-scoped delete addresses the one version and says which. +func TestRunObjectDeleteVersion(t *testing.T) { + var gotVersion string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotVersion = r.URL.Query().Get("versionId") + w.WriteHeader(http.StatusNoContent) + }) + + var buf bytes.Buffer + err := runObjectDelete(context.Background(), client, []string{"db-backups/dump.sql.gz"}, + &objectDeleteFlags{versionID: "v3"}, &buf) + if err != nil { + t.Fatal(err) + } + if gotVersion != "v3" { + t.Errorf("versionId = %q, want v3", gotVersion) + } + want := "Deleted object: db-backups/dump.sql.gz (version v3)\n" + if got := buf.String(); got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// batchDeleteHandler serves one listing page and records the keys of every +// batch delete it is sent. +func batchDeleteHandler(t *testing.T, deleted *[]string, contents string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Query().Has("delete") { + body, _ := io.ReadAll(r.Body) + *deleted = append(*deleted, keysInBatch(string(body))...) + _, _ = fmt.Fprint(w, ``) + return + } + _, _ = fmt.Fprint(w, `false`+contents+``) + } +} + +// keysInBatch pulls the keys out of a DeleteObjects request body. +func keysInBatch(body string) []string { + var keys []string + for rest := body; ; { + i := strings.Index(rest, "") + if i < 0 { + return keys + } + rest = rest[i+len(""):] + j := strings.Index(rest, "") + if j < 0 { + return keys + } + keys = append(keys, rest[:j]) + rest = rest[j:] + } +} diff --git a/internal/cli/s3/pool.go b/internal/cli/s3/pool.go new file mode 100644 index 0000000..e8fa797 --- /dev/null +++ b/internal/cli/s3/pool.go @@ -0,0 +1,94 @@ +package s3cli + +import ( + "context" + "sync" +) + +// pool runs transfers with a bounded number of goroutines. It exists because +// a recursive transfer of a thousand small objects is almost entirely round-trip +// latency: serially it takes a thousand round trips end to end, and with even +// four workers a quarter of that. s5cmd's headline speed is this and little +// else. +// +// The first failure cancels the pool's context so the rest stop pushing bytes at +// a store that is already refusing, and is what wait returns. Later failures are +// dropped: they are usually the cancellation, and a page of identical "context +// canceled" lines buries the one error that mattered. +type pool struct { + slots chan struct{} + wg sync.WaitGroup + cancel context.CancelFunc + + mu sync.Mutex + err error +} + +// newPool returns a pool of n workers and the context its work must use. +func newPool(ctx context.Context, n int) (*pool, context.Context) { + if n < 1 { + n = 1 + } + ctx, cancel := context.WithCancel(ctx) + return &pool{slots: make(chan struct{}, n), cancel: cancel}, ctx +} + +// run starts fn in the background once a worker slot frees up. It reports false +// when the pool has already failed or the context is done, which is the signal +// for the caller's producer loop to stop. +func (p *pool) run(ctx context.Context, fn func() error) bool { + select { + case p.slots <- struct{}{}: + case <-ctx.Done(): + p.fail(ctx.Err()) + return false + } + + p.wg.Add(1) + go func() { + defer p.wg.Done() + defer func() { <-p.slots }() + if err := fn(); err != nil { + p.fail(err) + } + }() + return true +} + +// fail records the first error and stops the others. +func (p *pool) fail(err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.err == nil { + p.err = err + p.cancel() + } +} + +// wait blocks until every started worker has finished and returns the first +// failure. It must be called exactly once, and the pool's cancel is released +// with it. +func (p *pool) wait() error { + p.wg.Wait() + p.cancel() + p.mu.Lock() + defer p.mu.Unlock() + return p.err +} + +// syncWriter serialises the per-object progress lines a pool's workers emit, so +// two transfers finishing at once cannot interleave half a line each. +// +// Lines therefore appear in completion order, which with more than one worker is +// not the listing order — a caller that needs a stable order must sort, and the +// tests do. +type syncWriter struct { + mu sync.Mutex + w interface{ Write([]byte) (int, error) } +} + +func (s *syncWriter) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.w.Write(p) +} diff --git a/internal/cli/s3/presign.go b/internal/cli/s3/presign.go new file mode 100644 index 0000000..74e9c15 --- /dev/null +++ b/internal/cli/s3/presign.go @@ -0,0 +1,104 @@ +package s3cli + +import ( + "errors" + "fmt" + "io" + "time" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +const presignLong = `Print a URL that reads (or writes) one object, without a key. + +The credential moves from the Authorization header into the query string, so the +whole request becomes a link anyone can fetch with curl or a browser until it +expires. It is how to hand a colleague one backup, or let a machine with no S3 +credentials at all deliver a dump, without sharing the access key. + +No request is made: this is local computation over the credentials koc already +has, so it works offline and leaves nothing on the store. That also means the URL +is not checked — presigning a key that does not exist succeeds and the URL 404s. + +Treat the output like a password. Anyone holding it can read that object until +it expires, and it will appear in shell history, CI logs and any chat it is +pasted into. Keep --expire as short as the recipient needs. + +--method put presigns an upload to the key instead of a download. SigV4 caps a +presigned URL's life at 7 days, and --s3-anonymous has nothing to sign with.` + +const presignExample = ` # A link good for an hour + koc s3 presign db-backups/dump-2026-09-13.sql.gz + + # A week, the protocol's maximum + koc s3 presign db-backups/dump.sql.gz --expire 168h + + # Hand an upload slot to a machine with no credentials + koc s3 presign db-backups/incoming.sql.gz --method put --expire 30m` + +func newPresignCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + var ( + expire time.Duration + method string + verID string + ) + cmd := &cobra.Command{ + Use: "presign /", + Short: "Print a presigned URL for an object", + Long: presignLong, + Example: presignExample, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, key, err := parseObjectRef(args[0]) + if err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runPresign(client, o, bucket, key, verID, method, expire, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.DurationVar(&expire, "expire", time.Hour, "how long the URL stays valid (maximum 168h)") + fl.StringVar(&method, "method", "get", "what the URL does: get or put") + fl.StringVar(&verID, "version-id", "", "presign this version of the object instead of the current one") + return cmd +} + +// runPresign is the test seam for "presign". It takes no context: presigning +// makes no request. +func runPresign(client *s3.Client, o *output.Options, bucket, key, versionID, method string, + expire time.Duration, w io.Writer) error { + var ( + url string + err error + ) + switch method { + case "get", "GET": + url, err = client.PresignGetObject(bucket, key, versionID, expire) + case "put", "PUT": + if versionID != "" { + return errors.New("--version-id addresses an existing object; it cannot be combined with --method put") + } + url, err = client.PresignPutObject(bucket, key, expire) + default: + return fmt.Errorf("--method must be get or put, got %q", method) + } + if err != nil { + return fmt.Errorf("presigning %s/%s: %w", bucket, key, err) + } + + return o.WriteSingle(w, + []string{"URL", "Method", "Expires"}, + []any{url, method, expire.String()}) +} diff --git a/internal/cli/s3/s3.go b/internal/cli/s3/s3.go index 2d6f1c1..2e313ae 100644 --- a/internal/cli/s3/s3.go +++ b/internal/cli/s3/s3.go @@ -10,7 +10,7 @@ import ( // s3Long documents the credential sources on the group itself. The connection // flags are this group's primary input and are registered here rather than // globally, so this is the only place they are discoverable. -const s3Long = `List buckets and objects in an S3-compatible store, and move files in and out. +const s3Long = `Manage buckets and objects in an S3-compatible store, and move data in and out. This group is koc-specific (S3 is not an OpenStack service, and upstream's object-store commands speak Swift) and does not authenticate against Keystone: @@ -42,7 +42,20 @@ variables, so export it from either: Addressing is path-style by default (//), matching the --host-bucket setting the backup pipeline gives s3cmd; pass --no-path-style for -a store fronted by wildcard DNS.` +a store fronted by wildcard DNS. + +Every reference accepts either "/" or the "s3:///" +spelling, so a path copied out of s5cmd or "aws s3" pastes in unchanged, and a +wildcard ("db-backups/2026/*.gz") selects many where a command takes one. + +Uploads are multipart past --part-size, so there is no 5 GiB ceiling, and "-" as +the source reads standard input. download, upload, copy, move and object delete +all take --recursive over a whole prefix or tree, --concurrency objects at a +time, with --include/--exclude and --dry-run. copy and move are server-side: the +bytes never travel through koc. + +A failed request is retried with backoff (--s3-retries); --s3-anonymous sends +requests unsigned, for a bucket granted to everyone.` const s3Example = ` # Everything the credentials can see koc s3 bucket list @@ -54,6 +67,26 @@ const s3Example = ` # Everything the credentials can see koc s3 download db-backups/ ./dump.sql.gz koc s3 download db-backups/.sha256 - + # How much space the backups take, per storage class + koc s3 du db-backups --group + + # A scratch bucket, and its removal once emptied + koc s3 bucket create scratch + koc s3 object delete scratch/ --recursive + koc s3 bucket delete scratch + + # Stream a dump straight in, no staging on disk + mysqldump --all-databases | gzip | koc s3 upload - db-backups/nightly.sql.gz + + # Restore a month, four objects at a time + koc s3 download db-backups/2026/08/ ./restore --recursive --concurrency 4 + + # Promote last night's dump, server-side + koc s3 copy db-backups/nightly-2026-09-13.sql.gz db-backups/latest.sql.gz + + # Hand someone one backup for an hour, without a key + koc s3 presign db-backups/latest.sql.gz --expire 1h + # GitLab's own key, straight from the cluster koc s3 --s3-creds-from-ns lcm-gitlab bucket list` @@ -72,5 +105,10 @@ func NewCommand(a *auth.Options, o *output.Options) *cobra.Command { cmd.AddCommand(newObjectCommand(a, o, f)) cmd.AddCommand(newDownloadCommand(a, o, f)) cmd.AddCommand(newUploadCommand(a, o, f)) + cmd.AddCommand(newDuCommand(a, o, f)) + cmd.AddCommand(newCopyCommand(a, o, f)) + cmd.AddCommand(newMoveCommand(a, o, f)) + cmd.AddCommand(newPresignCommand(a, o, f)) + cmd.AddCommand(newSyncCommand(a, o, f)) return cmd } diff --git a/internal/cli/s3/s3_test.go b/internal/cli/s3/s3_test.go index 43db5a2..e1d14f0 100644 --- a/internal/cli/s3/s3_test.go +++ b/internal/cli/s3/s3_test.go @@ -36,6 +36,18 @@ func newMockClient(t *testing.T, h http.HandlerFunc) *s3.Client { func valueOpts() *output.Options { return &output.Options{Format: output.FormatValue} } +// testUploadFlags is the flag set an "upload" seam gets when a test drives it +// directly: real defaults, but one part at a time so a mock endpoint sees the +// requests in a deterministic order. +func testUploadFlags() *uploadFlags { + return &uploadFlags{partSizeMiB: s3.DefaultPartSize >> 20, concurrency: 1} +} + +// testDownloadFlags is the same for "download". +func testDownloadFlags() *downloadFlags { + return &downloadFlags{concurrency: 1} +} + func TestRunBucketList(t *testing.T) { client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { _, _ = fmt.Fprint(w, ` @@ -88,7 +100,7 @@ func TestRunObjectShow(t *testing.T) { var buf bytes.Buffer err := runObjectShow(context.Background(), client, &output.Options{Format: output.FormatJSON}, - "db-backups", "e2e-mariadb.sql.gz.sha256", &buf) + "db-backups", "e2e-mariadb.sql.gz.sha256", "", false, &buf) if err != nil { t.Fatal(err) } @@ -98,7 +110,7 @@ func TestRunObjectShow(t *testing.T) { } } - err = runObjectShow(context.Background(), client, valueOpts(), "db-backups", "missing", &buf) + err = runObjectShow(context.Background(), client, valueOpts(), "db-backups", "missing", "", false, &buf) if err == nil || !strings.Contains(err.Error(), `no object "missing" in bucket "db-backups"`) { t.Errorf("err = %v, want a friendly not-found message", err) } @@ -115,7 +127,7 @@ func TestRunDownloadToFile(t *testing.T) { var buf bytes.Buffer dest := filepath.Join(dir, "out.gz") err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -132,12 +144,12 @@ func TestRunDownloadToFile(t *testing.T) { // A second run must refuse rather than clobber the local copy of a backup. err = runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, flags: &downloadFlags{}}, &buf) if err == nil || !strings.Contains(err.Error(), "--force") { t.Errorf("err = %v, want a refusal pointing at --force", err) } if err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, force: true}, &buf); err != nil { + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dest, flags: &downloadFlags{force: true}}, &buf); err != nil { t.Errorf("--force download failed: %v", err) } } @@ -150,7 +162,7 @@ func TestRunDownloadStdout(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz.sha256", dest: "-"}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz.sha256", dest: "-", flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -170,7 +182,7 @@ func TestRunDownloadFailureLeavesNoFile(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "missing", dest: dest}, &buf) + downloadRequest{bucket: "db-backups", key: "missing", dest: dest, flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("expected an error") } @@ -197,7 +209,8 @@ func TestRunUpload(t *testing.T) { // No key given: the file's basename is used. var buf bytes.Buffer - if err := runUpload(context.Background(), client, valueOpts(), file, "db-backups", "", &buf); err != nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: file, bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf); err != nil { t.Fatal(err) } if gotPath != "/db-backups/dump.sql.gz" { @@ -214,14 +227,16 @@ func TestRunUpload(t *testing.T) { } // A key ending in "/" is a prefix, not a key. - if err := runUpload(context.Background(), client, valueOpts(), file, "db-backups", "2026/", &buf); err != nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: file, bucket: "db-backups", key: "2026/", flags: testUploadFlags()}, &buf); err != nil { t.Fatal(err) } if gotPath != "/db-backups/2026/dump.sql.gz" { t.Errorf("path = %q, want the basename appended to the prefix", gotPath) } - if err := runUpload(context.Background(), client, valueOpts(), filepath.Dir(file), "db-backups", "", &buf); err == nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: filepath.Dir(file), bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf); err == nil { t.Error("uploading a directory was accepted") } } diff --git a/internal/cli/s3/stop.go b/internal/cli/s3/stop.go new file mode 100644 index 0000000..bc2ab82 --- /dev/null +++ b/internal/cli/s3/stop.go @@ -0,0 +1,11 @@ +package s3cli + +import "errors" + +// errStopListing ends a ListObjectsFunc walk early without making the caller +// treat it as a failure. The client propagates a callback error verbatim, which +// is exactly what a "stop here" signal needs — and what keeps a --limit from +// paging through the rest of a million-key bucket. +var errStopListing = errors.New("stop listing") + +func isStopListing(err error) bool { return errors.Is(err, errStopListing) } diff --git a/internal/cli/s3/sync.go b/internal/cli/s3/sync.go new file mode 100644 index 0000000..3feebd1 --- /dev/null +++ b/internal/cli/s3/sync.go @@ -0,0 +1,547 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// syncFlags holds the options accepted by "sync". +type syncFlags struct { + del bool + sizeOnly bool + dryRun bool + force bool + concurrency int + partSizeMiB int + filter keyFilter +} + +const syncLong = `Make a destination match a source, transferring only what differs. + +Both sides are named with the s3:// prefix for an S3 location and a plain path +for a local one: + + koc s3 sync ./restore s3://db-backups/2026/ # local -> S3 + koc s3 sync s3://db-backups/2026/ ./restore # S3 -> local + koc s3 sync s3://db-backups/ s3://archive/ # S3 -> S3 (server-side) + +sync is the one command in this group that *requires* the s3:// prefix. Every +other verb accepts a bare "/" because it has a single subject; sync +has two, and mistaking which one is local is how a --delete removes the wrong +side. A bare path is always local here. + +An S3 side is a prefix, treated as a directory: a trailing "/" is added if it is +missing, and each object's key below it is what the two sides are matched on. A +local side is a directory tree; symlinks and devices are skipped rather than +followed. + +An object is transferred when it is absent at the destination, when the sizes +differ, or when the source is newer. --size-only drops the timestamp test, which +is what you want for content that never changes in place (a dated backup) and +the cheapest correct rule there is. + +The timestamp on the S3 side is the object's Last-Modified, i.e. when it was +uploaded — S3 keeps no record of the source file's own mtime, and a listing +would not return one if it did. That is correct in the steady state, because an +upload always lands after the file was written and a download always lands after +the object was: neither direction re-transfers what it just moved. The one seam +is a round trip — files restored with "download" carry the restore time, so +syncing that tree *back* re-uploads it once, after which it settles. Pass +--size-only to avoid even that. + +--delete removes destination entries the source does not have, which is what +makes this a mirror rather than an overlay. It is refused when the source turned +out to be empty unless --force is also given: a mistyped source that lists +nothing would otherwise erase the destination. Entries excluded by +--include/--exclude are outside the sync and are never deleted. Local deletion +removes files, not the directories left behind. + +--dry-run prints exactly the transfers and deletions it would perform.` + +const syncExample = ` # Mirror a restore tree up, removing what is no longer local + koc s3 sync ./restore s3://db-backups/2026/ --delete + + # Pull a month down, eight objects at a time + koc s3 sync s3://db-backups/2026/08/ ./restore --concurrency 8 + + # Dated dumps never change in place, so size is the whole test + koc s3 sync s3://db-backups/ s3://archive/ --size-only + + # See what would move, before it moves + koc s3 sync ./restore s3://db-backups/2026/ --delete --dry-run + + # Only the compressed dumps + koc s3 sync s3://db-backups/ ./restore --include "*.sql.gz"` + +func newSyncCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + sf := &syncFlags{} + cmd := &cobra.Command{ + Use: "sync ", + Short: "Make a destination match a source, transferring only what differs", + Long: syncLong, + Example: syncExample, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + src, dst, err := parseSyncEndpoints(args[0], args[1]) + if err != nil { + return err + } + if err := sf.filter.compile(); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runSync(ctx, client, src, dst, sf, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVar(&sf.del, "delete", false, + "remove destination entries the source does not have") + fl.BoolVar(&sf.sizeOnly, "size-only", false, + "compare only sizes, ignoring timestamps") + fl.BoolVar(&sf.dryRun, "dry-run", false, + "print the transfers and deletions that would happen, and perform none") + fl.BoolVar(&sf.force, "force", false, + "allow --delete even when the source turned out to be empty") + fl.IntVar(&sf.concurrency, "concurrency", s3.DefaultConcurrency, + "objects to transfer at once") + fl.IntVar(&sf.partSizeMiB, "part-size", s3.DefaultPartSize>>20, + "multipart part size in MiB for uploads (minimum 5)") + sf.filter.addTo(fl) + return cmd +} + +// syncEndpoint is one end of a sync: a local directory, or a prefix in a +// bucket. Exactly one of dir / bucket is set. +type syncEndpoint struct { + dir string + + bucket string + // prefix always ends in "/" when non-empty, so a key below it trims + // cleanly to the relative path the two sides are matched on. + prefix string +} + +func (e syncEndpoint) isLocal() bool { return e.bucket == "" } + +func (e syncEndpoint) String() string { + if e.isLocal() { + return e.dir + } + return "s3://" + e.bucket + "/" + e.prefix +} + +// join renders the full address of one relative key on this side. +func (e syncEndpoint) join(key string) string { + if e.isLocal() { + return filepath.Join(e.dir, filepath.FromSlash(key)) + } + return e.prefix + key +} + +// parseSyncEndpoints resolves both arguments and rejects the pairs that are not +// a sync. +func parseSyncEndpoints(srcRef, dstRef string) (src, dst syncEndpoint, err error) { + if src, err = parseSyncEndpoint(srcRef); err != nil { + return src, dst, err + } + if dst, err = parseSyncEndpoint(dstRef); err != nil { + return src, dst, err + } + if src.isLocal() && dst.isLocal() { + return src, dst, errors.New("at least one side must be an s3:// location; " + + "two local paths are a job for cp or rsync") + } + if src == dst { + return src, dst, fmt.Errorf("source and destination are the same location (%s)", src) + } + return src, dst, nil +} + +// parseSyncEndpoint reads one side. The s3:// prefix is mandatory for an S3 +// location here — see the command's Long for why sync alone insists on it. +func parseSyncEndpoint(ref string) (syncEndpoint, error) { + rest, ok := strings.CutPrefix(ref, "s3://") + if !ok { + if ref == "" { + return syncEndpoint{}, errors.New("a sync side cannot be empty") + } + return syncEndpoint{dir: ref}, nil + } + + bucket, prefix, _ := strings.Cut(strings.TrimPrefix(rest, "/"), "/") + if bucket == "" { + return syncEndpoint{}, fmt.Errorf("%q names no bucket: expected s3://[/]", ref) + } + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + return syncEndpoint{bucket: bucket, prefix: prefix}, nil +} + +// checkLocalSides rejects a local side that is not a directory tree. A source +// that is a single file would otherwise walk to the relative path "." and +// upload itself under that key; a destination that is a file would fail later, +// halfway through the transfers. +func checkLocalSides(src, dst syncEndpoint) error { + if src.isLocal() { + info, err := os.Stat(src.dir) + if err != nil { + return fmt.Errorf("reading source %q: %w", src.dir, err) + } + if !info.IsDir() { + return fmt.Errorf("source %q is not a directory; sync mirrors trees, "+ + "use \"koc s3 upload\" for one file", src.dir) + } + } + // A destination that does not exist yet is about to be created, so only an + // existing non-directory is a problem. + if dst.isLocal() { + if info, err := os.Stat(dst.dir); err == nil && !info.IsDir() { + return fmt.Errorf("destination %q is not a directory", dst.dir) + } + } + return nil +} + +// syncEntry is one item on either side, identified by its path relative to that +// side's root — which is what makes a local file and an object comparable. +type syncEntry struct { + key string + size int64 + mtime time.Time + // path is the local file's path; empty for an object. + path string +} + +// runSync is the test seam for "sync". +func runSync(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + f *syncFlags, w io.Writer) error { + if err := checkLocalSides(src, dst); err != nil { + return err + } + + // The destination is indexed rather than the source: a lookup per source + // entry is what decides a transfer, and --delete then needs exactly the + // entries nothing looked up. Memory is therefore the destination's size — + // the two sides cannot be streamed against each other because S3 lists + // keys lexicographically while a directory walk descends depth-first, so + // "a.txt" and "a/b" come out in opposite orders. + index, err := indexEndpoint(ctx, client, dst, f) + if err != nil { + return err + } + + seen := make(map[string]bool, len(index)) + out := &syncWriter{w: w} + p, workCtx := newPool(ctx, f.concurrency) + source, transferred := 0, 0 + + err = walkEndpoint(ctx, client, src, f, func(e syncEntry) error { + source++ + seen[e.key] = true + if at, ok := index[e.key]; ok && !f.needsTransfer(e, at) { + return nil + } + transferred++ + if !p.run(workCtx, func() error { return syncOne(workCtx, client, src, dst, e, f, out) }) { + return errStopListing + } + return nil + }) + if waitErr := p.wait(); err == nil || isStopListing(err) { + err = waitErr + } + if err != nil && !isStopListing(err) { + return fmt.Errorf("syncing %s to %s: %w", src, dst, err) + } + + removed, err := syncDelete(ctx, client, dst, index, seen, source, f, out) + if err != nil { + return err + } + if transferred == 0 && removed == 0 { + _, err = fmt.Fprintf(w, "Already in sync: %s and %s (%d objects)\n", src, dst, source) + return err + } + return nil +} + +// needsTransfer is the comparison rule. Timestamps are truncated to the second +// because S3 reports Last-Modified at that resolution while a local mtime +// carries nanoseconds, and the difference alone would make every file look +// newer than its own copy. +func (f *syncFlags) needsTransfer(src, dst syncEntry) bool { + if src.size != dst.size { + return true + } + if f.sizeOnly { + return false + } + return src.mtime.Truncate(time.Second).After(dst.mtime.Truncate(time.Second)) +} + +// indexEndpoint reads one side into a map keyed by relative path. A local +// destination that does not exist yet is an empty index, not an error — it is +// about to be created. +func indexEndpoint(ctx context.Context, client *s3.Client, e syncEndpoint, + f *syncFlags) (map[string]syncEntry, error) { + index := map[string]syncEntry{} + err := walkEndpoint(ctx, client, e, f, func(entry syncEntry) error { + index[entry.key] = entry + return nil + }) + if err != nil && e.isLocal() && errors.Is(err, os.ErrNotExist) { + return index, nil + } + if err != nil { + return nil, fmt.Errorf("reading %s: %w", e, err) + } + return index, nil +} + +// walkEndpoint enumerates one side, applying --include/--exclude so that a +// filtered-out entry is invisible to both the comparison and --delete. +func walkEndpoint(ctx context.Context, client *s3.Client, e syncEndpoint, + f *syncFlags, fn func(syncEntry) error) error { + keep := func(entry syncEntry) error { + if !f.filter.match(entry.key) { + return nil + } + return fn(entry) + } + if e.isLocal() { + return walkLocalTree(e.dir, keep) + } + return walkRemotePrefix(ctx, client, e, keep) +} + +// walkLocalTree reports every regular file under root. +func walkLocalTree(root string, fn func(syncEntry) error) error { + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + // Symlinks and devices are skipped rather than followed: a tree walked + // from a backup directory should not chase a link out of it. + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return fmt.Errorf("resolving %q under %q: %w", path, root, err) + } + info, err := d.Info() + if err != nil { + return fmt.Errorf("stat %q: %w", path, err) + } + return fn(syncEntry{ + key: filepath.ToSlash(rel), + size: info.Size(), + mtime: info.ModTime(), + path: path, + }) + }) +} + +// walkRemotePrefix reports every object below the endpoint's prefix. +func walkRemotePrefix(ctx context.Context, client *s3.Client, e syncEndpoint, + fn func(syncEntry) error) error { + return client.ListObjectsFunc(ctx, e.bucket, s3.ListOptions{Prefix: e.prefix}, + func(obj s3.Object) error { + // A "directory marker" — a zero-length key ending in "/" that some + // tools create — has no counterpart on a filesystem. + if strings.HasSuffix(obj.Key, "/") { + return nil + } + return fn(syncEntry{ + key: strings.TrimPrefix(obj.Key, e.prefix), + size: obj.Size, + mtime: obj.LastModified, + }) + }) +} + +// syncOne transfers one entry from src to dst, by whichever of the three routes +// the pair of endpoints implies. A local-to-local pair was rejected at parse. +func syncOne(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + switch { + case src.isLocal(): + return syncUpload(ctx, client, dst, e, f, w) + case dst.isLocal(): + return syncDownload(ctx, client, src, dst, e, f, w) + default: + return syncCopy(ctx, client, src, dst, e, f, w) + } +} + +// syncUpload sends one local file to the destination prefix. +func syncUpload(ctx context.Context, client *s3.Client, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + key := dst.join(e.key) + if f.dryRun { + _, err := fmt.Fprintf(w, "Would upload %s to %s/%s\n", e.path, dst.bucket, key) + return err + } + + src, err := os.Open(e.path) + if err != nil { + return fmt.Errorf("opening %q: %w", e.path, err) + } + defer func() { _ = src.Close() }() + + // One part at a time: the pool's workers are already objects, and the + // product of the two would put concurrency² part buffers in memory. + opts := s3.UploadOptions{ + ContentType: contentTypeFor(e.path), + PartSize: int64(f.partSizeMiB) << 20, + Concurrency: 1, + Size: e.size, + } + obj, err := client.PutObjectStream(ctx, dst.bucket, key, src, opts) + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", e.path, dst.bucket, key, err) + } + _, err = fmt.Fprintf(w, "Uploaded: %s -> %s/%s (%d bytes)\n", e.path, dst.bucket, key, obj.Size) + return err +} + +// syncDownload fetches one object into the destination tree. +func syncDownload(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + key := src.join(e.key) + // A key is server-supplied data, and "../../etc/cron.d/x" is a legal one, + // so the destination is checked to be inside the tree before anything is + // created. + path, err := destForKey(dst.dir, "", e.key, false) + if err != nil { + return err + } + if f.dryRun { + _, err := fmt.Fprintf(w, "Would download %s/%s to %s\n", src.bucket, key, path) + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + + // force: replacing a stale copy is the whole point of a sync, unlike the + // single-object "download" where an existing file means a mistyped key. + n, err := downloadToFile(ctx, client, src.bucket, key, "", path, true) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "Downloaded: %s/%s -> %s (%d bytes)\n", src.bucket, key, path, n) + return err +} + +// syncCopy moves one object between two S3 locations, server-side. +func syncCopy(ctx context.Context, client *s3.Client, src, dst syncEndpoint, + e syncEntry, f *syncFlags, w io.Writer) error { + from := s3.ObjectRef{Bucket: src.bucket, Key: src.join(e.key)} + to := s3.ObjectRef{Bucket: dst.bucket, Key: dst.join(e.key)} + if f.dryRun { + return sayCopy(w, from, to, false, true) + } + if _, err := client.CopyObject(ctx, from, to, "", ""); err != nil { + return err + } + return sayCopy(w, from, to, false, false) +} + +// syncDelete removes the destination entries the source did not have, and +// reports how many it took. +func syncDelete(ctx context.Context, client *s3.Client, dst syncEndpoint, + index map[string]syncEntry, seen map[string]bool, source int, + f *syncFlags, w io.Writer) (int, error) { + if !f.del { + return 0, nil + } + // A mistyped source that listed nothing would otherwise erase the + // destination, and that is not a mistake anyone recovers from by re-running + // the command. + if source == 0 && !f.force { + return 0, fmt.Errorf("the source is empty, so --delete would remove all %d "+ + "entries under %s; pass --force if that is what you meant", len(index), dst) + } + + extra := make([]string, 0, len(index)) + for key := range index { + if !seen[key] { + extra = append(extra, key) + } + } + // Sorted so a run is reproducible and its output diffs against the next. + sort.Strings(extra) + if len(extra) == 0 { + return 0, nil + } + + if dst.isLocal() { + return len(extra), deleteLocalExtras(index, extra, f, w) + } + return len(extra), deleteRemoteExtras(ctx, client, dst, extra, f, w) +} + +// deleteLocalExtras removes local files the source no longer has. The +// directories they leave behind are not removed — an empty directory is not +// something the source can be said to have deleted. +func deleteLocalExtras(index map[string]syncEntry, extra []string, f *syncFlags, w io.Writer) error { + for _, key := range extra { + path := index[key].path + if f.dryRun { + if _, err := fmt.Fprintf(w, "Would delete file: %s\n", path); err != nil { + return err + } + continue + } + if err := os.Remove(path); err != nil { + return fmt.Errorf("deleting %q: %w", path, err) + } + if _, err := fmt.Fprintf(w, "Deleted file: %s\n", path); err != nil { + return err + } + } + return nil +} + +// deleteRemoteExtras removes objects the source no longer has, batched so a +// large mirror does not cost one request per key. +func deleteRemoteExtras(ctx context.Context, client *s3.Client, dst syncEndpoint, + extra []string, f *syncFlags, w io.Writer) error { + targets := make([]s3.DeleteTarget, 0, len(extra)) + for _, key := range extra { + targets = append(targets, s3.DeleteTarget{Key: dst.join(key)}) + } + for len(targets) > 0 { + batch := targets + if len(batch) > s3.MaxDeleteBatch { + batch = batch[:s3.MaxDeleteBatch] + } + if err := deleteBatch(ctx, client, dst.bucket, batch, f.dryRun, w); err != nil { + return err + } + targets = targets[len(batch):] + } + return nil +} diff --git a/internal/cli/s3/sync_test.go b/internal/cli/s3/sync_test.go new file mode 100644 index 0000000..0270988 --- /dev/null +++ b/internal/cli/s3/sync_test.go @@ -0,0 +1,482 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +func testSyncFlags() *syncFlags { + return &syncFlags{concurrency: 1, partSizeMiB: s3.DefaultPartSize >> 20} +} + +// syncMock serves a listing whose objects carry a size and a Last-Modified, and +// records every write it is sent. +type syncMock struct { + objects map[string]syncObject + puts []string + deletes []string + copies []string + gets []string +} + +type syncObject struct { + size int64 + mtime string +} + +func (m *syncMock) handler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case q.Has("list-type"): + prefix := q.Get("prefix") + var rows string + for key, o := range m.objects { + if !strings.HasPrefix(key, prefix) { + continue + } + rows += fmt.Sprintf(`%s%d`+ + `%s"e"`, key, o.size, o.mtime) + } + _, _ = fmt.Fprintf(w, `false%s`, rows) + case r.Method == http.MethodPost && q.Has("delete"): + body := readAll(r) + m.deletes = append(m.deletes, keysInBatch(body)...) + _, _ = fmt.Fprint(w, ``) + case r.Method == http.MethodPut && r.Header.Get("x-amz-copy-source") != "": + m.copies = append(m.copies, r.Header.Get("x-amz-copy-source")+" -> "+r.URL.Path) + _, _ = fmt.Fprint(w, `"e"`) + case r.Method == http.MethodPut: + m.puts = append(m.puts, r.URL.Path) + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + default: + m.gets = append(m.gets, r.URL.Path) + _, _ = w.Write([]byte("data")) + } + } +} + +func readAll(r *http.Request) string { + var buf bytes.Buffer + _, _ = buf.ReadFrom(r.Body) + return buf.String() +} + +func localSide(dir string) syncEndpoint { return syncEndpoint{dir: dir} } +func s3Side(b, p string) syncEndpoint { return syncEndpoint{bucket: b, prefix: p} } + +// The s3:// prefix is mandatory on this command alone, because mistaking which +// side is local is how a --delete removes the wrong one. +func TestParseSyncEndpoints(t *testing.T) { + src, dst, err := parseSyncEndpoints("./restore", "s3://db-backups/2026") + if err != nil { + t.Fatal(err) + } + if !src.isLocal() || src.dir != "./restore" { + t.Errorf("source = %+v, want the local path", src) + } + // A prefix is a directory, so the trailing slash is supplied. + if dst.bucket != "db-backups" || dst.prefix != "2026/" { + t.Errorf("destination = %+v, want prefix 2026/", dst) + } + // A bare bucket has an empty prefix rather than "/". + remote, err := parseSyncEndpoint("s3://db-backups") + if err != nil { + t.Fatal(err) + } + if remote.prefix != "" { + t.Errorf("prefix = %q, want empty", remote.prefix) + } +} + +func TestParseSyncEndpointsRejectsBadPairs(t *testing.T) { + for _, tc := range []struct{ name, src, dst, want string }{ + {"two local paths", "./a", "./b", "at least one side"}, + {"same location", "s3://b/p", "s3://b/p/", "the same location"}, + {"no bucket", "s3://", "./a", "names no bucket"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, err := parseSyncEndpoints(tc.src, tc.dst); err == nil || + !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// The comparison rule: absent, a different size, or a newer source. +func TestNeedsTransfer(t *testing.T) { + old := time.Date(2026, 9, 1, 10, 0, 0, 0, time.UTC) + recent := old.Add(time.Hour) + + for _, tc := range []struct { + name string + src, dst syncEntry + sizeOnly bool + want bool + }{ + {"same size and time", syncEntry{size: 10, mtime: old}, syncEntry{size: 10, mtime: old}, false, false}, + {"different size", syncEntry{size: 11, mtime: old}, syncEntry{size: 10, mtime: old}, false, true}, + {"source newer", syncEntry{size: 10, mtime: recent}, syncEntry{size: 10, mtime: old}, false, true}, + {"destination newer", syncEntry{size: 10, mtime: old}, syncEntry{size: 10, mtime: recent}, false, false}, + {"--size-only ignores a newer source", syncEntry{size: 10, mtime: recent}, syncEntry{size: 10, mtime: old}, true, false}, + {"--size-only still sees a size change", syncEntry{size: 11, mtime: old}, syncEntry{size: 10, mtime: old}, true, true}, + // S3 reports Last-Modified to the second while a local mtime carries + // nanoseconds; without truncation every file looks newer than its copy. + {"sub-second difference is not newer", + syncEntry{size: 10, mtime: old.Add(400 * time.Millisecond)}, syncEntry{size: 10, mtime: old}, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := &syncFlags{sizeOnly: tc.sizeOnly} + if got := f.needsTransfer(tc.src, tc.dst); got != tc.want { + t.Errorf("needsTransfer = %v, want %v", got, tc.want) + } + }) + } +} + +// Only what differs moves: a matching object is left alone, so a re-run of a +// finished sync costs one listing per side and no transfers. +func TestRunSyncLocalToS3TransfersOnlyDifferences(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "same.txt"), "1234") + mustWrite(t, filepath.Join(dir, "changed.txt"), "1234567") + mustWrite(t, filepath.Join(dir, "new.txt"), "12") + // Both existing objects are newer than the files, so only the size + // difference should move. + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + m := &syncMock{objects: map[string]syncObject{ + "2026/same.txt": {size: 4, mtime: future}, + "2026/changed.txt": {size: 99, mtime: future}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(dir), s3Side("db-backups", "2026/"), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + want := "/db-backups/2026/changed.txt,/db-backups/2026/new.txt" + if got := strings.Join(sortedStrings(m.puts), ","); got != want { + t.Errorf("uploaded = %q, want %q", got, want) + } +} + +// A finished sync says so, rather than printing nothing — which would be +// indistinguishable from "the source matched nothing". +func TestRunSyncAlreadyInSync(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "a.txt"), "1234") + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + m := &syncMock{objects: map[string]syncObject{"a.txt": {size: 4, mtime: future}}} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(dir), s3Side("db-backups", ""), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + if len(m.puts) != 0 { + t.Errorf("uploaded %v, want nothing", m.puts) + } + if !strings.Contains(buf.String(), "Already in sync") { + t.Errorf("output = %q, want it to say the sides match", buf.String()) + } +} + +// S3 -> local: the tree is created and each key lands at its path below the +// prefix. +func TestRunSyncS3ToLocal(t *testing.T) { + dir := filepath.Join(t.TempDir(), "restore") + m := &syncMock{objects: map[string]syncObject{ + "2026/08/a.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + "2026/09/b.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + "2025/old.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, s3Side("db-backups", "2026/"), localSide(dir), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + for _, rel := range []string{"08/a.sql.gz", "09/b.sql.gz"} { + if _, err := os.Stat(filepath.Join(dir, rel)); err != nil { + t.Errorf("%s was not written: %v", rel, err) + } + } + // The prefix is the scope: 2025/ is outside it. + if _, err := os.Stat(filepath.Join(dir, "..", "old.sql.gz")); err == nil { + t.Error("an object outside the prefix was transferred") + } +} + +// A stale local copy is replaced, unlike the single-object "download" where an +// existing file means a mistyped key. +func TestRunSyncS3ToLocalReplacesStaleFiles(t *testing.T) { + dir := t.TempDir() + stale := filepath.Join(dir, "a.txt") + mustWrite(t, stale, "old") + m := &syncMock{objects: map[string]syncObject{ + "a.txt": {size: 4, mtime: time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z")}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + if body, _ := os.ReadFile(stale); string(body) != "data" { + t.Errorf("the stale file was not replaced: %q", body) + } +} + +// S3 -> S3 is server-side: the bytes never travel through koc. +func TestRunSyncS3ToS3IsServerSide(t *testing.T) { + m := &syncMock{objects: map[string]syncObject{ + "2026/a.sql.gz": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + err := runSync(context.Background(), client, s3Side("db-backups", "2026/"), s3Side("archive", "old/"), + testSyncFlags(), &buf) + if err != nil { + t.Fatal(err) + } + want := "/db-backups/2026/a.sql.gz -> /archive/old/a.sql.gz" + if got := strings.Join(m.copies, ","); got != want { + t.Errorf("copies = %q, want %q", got, want) + } + if len(m.gets) != 0 { + t.Errorf("objects were fetched (%v); a server-side copy transfers nothing", m.gets) + } +} + +// --delete is what makes this a mirror: a destination entry the source does not +// have goes away, batched into one request. +func TestRunSyncDeleteRemovesExtraObjects(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "keep.txt"), "1234") + future := time.Now().Add(time.Hour).UTC().Format("2006-01-02T15:04:05.000Z") + m := &syncMock{objects: map[string]syncObject{ + "keep.txt": {size: 4, mtime: future}, + "stale.txt": {size: 4, mtime: future}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del = true + var buf bytes.Buffer + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(m.deletes, ","), "stale.txt"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } + if len(m.puts) != 0 { + t.Errorf("uploaded %v, want nothing — only the extra object changed", m.puts) + } +} + +func TestRunSyncDeleteRemovesExtraLocalFiles(t *testing.T) { + dir := t.TempDir() + stale := filepath.Join(dir, "sub", "stale.txt") + mustWrite(t, stale, "old") + m := &syncMock{objects: map[string]syncObject{}} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del, f.force = true, true // the source is empty on purpose here + var buf bytes.Buffer + if err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), f, &buf); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); err == nil { + t.Error("the extra local file was not deleted") + } + // The directory it was in stays: an empty directory is not something the + // source can be said to have deleted. + if _, err := os.Stat(filepath.Dir(stale)); err != nil { + t.Errorf("the containing directory was removed: %v", err) + } +} + +// The sharp edge: a mistyped source that lists nothing must not erase the +// destination. +func TestRunSyncDeleteRefusesAnEmptySource(t *testing.T) { + dir := t.TempDir() // empty + m := &syncMock{objects: map[string]syncObject{ + "a.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + "b.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del = true + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf) + if err == nil { + t.Fatal("an empty source was allowed to delete the whole destination") + } + if !strings.Contains(err.Error(), "--force") || !strings.Contains(err.Error(), "2 entries") { + t.Errorf("error %q does not say what it refused and how to override", err) + } + if len(m.deletes) != 0 { + t.Errorf("deleted %v despite refusing", m.deletes) + } + + // With --force it goes ahead, because that is now an explicit instruction. + f.force = true + buf.Reset() + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(sortedStrings(m.deletes), ","), "a.txt,b.txt"; got != want { + t.Errorf("deleted = %q, want %q", got, want) + } +} + +// An entry the filter excludes is outside the sync altogether: it is neither +// transferred nor deleted. +func TestRunSyncFilterExcludesFromBothSides(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "a.sql.gz"), "1234") + mustWrite(t, filepath.Join(dir, "a.sha256"), "12") + m := &syncMock{objects: map[string]syncObject{ + "other.sha256": {size: 9, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del = true + f.filter = keyFilter{include: []string{"*.sql.gz"}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if got, want := strings.Join(m.puts, ","), "/b/a.sql.gz"; got != want { + t.Errorf("uploaded = %q, want only the included file", got) + } + if len(m.deletes) != 0 { + t.Errorf("deleted %v — an excluded destination entry is not the sync's to remove", m.deletes) + } +} + +// --dry-run must name every transfer and deletion it would perform, and perform +// none of them. +func TestRunSyncDryRun(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "new.txt"), "12") + m := &syncMock{objects: map[string]syncObject{ + "stale.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + f := testSyncFlags() + f.del, f.dryRun = true, true + var buf bytes.Buffer + if err := runSync(context.Background(), client, localSide(dir), s3Side("b", ""), f, &buf); err != nil { + t.Fatal(err) + } + if len(m.puts)+len(m.deletes)+len(m.copies) != 0 { + t.Errorf("--dry-run wrote: puts=%v deletes=%v copies=%v", m.puts, m.deletes, m.copies) + } + out := buf.String() + if !strings.Contains(out, "Would upload") || !strings.Contains(out, "Would delete object: b/stale.txt") { + t.Errorf("output = %q, want both planned actions named", out) + } +} + +// A local side that is a single file, or an existing non-directory destination, +// is a mistake worth naming before any transfer. +func TestRunSyncChecksLocalSides(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("a request was made despite an invalid local side: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + }) + + file := filepath.Join(t.TempDir(), "one.txt") + mustWrite(t, file, "x") + + var buf bytes.Buffer + err := runSync(context.Background(), client, localSide(file), s3Side("b", ""), testSyncFlags(), &buf) + if err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Errorf("err = %v, want it to reject a file as the source", err) + } + + err = runSync(context.Background(), client, s3Side("b", ""), localSide(file), testSyncFlags(), &buf) + if err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Errorf("err = %v, want it to reject a file as the destination", err) + } + + err = runSync(context.Background(), client, localSide(filepath.Join(t.TempDir(), "absent")), + s3Side("b", ""), testSyncFlags(), &buf) + if err == nil || !strings.Contains(err.Error(), "reading source") { + t.Errorf("err = %v, want it to name the missing source", err) + } +} + +// A local destination that does not exist yet is an empty index, not an error: +// it is about to be created. +func TestRunSyncCreatesAMissingLocalDestination(t *testing.T) { + dir := filepath.Join(t.TempDir(), "new", "deep") + m := &syncMock{objects: map[string]syncObject{ + "a.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + if err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), testSyncFlags(), &buf); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "a.txt")); err != nil { + t.Errorf("the destination tree was not created: %v", err) + } +} + +// A directory marker — a zero-length key ending in "/" that some tools leave +// behind — has no counterpart on a filesystem and must not become a file. +func TestRunSyncSkipsDirectoryMarkers(t *testing.T) { + dir := t.TempDir() + m := &syncMock{objects: map[string]syncObject{ + "2026/": {size: 0, mtime: "2026-09-01T10:00:00.000Z"}, + "2026/a.txt": {size: 4, mtime: "2026-09-01T10:00:00.000Z"}, + }} + client := newMockClient(t, m.handler(t)) + + var buf bytes.Buffer + if err := runSync(context.Background(), client, s3Side("b", ""), localSide(dir), testSyncFlags(), &buf); err != nil { + t.Fatal(err) + } + if got := strings.Join(sortedStrings(m.gets), ","); got != "/b/2026/a.txt" { + t.Errorf("fetched = %q, want the marker skipped", got) + } +} + +// sortedStrings makes an assertion independent of completion order, which with +// more than one worker is not the listing order. +func sortedStrings(in []string) []string { + out := append([]string(nil), in...) + sort.Strings(out) + return out +} diff --git a/internal/cli/s3/transfer.go b/internal/cli/s3/transfer.go deleted file mode 100644 index aeb5932..0000000 --- a/internal/cli/s3/transfer.go +++ /dev/null @@ -1,207 +0,0 @@ -package s3cli - -import ( - "context" - "errors" - "fmt" - "io" - "mime" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - - "github.com/ftarasenko/go-openstackclient/internal/auth" - "github.com/ftarasenko/go-openstackclient/internal/output" - "github.com/ftarasenko/go-openstackclient/internal/s3" -) - -// downloadRequest is one "download" invocation: the object to fetch, where to -// put it, and the one flag that governs it. The positional arguments travel -// with the flag because runDownload needs all four and eight parameters is one -// too many (go:S107). -type downloadRequest struct { - bucket, key string - // dest is the destination path: "" means the key's basename, "-" means - // stream to stdout. - dest string - force bool -} - -const downloadLong = `Download an object to a file. - -With no FILE the object's key basename is used. FILE "-" streams the object to -stdout as raw bytes and prints nothing else, so it pipes. - -An existing file is never overwritten without --force: these are backups, and a -half-typed key that clobbers the local copy of one is not a recoverable -mistake. A transfer that fails partway removes the partial file rather than -leaving a truncated dump that looks complete.` - -func newDownloadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { - var force bool - cmd := &cobra.Command{ - Use: "download / [file]", - Short: "Download an object to a file", - Long: downloadLong, - Args: cobra.RangeArgs(1, 2), - RunE: func(cmd *cobra.Command, args []string) error { - if err := o.Validate(); err != nil { - return err - } - bucket, key, err := parseObjectRef(args[0]) - if err != nil { - return err - } - req := downloadRequest{bucket: bucket, key: key, force: force} - if len(args) == 2 { - req.dest = args[1] - } - ctx := cmd.Context() - client, err := f.client(ctx, a) - if err != nil { - return err - } - return runDownload(ctx, client, o, req, cmd.OutOrStdout()) - }, - } - cmd.Flags().BoolVar(&force, "force", false, "overwrite the destination file if it exists") - return cmd -} - -// runDownload is the test seam for "download". w receives either the object's -// raw bytes (dest "-") or the summary table. -func runDownload(ctx context.Context, client *s3.Client, o *output.Options, - r downloadRequest, w io.Writer) error { - if r.dest == "-" { - if _, err := client.GetObject(ctx, r.bucket, r.key, w); err != nil { - return downloadError(r.bucket, r.key, err) - } - return nil - } - dest := r.dest - if dest == "" { - dest = filepath.Base(r.key) - } - if info, err := os.Stat(dest); err == nil && info.IsDir() { - dest = filepath.Join(dest, filepath.Base(r.key)) - } - - n, err := downloadToFile(ctx, client, r.bucket, r.key, dest, r.force) - if err != nil { - return err - } - return o.WriteSingle(w, - []string{"Bucket", "Key", "File", "Size"}, - []any{r.bucket, r.key, dest, n}) -} - -// downloadToFile streams the object into dest, leaving nothing behind if it -// fails. -func downloadToFile(ctx context.Context, client *s3.Client, bucket, key, dest string, force bool) (n int64, err error) { - flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL - if force { - flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC - } - out, err := os.OpenFile(dest, flags, 0o600) //nolint:gosec // G304: operator-supplied output path - if err != nil { - if errors.Is(err, os.ErrExist) { - return 0, fmt.Errorf("%s already exists (pass --force to overwrite)", dest) - } - return 0, fmt.Errorf("creating %q: %w", dest, err) - } - defer func() { - closeErr := out.Close() - if err == nil && closeErr != nil { - err = fmt.Errorf("closing %q: %w", dest, closeErr) - } - if err != nil { - _ = os.Remove(dest) - } - }() - - if n, err = client.GetObject(ctx, bucket, key, out); err != nil { - return 0, downloadError(bucket, key, err) - } - return n, nil -} - -// downloadError turns a missing key into the message an operator can act on. -func downloadError(bucket, key string, err error) error { - if s3.IsNotFound(err) { - return fmt.Errorf("no object %q in bucket %q", key, bucket) - } - return fmt.Errorf("downloading %s/%s: %w", bucket, key, err) -} - -const uploadLong = `Upload a file to a bucket. - -With no key, or a key ending in "/", the file's basename is used. The upload is -a single signed PUT — there is no multipart support, so the server's own -single-part ceiling (5 GiB on Garage and on AWS) applies.` - -func newUploadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { - cmd := &cobra.Command{ - Use: "upload [/]", - Short: "Upload a file to a bucket", - Long: uploadLong, - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { - if err := o.Validate(); err != nil { - return err - } - bucket, key, err := parseRef(args[1]) - if err != nil { - return err - } - ctx := cmd.Context() - client, err := f.client(ctx, a) - if err != nil { - return err - } - return runUpload(ctx, client, o, args[0], bucket, key, cmd.OutOrStdout()) - }, - } - return cmd -} - -// runUpload is the test seam for "upload". -func runUpload(ctx context.Context, client *s3.Client, o *output.Options, - file, bucket, key string, w io.Writer) error { - if key == "" || strings.HasSuffix(key, "/") { - key += filepath.Base(file) - } - - src, err := os.Open(file) //nolint:gosec // G304: operator-supplied upload path - if err != nil { - return fmt.Errorf("opening %q: %w", file, err) - } - defer func() { _ = src.Close() }() - - info, err := src.Stat() - if err != nil { - return fmt.Errorf("stat %q: %w", file, err) - } - if info.IsDir() { - return fmt.Errorf("%q is a directory; upload takes a single file", file) - } - - obj, err := client.PutObject(ctx, bucket, key, src, info.Size(), contentTypeFor(file)) - if err != nil { - return fmt.Errorf("uploading %s to %s/%s: %w", file, bucket, key, err) - } - return o.WriteSingle(w, - []string{"Bucket", "Key", "File", "Size", "ETag"}, - []any{bucket, key, file, obj.Size, obj.ETag}) -} - -// contentTypeFor guesses a Content-Type from the file extension, falling back to -// the S3 default. Getting this right matters for the objects koc uploads next to -// a backup — a .sha256 sibling should read as text in a browser, not download. -func contentTypeFor(file string) string { - if ct := mime.TypeByExtension(filepath.Ext(file)); ct != "" { - return ct - } - return "application/octet-stream" -} diff --git a/internal/cli/s3/transfer_bulk_test.go b/internal/cli/s3/transfer_bulk_test.go new file mode 100644 index 0000000..731fd7f --- /dev/null +++ b/internal/cli/s3/transfer_bulk_test.go @@ -0,0 +1,477 @@ +package s3cli + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// listingHandler serves one page of the given and the bytes "data" +// for every object GET — enough to drive a recursive transfer. +func listingHandler(t *testing.T, contents string, seen *[]string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("list-type") { + _, _ = fmt.Fprint(w, `false`+contents+``) + return + } + if seen != nil { + *seen = append(*seen, r.URL.Path) + } + _, _ = w.Write([]byte("data")) + } +} + +// sortedLines makes an assertion independent of completion order, which with +// more than one worker is not the listing order. +func sortedLines(s string) []string { + lines := strings.Split(strings.TrimSuffix(s, "\n"), "\n") + sort.Strings(lines) + return lines +} + +// A recursive download mirrors each key's path below the prefix, so a restore +// reproduces the layout rather than a flat pile of basenames. +func TestRunDownloadRecursiveMirrorsKeyPaths(t *testing.T) { + client := newMockClient(t, listingHandler(t, + `2026/08/a.sql.gz4`+ + `2026/09/b.sql.gz4`, nil)) + + dir := t.TempDir() + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "2026/", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + for _, rel := range []string{"08/a.sql.gz", "09/b.sql.gz"} { + if _, err := os.Stat(filepath.Join(dir, rel)); err != nil { + t.Errorf("%s was not written: %v", rel, err) + } + } +} + +// --flatten puts every object straight in the destination. +func TestRunDownloadRecursiveFlatten(t *testing.T) { + client := newMockClient(t, listingHandler(t, + `2026/08/a.sql.gz4`, nil)) + + dir := t.TempDir() + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive, f.flatten = true, true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "2026/", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "a.sql.gz")); err != nil { + t.Errorf("the object was not flattened into the destination: %v", err) + } +} + +// A wildcard reference is its own recursion, and narrows what the prefix took. +func TestRunDownloadWildcard(t *testing.T) { + var fetched []string + client := newMockClient(t, listingHandler(t, + `2026/a.sql.gz4`+ + `2026/a.sha2564`, &fetched)) + + dir := t.TempDir() + var buf bytes.Buffer + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "2026/*.sql.gz", dest: dir, flags: testDownloadFlags()}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(fetched, ","), "/db-backups/2026/a.sql.gz"; got != want { + t.Errorf("fetched = %q, want only the pattern match", got) + } +} + +// In bulk mode an existing file is a resumption point, not a failure: a restore +// that was interrupted is finished by re-running the same command. +func TestRunDownloadRecursiveSkipsExistingFiles(t *testing.T) { + var fetched []string + client := newMockClient(t, listingHandler(t, + `a.sql.gz4`+ + `b.sql.gz4`, &fetched)) + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.sql.gz"), []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(fetched, ","), "/db-backups/b.sql.gz"; got != want { + t.Errorf("fetched = %q, want only the missing object", got) + } + if !strings.Contains(buf.String(), "Skipped (exists)") { + t.Errorf("output %q does not report the skip", buf.String()) + } + // The existing file must be untouched, not truncated. + if body, _ := os.ReadFile(filepath.Join(dir, "a.sql.gz")); string(body) != "old" { + t.Errorf("the existing file was overwritten: %q", body) + } +} + +// An object key is server-supplied data: "../../etc/x" is a legal S3 key, and a +// restore that wrote it where it says would let whoever can write to the bucket +// write anywhere koc can. +func TestRunDownloadRecursiveRefusesEscapingKeys(t *testing.T) { + client := newMockClient(t, listingHandler(t, + `../escaped.sh4`, nil)) + + dir := filepath.Join(t.TempDir(), "restore") + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "", dest: dir, flags: f}, &buf) + if err == nil { + t.Fatal("a key escaping the destination was accepted") + } + if !strings.Contains(err.Error(), "outside") { + t.Errorf("error %q does not say what was refused", err) + } + if _, statErr := os.Stat(filepath.Join(filepath.Dir(dir), "escaped.sh")); statErr == nil { + t.Fatal("the escaping key was written") + } +} + +// A "directory marker" — a zero-length key ending in "/" — must not become a +// file, or it would shadow the directory its siblings need. +func TestRunDownloadRecursiveSkipsDirectoryMarkers(t *testing.T) { + var fetched []string + client := newMockClient(t, listingHandler(t, + `2026/0`+ + `2026/a.sql.gz4`, &fetched)) + + dir := t.TempDir() + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "", dest: dir, flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(fetched, ","), "/db-backups/2026/a.sql.gz"; got != want { + t.Errorf("fetched = %q, want the marker skipped", got) + } +} + +func TestRunDownloadRecursiveNoMatches(t *testing.T) { + client := newMockClient(t, listingHandler(t, "", nil)) + + var buf bytes.Buffer + f := testDownloadFlags() + f.recursive = true + err := runDownload(context.Background(), client, valueOpts(), + downloadRequest{bucket: "db-backups", key: "gone/", dest: t.TempDir(), flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := buf.String(), "No objects under db-backups/gone/\n"; got != want { + t.Errorf("output = %q, want %q", got, want) + } +} + +// The combinations that cannot mean anything must be refused when the arguments +// are parsed, before a client is even built. +func TestNewDownloadRequestValidation(t *testing.T) { + for _, tc := range []struct { + name string + args []string + flags downloadFlags + want string + }{ + {"recursive to stdout", []string{"b/k", "-"}, downloadFlags{recursive: true}, "not stdout"}, + {"recursive with a version", []string{"b/k", "d"}, downloadFlags{recursive: true, versionID: "v"}, "--recursive"}, + {"no key", []string{"b"}, downloadFlags{}, "names no object"}, + {"filters without recursion", []string{"b/k"}, + downloadFlags{filter: keyFilter{include: []string{"*"}}}, "--recursive"}, + {"flatten without recursion", []string{"b/k"}, downloadFlags{flatten: true}, "--flatten"}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := tc.flags + _, err := newDownloadRequest(tc.args, &flags) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// A recursive upload mirrors the local tree under the destination prefix. +func TestRunUploadRecursive(t *testing.T) { + var puts []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + puts = append(puts, r.URL.Path) + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + root := t.TempDir() + mustWrite(t, filepath.Join(root, "a.sql.gz"), "one") + mustWrite(t, filepath.Join(root, "nested", "b.sql.gz"), "two") + + var buf bytes.Buffer + f := testUploadFlags() + f.recursive = true + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: root, bucket: "db-backups", key: "2026/", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + sort.Strings(puts) + want := "/db-backups/2026/a.sql.gz,/db-backups/2026/nested/b.sql.gz" + if got := strings.Join(puts, ","); got != want { + t.Errorf("uploaded = %q, want %q", got, want) + } +} + +// --include selects files by the key they would get, so a filter reads the same +// way as it does on a listing. +func TestRunUploadRecursiveFilters(t *testing.T) { + var puts []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + puts = append(puts, r.URL.Path) + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + root := t.TempDir() + mustWrite(t, filepath.Join(root, "a.sql.gz"), "one") + mustWrite(t, filepath.Join(root, "a.sha256"), "two") + + var buf bytes.Buffer + f := testUploadFlags() + f.recursive = true + f.filter = keyFilter{include: []string{"*.sql.gz"}} + if err := f.filter.compile(); err != nil { + t.Fatal(err) + } + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: root, bucket: "db-backups", key: "", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(puts, ","), "/db-backups/a.sql.gz"; got != want { + t.Errorf("uploaded = %q, want %q", got, want) + } +} + +// --no-clobber costs one HEAD and leaves an object that is already there alone, +// which is how an interrupted bulk upload resumes. +func TestRunUploadNoClobberSkipsExisting(t *testing.T) { + var methods []string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method+" "+r.URL.Path) + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "3") + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + src := filepath.Join(t.TempDir(), "dump.sql.gz") + mustWrite(t, src, "payload") + + var buf bytes.Buffer + f := testUploadFlags() + f.noClobber = true + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "dump.sql.gz", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got, want := strings.Join(methods, ","), "HEAD /db-backups/dump.sql.gz"; got != want { + t.Errorf("requests = %q, want only the existence check", got) + } + if !strings.Contains(buf.String(), "Skipped (exists)") { + t.Errorf("output %q does not report the skip", buf.String()) + } +} + +// Standard input has no length, so it must take the multipart path — and it +// needs a full key, since a stream has no basename to fall back on. +func TestRunUploadStdinNeedsAFullKey(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("no request should be made without a key") + w.WriteHeader(http.StatusOK) + }) + + var buf bytes.Buffer + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: "-", bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf) + if err == nil || !strings.Contains(err.Error(), "full key") { + t.Errorf("err = %v, want it to ask for a full key", err) + } +} + +func TestUploadFlagValidation(t *testing.T) { + for _, tc := range []struct { + name string + req uploadRequest + flags uploadFlags + want string + }{ + {"stdin with recursive", uploadRequest{src: "-"}, uploadFlags{recursive: true}, "--recursive"}, + {"flatten without recursion", uploadRequest{src: "x"}, uploadFlags{flatten: true}, "--flatten"}, + {"filters without recursion", uploadRequest{src: "x"}, + uploadFlags{filter: keyFilter{exclude: []string{"*"}}}, "--recursive"}, + } { + t.Run(tc.name, func(t *testing.T) { + flags := tc.flags + if err := flags.validate(tc.req); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// An object past the part size has to go out as a multipart upload, or the +// 5 GiB single-PUT ceiling would apply. +func TestRunUploadUsesMultipartPastThePartSize(t *testing.T) { + var started bool + var parts int + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case q.Has("uploads"): + started = true + _, _ = fmt.Fprint(w, `u`) + case q.Has("partNumber"): + parts++ + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("ETag", fmt.Sprintf(`"p%s"`, q.Get("partNumber"))) + w.WriteHeader(http.StatusOK) + default: + _, _ = fmt.Fprint(w, `"final"`) + } + }) + + src := filepath.Join(t.TempDir(), "big.bin") + mustWrite(t, src, strings.Repeat("x", s3.MinPartSize+1024)) + + var buf bytes.Buffer + f := testUploadFlags() + f.partSizeMiB = s3.MinPartSize >> 20 // 5 MiB, the protocol minimum + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "big.bin", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if !started { + t.Error("an object past the part size was sent as a single PUT") + } + if parts != 2 { + t.Errorf("parts = %d, want 2", parts) + } + if !strings.Contains(buf.String(), "final") { + t.Errorf("output %q does not carry the completed ETag", buf.String()) + } +} + +// --content-type overrides the extension guess for the whole run. +func TestRunUploadContentTypeOverride(t *testing.T) { + var gotType string + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + gotType = r.Header.Get("Content-Type") + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + }) + + src := filepath.Join(t.TempDir(), "dump.txt") + mustWrite(t, src, "payload") + + var buf bytes.Buffer + f := testUploadFlags() + f.contentType = "application/x-custom" + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "b", key: "k", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if gotType != "application/x-custom" { + t.Errorf("Content-Type = %q, want the override", gotType) + } +} + +// --dry-run must name every transfer it would make and make none. +func TestRunUploadRecursiveDryRun(t *testing.T) { + client := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("--dry-run sent %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + }) + + root := t.TempDir() + mustWrite(t, filepath.Join(root, "a.sql.gz"), "one") + + var buf bytes.Buffer + f := testUploadFlags() + f.recursive, f.dryRun = true, true + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: root, bucket: "db-backups", key: "p/", flags: f}, &buf) + if err != nil { + t.Fatal(err) + } + if got := sortedLines(buf.String()); len(got) != 1 || !strings.Contains(got[0], "Would upload") { + t.Errorf("output = %q, want one 'Would upload' line", buf.String()) + } +} + +func TestKeyForFile(t *testing.T) { + root := filepath.Join("tmp", "src") + for _, tc := range []struct { + name, prefix, path string + flatten bool + want string + }{ + {"mirrors the tree", "2026/", filepath.Join(root, "a", "b.gz"), false, "2026/a/b.gz"}, + {"no prefix", "", filepath.Join(root, "a", "b.gz"), false, "a/b.gz"}, + {"flattened", "2026/", filepath.Join(root, "a", "b.gz"), true, "2026/b.gz"}, + {"prefix without a slash", "2026", filepath.Join(root, "b.gz"), false, "2026/b.gz"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := keyForFile(tc.prefix, root, tc.path, tc.flatten) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("keyForFile = %q, want %q", got, tc.want) + } + }) + } +} + +func mustWrite(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/internal/cli/s3/transfer_paths_test.go b/internal/cli/s3/transfer_paths_test.go index ad0b4c3..1ebeaf5 100644 --- a/internal/cli/s3/transfer_paths_test.go +++ b/internal/cli/s3/transfer_paths_test.go @@ -23,7 +23,7 @@ func TestRunDownload_DefaultsToTheKeyBasename(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "nested/e2e-mariadb.sql.gz"}, &buf) + downloadRequest{bucket: "db-backups", key: "nested/e2e-mariadb.sql.gz", flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestRunDownload_DirectoryDestination(t *testing.T) { dir := t.TempDir() var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dir}, &buf) + downloadRequest{bucket: "db-backups", key: "e2e-mariadb.sql.gz", dest: dir, flags: &downloadFlags{}}, &buf) if err != nil { t.Fatal(err) } @@ -65,7 +65,7 @@ func TestRunDownload_UncreatableDestination(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(file, "dump")}, &buf) + downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(file, "dump"), flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("an uncreatable destination was reported as success") } @@ -82,7 +82,7 @@ func TestRunDownload_StdoutMissingKey(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "missing", dest: "-"}, &buf) + downloadRequest{bucket: "db-backups", key: "missing", dest: "-", flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("a 404 streaming to stdout was reported as success") } @@ -101,7 +101,7 @@ func TestRunDownload_NonNotFoundErrorIsWrapped(t *testing.T) { var buf bytes.Buffer err := runDownload(context.Background(), client, valueOpts(), - downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(t.TempDir(), "dump")}, &buf) + downloadRequest{bucket: "db-backups", key: "key", dest: filepath.Join(t.TempDir(), "dump"), flags: &downloadFlags{}}, &buf) if err == nil { t.Fatal("a 403 was reported as success") } @@ -120,7 +120,7 @@ func TestRunUpload_MissingSourceFile(t *testing.T) { var buf bytes.Buffer err := runUpload(context.Background(), client, valueOpts(), - filepath.Join(t.TempDir(), "absent"), "db-backups", "key", &buf) + uploadRequest{src: filepath.Join(t.TempDir(), "absent"), bucket: "db-backups", key: "key", flags: testUploadFlags()}, &buf) if err == nil { t.Fatal("a missing source file was reported as success") } @@ -139,7 +139,8 @@ func TestRunUpload_RejectsADirectory(t *testing.T) { }) var buf bytes.Buffer - err := runUpload(context.Background(), client, valueOpts(), t.TempDir(), "db-backups", "", &buf) + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: t.TempDir(), bucket: "db-backups", key: "", flags: testUploadFlags()}, &buf) if err == nil { t.Fatal("a directory was accepted as an upload source") } @@ -166,7 +167,8 @@ func TestRunUpload_PrefixKeyAppendsBasename(t *testing.T) { } var buf bytes.Buffer - if err := runUpload(context.Background(), client, valueOpts(), src, "db-backups", "daily/", &buf); err != nil { + if err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "daily/", flags: testUploadFlags()}, &buf); err != nil { t.Fatal(err) } if gotPath != "/db-backups/daily/dump.sql.gz" { @@ -186,7 +188,8 @@ func TestRunUpload_APIErrorIsWrapped(t *testing.T) { } var buf bytes.Buffer - err := runUpload(context.Background(), client, valueOpts(), src, "db-backups", "key", &buf) + err := runUpload(context.Background(), client, valueOpts(), + uploadRequest{src: src, bucket: "db-backups", key: "key", flags: testUploadFlags()}, &buf) if err == nil { t.Fatal("a 403 was reported as success") } diff --git a/internal/cli/s3/upload.go b/internal/cli/s3/upload.go new file mode 100644 index 0000000..5771fa8 --- /dev/null +++ b/internal/cli/s3/upload.go @@ -0,0 +1,370 @@ +package s3cli + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/ftarasenko/go-openstackclient/internal/auth" + "github.com/ftarasenko/go-openstackclient/internal/output" + "github.com/ftarasenko/go-openstackclient/internal/s3" +) + +// uploadFlags holds the options accepted by "upload". +type uploadFlags struct { + recursive bool + dryRun bool + flatten bool + noClobber bool + contentType string + partSizeMiB int + concurrency int + filter keyFilter +} + +// uploadRequest is one "upload" invocation. +type uploadRequest struct { + // src is a file, a directory (with --recursive), or "-" for stdin. + src string + bucket string + key string + flags *uploadFlags +} + +const uploadLong = `Upload a file, a directory, or standard input. + +With no key, or a key ending in "/", the file's basename is used. + +Objects larger than the part size go out as a multipart upload, so there is no +5 GiB single-PUT ceiling: a dump of any size uploads, in --part-size chunks, +--concurrency of them at a time. An upload that fails is aborted, so a +half-written object never becomes visible. + +SRC "-" reads standard input, which is what makes a dump streamable without +staging it on disk first: + + mysqldump ... | gzip | koc s3 upload - db-backups/dump-$(date +%F).sql.gz + +A stream's length is not known ahead of time, so it is always multipart; memory +stays at --part-size × --concurrency however long the stream turns out to be. + +--recursive uploads a directory tree, mirroring each file's path under the key +as a prefix; --flatten puts every file at the top of that prefix instead. +--include/--exclude select which files are taken, matched against the *key* the +file would get. --no-clobber skips a key that already exists, so an interrupted +bulk upload resumes by re-running the same command. + +Content types are guessed from the file extension, which is what makes a +".sha256" sibling read as text in a browser rather than download; --content-type +overrides that for every object in the run. Standard input has no extension, so +it gets the S3 default unless --content-type says otherwise.` + +const uploadExample = ` # One file + koc s3 upload ./dump.sql.gz db-backups/ + + # Straight from a pipe, no staging on disk + mysqldump --all-databases | gzip | koc s3 upload - db-backups/nightly.sql.gz + + # A tree, skipping what is already there + koc s3 upload ./restore db-backups/2026/ --recursive --no-clobber + + # Bigger parts for a fat link + koc s3 upload ./huge.tar db-backups/ --part-size 128 --concurrency 8` + +func newUploadCommand(a *auth.Options, o *output.Options, f *connFlags) *cobra.Command { + uf := &uploadFlags{} + cmd := &cobra.Command{ + Use: "upload [/]", + Short: "Upload a file, a directory, or standard input", + Long: uploadLong, + Example: uploadExample, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + bucket, key, err := parseRef(args[1]) + if err != nil { + return err + } + req := uploadRequest{src: args[0], bucket: bucket, key: key, flags: uf} + if err := uf.validate(req); err != nil { + return err + } + ctx := cmd.Context() + client, err := f.client(ctx, a) + if err != nil { + return err + } + return runUpload(ctx, client, o, req, cmd.OutOrStdout()) + }, + } + fl := cmd.Flags() + fl.BoolVarP(&uf.recursive, "recursive", "r", false, "upload a directory tree") + fl.BoolVar(&uf.dryRun, "dry-run", false, "print what would be uploaded and transfer nothing") + fl.BoolVar(&uf.flatten, "flatten", false, + "put every file at the top of the destination prefix instead of mirroring its path") + fl.BoolVar(&uf.noClobber, "no-clobber", false, "skip a key that already exists") + fl.StringVar(&uf.contentType, "content-type", "", "Content-Type for every object in this run") + fl.IntVar(&uf.partSizeMiB, "part-size", s3.DefaultPartSize>>20, + "multipart part size in MiB (minimum 5)") + fl.IntVar(&uf.concurrency, "concurrency", s3.DefaultConcurrency, + "parts — or, under --recursive, files — to upload at once") + uf.filter.addTo(fl) + return cmd +} + +// validate rejects flag combinations that cannot mean anything. +func (f *uploadFlags) validate(r uploadRequest) error { + switch { + case f.recursive && r.src == "-": + return errors.New("standard input is one object; it cannot be combined with --recursive") + case !f.recursive && f.flatten: + return errors.New("--flatten only applies to a recursive upload") + case !f.recursive && f.filter.active(): + return errors.New("--include/--exclude select files from a tree; pass --recursive") + } + return f.filter.compile() +} + +// uploadOptions renders the flags into the client's options. +func (f *uploadFlags) uploadOptions(size int64, contentType string) s3.UploadOptions { + return s3.UploadOptions{ + ContentType: contentType, + PartSize: int64(f.partSizeMiB) << 20, + Concurrency: f.concurrency, + Size: size, + } +} + +// runUpload is the test seam for "upload". +func runUpload(ctx context.Context, client *s3.Client, o *output.Options, + r uploadRequest, w io.Writer) error { + switch { + case r.flags.recursive: + return runUploadRecursive(ctx, client, r, w) + case r.src == "-": + return runUploadStdin(ctx, client, o, r, w) + default: + return runUploadFile(ctx, client, o, r, w) + } +} + +// runUploadStdin uploads standard input under the key given, which must be a +// full key: a stream has no basename to fall back on. +func runUploadStdin(ctx context.Context, client *s3.Client, o *output.Options, + r uploadRequest, w io.Writer) error { + if r.key == "" || strings.HasSuffix(r.key, "/") { + return fmt.Errorf("uploading standard input needs a full key, got %q", r.bucket+"/"+r.key) + } + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would upload standard input to %s/%s\n", r.bucket, r.key) + return err + } + if skip, err := checkClobber(ctx, client, r.bucket, r.key, r.flags, w); skip || err != nil { + return err + } + + // Size -1 says "unknown", which forces the multipart path: a pipe cannot be + // rewound, and a single signed PUT has to hash its body before sending it. + opts := r.flags.uploadOptions(-1, r.flags.contentType) + obj, err := client.PutObjectStream(ctx, r.bucket, r.key, os.Stdin, opts) + if err != nil { + return fmt.Errorf("uploading standard input to %s/%s: %w", r.bucket, r.key, err) + } + return o.WriteSingle(w, + []string{"Bucket", "Key", "File", "Size", "ETag"}, + []any{r.bucket, r.key, "-", obj.Size, obj.ETag}) +} + +// runUploadFile uploads one local file. +func runUploadFile(ctx context.Context, client *s3.Client, o *output.Options, + r uploadRequest, w io.Writer) error { + key := r.key + if key == "" || strings.HasSuffix(key, "/") { + key += filepath.Base(r.src) + } + + src, err := os.Open(r.src) + if err != nil { + return fmt.Errorf("opening %q: %w", r.src, err) + } + defer func() { _ = src.Close() }() + + info, err := src.Stat() + if err != nil { + return fmt.Errorf("stat %q: %w", r.src, err) + } + if info.IsDir() { + return fmt.Errorf("%q is a directory; pass --recursive to upload a tree", r.src) + } + if r.flags.dryRun { + _, err := fmt.Fprintf(w, "Would upload %s to %s/%s\n", r.src, r.bucket, key) + return err + } + if skip, err := checkClobber(ctx, client, r.bucket, key, r.flags, w); skip || err != nil { + return err + } + + opts := r.flags.uploadOptions(info.Size(), r.contentTypeFor(r.src)) + obj, err := client.PutObjectStream(ctx, r.bucket, key, src, opts) + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", r.src, r.bucket, key, err) + } + return o.WriteSingle(w, + []string{"Bucket", "Key", "File", "Size", "ETag"}, + []any{r.bucket, key, r.src, obj.Size, obj.ETag}) +} + +// runUploadRecursive uploads every file under a directory. +func runUploadRecursive(ctx context.Context, client *s3.Client, r uploadRequest, w io.Writer) error { + root, err := os.Stat(r.src) + if err != nil { + return fmt.Errorf("reading %q: %w", r.src, err) + } + if !root.IsDir() { + return fmt.Errorf("%q is not a directory; drop --recursive to upload one file", r.src) + } + + out := &syncWriter{w: w} + // Under --recursive the workers are files, and each file's own multipart + // then goes out a part at a time: the product of the two would put + // concurrency² parts in flight and that many part buffers in memory. + fileFlags := *r.flags + fileFlags.concurrency = 1 + p, workCtx := newPool(ctx, r.flags.concurrency) + + taken := 0 + err = filepath.WalkDir(r.src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !d.Type().IsRegular() { + // Symlinks and devices are skipped rather than followed: a tree + // walked from a backup directory should not chase a link out of it. + return nil + } + key, err := keyForFile(r.key, r.src, path, r.flags.flatten) + if err != nil { + return err + } + if !r.flags.filter.match(key) { + return nil + } + taken++ + if !p.run(workCtx, func() error { + return uploadOne(workCtx, client, path, r.bucket, key, &fileFlags, out) + }) { + return filepath.SkipAll + } + return nil + }) + if waitErr := p.wait(); err == nil { + err = waitErr + } + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", r.src, r.bucket, r.key, err) + } + if taken == 0 { + _, err = fmt.Fprintf(w, "No files under %s\n", r.src) + return err + } + return nil +} + +// uploadOne uploads one file of a recursive run. +func uploadOne(ctx context.Context, client *s3.Client, path, bucket, key string, + f *uploadFlags, w io.Writer) error { + if f.dryRun { + _, err := fmt.Fprintf(w, "Would upload %s to %s/%s\n", path, bucket, key) + return err + } + if skip, err := checkClobber(ctx, client, bucket, key, f, w); skip || err != nil { + return err + } + + src, err := os.Open(path) //nolint:gosec // G304: path from the operator-named tree + if err != nil { + return fmt.Errorf("opening %q: %w", path, err) + } + defer func() { _ = src.Close() }() + + info, err := src.Stat() + if err != nil { + return fmt.Errorf("stat %q: %w", path, err) + } + + contentType := f.contentType + if contentType == "" { + contentType = contentTypeFor(path) + } + obj, err := client.PutObjectStream(ctx, bucket, key, src, f.uploadOptions(info.Size(), contentType)) + if err != nil { + return fmt.Errorf("uploading %s to %s/%s: %w", path, bucket, key, err) + } + _, err = fmt.Fprintf(w, "Uploaded: %s -> %s/%s (%d bytes)\n", path, bucket, key, obj.Size) + return err +} + +// checkClobber implements --no-clobber: one HEAD before the transfer, and the +// object is left alone if it is already there. It reports whether the caller +// should skip this object. +func checkClobber(ctx context.Context, client *s3.Client, bucket, key string, + f *uploadFlags, w io.Writer) (skip bool, err error) { + if !f.noClobber { + return false, nil + } + if _, err := client.HeadObject(ctx, bucket, key, ""); err != nil { + if s3.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("checking %s/%s: %w", bucket, key, err) + } + _, err = fmt.Fprintf(w, "Skipped (exists): %s/%s\n", bucket, key) + return true, err +} + +// keyForFile maps one local path to its object key, mirroring the tree under +// the destination prefix unless flattened. +func keyForFile(prefix, root, path string, flatten bool) (string, error) { + rel := filepath.Base(path) + if !flatten { + var err error + if rel, err = filepath.Rel(root, path); err != nil { + return "", fmt.Errorf("resolving %q under %q: %w", path, root, err) + } + } + // Object keys are slash-separated whatever the local separator is, so a + // tree uploaded from Windows lands under the same keys as from Linux. + rel = filepath.ToSlash(rel) + + if prefix == "" { + return rel, nil + } + return strings.TrimSuffix(prefix, "/") + "/" + rel, nil +} + +// contentTypeFor guesses a Content-Type from the file extension, falling back to +// the S3 default. Getting this right matters for the objects koc uploads next to +// a backup — a .sha256 sibling should read as text in a browser, not download. +func contentTypeFor(file string) string { + if ct := mime.TypeByExtension(filepath.Ext(file)); ct != "" { + return ct + } + return "application/octet-stream" +} + +// contentTypeFor on the request prefers an explicit --content-type. +func (r uploadRequest) contentTypeFor(file string) string { + if r.flags.contentType != "" { + return r.flags.contentType + } + return contentTypeFor(file) +} diff --git a/internal/output/output.go b/internal/output/output.go index 3dfdd5c..3ab0696 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -968,3 +968,40 @@ func cellRaw(v any) string { return string(b) } } + +// byteUnits are the binary (IEC) multiples HumanBytes renders. Binary rather +// than decimal because that is what df, ls -h and every S3 console show, so a +// "14.2 GiB" from koc lines up with what an operator sees elsewhere. +var byteUnits = []string{"KiB", "MiB", "GiB", "TiB", "PiB", "EiB"} + +// HumanBytes renders a byte count for a reader: exact bytes below 1 KiB, then +// three significant figures with a binary unit. +// +// It is opt-in everywhere it appears (a --human flag), never the default: koc's +// tables are meant to survive a pipe into awk, and a rounded "14.2 GiB" is not +// a number a script can add up. +func HumanBytes(n int64) string { + if n < 0 { + return "-" + HumanBytes(-n) + } + if n < 1024 { + return strconv.FormatInt(n, 10) + " B" + } + + value, unit := float64(n)/1024, byteUnits[0] + for _, next := range byteUnits[1:] { + if value < 1024 { + break + } + value, unit = value/1024, next + } + + switch { + case value < 10: + return strconv.FormatFloat(value, 'f', 2, 64) + " " + unit + case value < 100: + return strconv.FormatFloat(value, 'f', 1, 64) + " " + unit + default: + return strconv.FormatFloat(value, 'f', 0, 64) + " " + unit + } +} diff --git a/internal/s3/copy.go b/internal/s3/copy.go new file mode 100644 index 0000000..0a7dec4 --- /dev/null +++ b/internal/s3/copy.go @@ -0,0 +1,82 @@ +package s3 + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" +) + +// CopyObject copies an object inside the store, without the bytes travelling +// through koc. The source is named in a header rather than a body, so a 100 GiB +// copy costs one small request. +// +// versionID selects a specific version of the source, or "" for the current one. +// contentType, when empty, is carried over from the source along with the rest +// of its metadata (S3's default COPY directive). +func (c *Client) CopyObject(ctx context.Context, src, dst ObjectRef, versionID, contentType string) (*ObjectInfo, error) { + // The copy source is a path, so its slashes must survive encoding while + // everything else in the key is escaped — the same rule as a request URI. + source := "/" + src.Bucket + "/" + strings.TrimPrefix(src.Key, "/") + if versionID != "" { + source += "?versionId=" + uriEncode(versionID, false) + } + + hdr := map[string]string{"x-amz-copy-source": uriEncode(source, true)} + if contentType != "" { + hdr["Content-Type"] = contentType + // Without this S3 keeps the source's metadata and ignores the header. + hdr["x-amz-metadata-directive"] = "REPLACE" + } + + var result struct { + ETag string `xml:"ETag"` + LastModified string `xml:"LastModified"` + } + req := request{ + method: http.MethodPut, + url: c.url(dst.Bucket, dst.Key, nil), + payloadHash: emptySHA256, + header: hdr, + } + // Like a multipart completion, a copy can fail inside a 200: the server + // holds the connection open while it copies and then writes an . + err := c.do(ctx, req, func(resp *http.Response) error { + payload, readErr := io.ReadAll(io.LimitReader(resp.Body, errorBodyLimit)) + if readErr != nil { + return fmt.Errorf("reading copy result: %w", readErr) + } + if code := errorCodeInBody(payload); code != "" { + return &APIError{ + StatusCode: resp.StatusCode, + Code: code, + Message: "copy was refused after the request was accepted", + Method: http.MethodPut, + Path: req.url.Path, + } + } + return xml.Unmarshal(payload, &result) + }) + if err != nil { + return nil, fmt.Errorf("copying %s to %s: %w", src, dst, err) + } + + return &ObjectInfo{ + Bucket: dst.Bucket, + Key: dst.Key, + ETag: strings.Trim(result.ETag, `"`), + LastModified: parseS3Time(result.LastModified), + ContentType: contentType, + }, nil +} + +// ObjectRef is a bucket/key pair. It exists so CopyObject's four string +// arguments cannot be transposed into a copy in the wrong direction. +type ObjectRef struct { + Bucket string + Key string +} + +func (r ObjectRef) String() string { return r.Bucket + "/" + r.Key } diff --git a/internal/s3/copy_delete_test.go b/internal/s3/copy_delete_test.go new file mode 100644 index 0000000..88a2061 --- /dev/null +++ b/internal/s3/copy_delete_test.go @@ -0,0 +1,206 @@ +package s3 + +import ( + "context" + "crypto/md5" + "encoding/base64" + "fmt" + "io" + "net/http" + "strings" + "testing" +) + +func TestCopyObject(t *testing.T) { + var gotMethod, gotPath, gotSource, gotDirective string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + gotMethod, gotPath = r.Method, r.URL.Path + gotSource = r.Header.Get("x-amz-copy-source") + gotDirective = r.Header.Get("x-amz-metadata-directive") + _, _ = fmt.Fprint(w, `"copied"`+ + `2026-09-14T10:00:00.000Z`) + }) + + src := ObjectRef{Bucket: "db-backups", Key: "nightly.sql.gz"} + dst := ObjectRef{Bucket: "archive", Key: "2026/nightly.sql.gz"} + obj, err := c.CopyObject(context.Background(), src, dst, "", "") + if err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut || gotPath != "/archive/2026/nightly.sql.gz" { + t.Errorf("request = %s %s, want PUT the destination", gotMethod, gotPath) + } + if gotSource != "/db-backups/nightly.sql.gz" { + t.Errorf("x-amz-copy-source = %q", gotSource) + } + // Without --content-type the source's metadata is carried over, which is + // S3's default and must not be overridden. + if gotDirective != "" { + t.Errorf("x-amz-metadata-directive = %q, want it unset", gotDirective) + } + if obj.ETag != "copied" { + t.Errorf("ETag = %q, want copied", obj.ETag) + } + if obj.LastModified.IsZero() { + t.Error("LastModified was not parsed") + } +} + +// A key with a space or a "+" has to reach the header percent-encoded, with its +// slashes intact — the copy source is a path, and the signature covers it. +func TestCopyObjectEncodesTheSource(t *testing.T) { + var gotSource string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotSource = r.Header.Get("x-amz-copy-source") + _, _ = fmt.Fprint(w, `"e"`) + }) + + src := ObjectRef{Bucket: "b", Key: "dir/a b+c.txt"} + if _, err := c.CopyObject(context.Background(), src, ObjectRef{Bucket: "b", Key: "d"}, "", ""); err != nil { + t.Fatal(err) + } + if gotSource != "/b/dir/a%20b%2Bc.txt" { + t.Errorf("x-amz-copy-source = %q, want the key encoded and the slashes kept", gotSource) + } +} + +// Replacing the content type needs the REPLACE directive, or S3 keeps the +// source's and ignores the header. +func TestCopyObjectReplacesMetadataForANewContentType(t *testing.T) { + var gotType, gotDirective string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotType, gotDirective = r.Header.Get("Content-Type"), r.Header.Get("x-amz-metadata-directive") + _, _ = fmt.Fprint(w, `"e"`) + }) + + _, err := c.CopyObject(context.Background(), + ObjectRef{Bucket: "b", Key: "k"}, ObjectRef{Bucket: "b", Key: "k2"}, "", "text/plain") + if err != nil { + t.Fatal(err) + } + if gotType != "text/plain" || gotDirective != "REPLACE" { + t.Errorf("Content-Type = %q, directive = %q; want text/plain and REPLACE", gotType, gotDirective) + } +} + +// A version ID travels in the copy-source header as a query string, not as a +// request parameter. +func TestCopyObjectCarriesTheSourceVersion(t *testing.T) { + var gotSource string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotSource = r.Header.Get("x-amz-copy-source") + _, _ = fmt.Fprint(w, `"e"`) + }) + + _, err := c.CopyObject(context.Background(), + ObjectRef{Bucket: "b", Key: "k"}, ObjectRef{Bucket: "b", Key: "k2"}, "v7", "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(gotSource, "versionId%3Dv7") && !strings.Contains(gotSource, "versionId=v7") { + t.Errorf("x-amz-copy-source = %q, want it to name version v7", gotSource) + } +} + +// Like a multipart completion, a copy can be refused inside a 200. +func TestCopyObjectDetectsAnErrorInsideA200(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, `SlowDowntry later`) + }) + + _, err := c.CopyObject(context.Background(), + ObjectRef{Bucket: "b", Key: "k"}, ObjectRef{Bucket: "b", Key: "k2"}, "", "") + if err == nil { + t.Fatal("an inside a 200 was reported as a successful copy") + } + if got := ErrorCode(err); got != "SlowDown" { + t.Errorf("ErrorCode = %q, want SlowDown", got) + } +} + +func TestDeleteObjectsBatch(t *testing.T) { + var gotMethod, gotBody, gotMD5, gotType string + var hasDeleteParam bool + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + body, _ := io.ReadAll(r.Body) + gotMethod, gotBody = r.Method, string(body) + gotMD5, gotType = r.Header.Get("Content-MD5"), r.Header.Get("Content-Type") + hasDeleteParam = r.URL.Query().Has("delete") + _, _ = fmt.Fprint(w, ``) + }) + + targets := []DeleteTarget{{Key: "a"}, {Key: "b", VersionID: "v1"}} + failures, err := c.DeleteObjects(context.Background(), "db-backups", targets) + if err != nil { + t.Fatal(err) + } + if len(failures) != 0 { + t.Errorf("failures = %v, want none", failures) + } + if gotMethod != http.MethodPost || !hasDeleteParam { + t.Errorf("request = %s (delete param: %v), want POST ?delete", gotMethod, hasDeleteParam) + } + if gotType != "application/xml" { + t.Errorf("Content-Type = %q", gotType) + } + // Quiet mode: only failures come back, which is all this call reports. + for _, want := range []string{"true", "a", "b", "v1"} { + if !strings.Contains(gotBody, want) { + t.Errorf("body %s missing %q", gotBody, want) + } + } + // AWS refuses a batch delete without the Content-MD5 of the body. + sum := md5.Sum([]byte(gotBody)) + if want := base64.StdEncoding.EncodeToString(sum[:]); gotMD5 != want { + t.Errorf("Content-MD5 = %q, want %q", gotMD5, want) + } +} + +// S3 reports per-key refusals inside a 200, so they are returned rather than +// raised: the rest of the batch did happen. +func TestDeleteObjectsReportsPerKeyFailures(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, ` + lockedAccessDeniedno + `) + }) + + failures, err := c.DeleteObjects(context.Background(), "b", []DeleteTarget{{Key: "ok"}, {Key: "locked"}}) + if err != nil { + t.Fatalf("a partial failure must not fail the call: %v", err) + } + if len(failures) != 1 || failures[0].Key != "locked" || failures[0].Code != "AccessDenied" { + t.Fatalf("failures = %+v, want the one refused key", failures) + } + if got := failures[0].Error(); !strings.Contains(got, "locked") || !strings.Contains(got, "AccessDenied") { + t.Errorf("failure message %q does not name the key and the code", got) + } +} + +func TestDeleteObjectsRejectsAnOversizedBatch(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an oversized batch must be refused before any request") + w.WriteHeader(http.StatusOK) + }) + + targets := make([]DeleteTarget, MaxDeleteBatch+1) + if _, err := c.DeleteObjects(context.Background(), "b", targets); err == nil { + t.Fatal("an oversized batch was accepted") + } +} + +// An empty batch is a no-op, not a malformed request. +func TestDeleteObjectsOnAnEmptyBatch(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an empty batch must make no request") + w.WriteHeader(http.StatusOK) + }) + + failures, err := c.DeleteObjects(context.Background(), "b", nil) + if err != nil || failures != nil { + t.Errorf("DeleteObjects(nil) = %v, %v; want no-op", failures, err) + } +} diff --git a/internal/s3/delete.go b/internal/s3/delete.go new file mode 100644 index 0000000..fbede31 --- /dev/null +++ b/internal/s3/delete.go @@ -0,0 +1,118 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" +) + +// MaxDeleteBatch is the protocol's limit on keys per DeleteObjects call. +const MaxDeleteBatch = 1000 + +// DeleteTarget is one key, optionally one specific version of it, for a batch +// delete. +type DeleteTarget struct { + Key string + VersionID string +} + +// DeleteError is one key a batch delete refused. S3 reports these inside a +// 200 response, so they are returned rather than raised: the rest of the batch +// did happen. +type DeleteError struct { + Key string + VersionID string + Code string + Message string +} + +func (f DeleteError) Error() string { + if f.Message != "" { + return fmt.Sprintf("%s: %s (%s)", f.Key, f.Message, f.Code) + } + return fmt.Sprintf("%s: %s", f.Key, f.Code) +} + +// DeleteObjects removes up to MaxDeleteBatch keys in one request, which is what +// makes emptying a bucket of a million objects finish: it is a thousandth of the +// requests one-DELETE-per-key costs. +// +// The returned slice names the keys the server refused. A nil error with a +// non-empty slice is the normal partial-failure case, not a contradiction — +// S3 answers 200 and itemises what it would not do. +func (c *Client) DeleteObjects(ctx context.Context, bucket string, targets []DeleteTarget) ([]DeleteError, error) { + if len(targets) == 0 { + return nil, nil + } + if len(targets) > MaxDeleteBatch { + return nil, fmt.Errorf("a batch delete takes at most %d keys, got %d", MaxDeleteBatch, len(targets)) + } + + // Field-for-field a DeleteTarget, so one converts to the other; the tags + // are what this copy exists for. + type object struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + } + doc := struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet"` + Objects []object `xml:"Object"` + }{ + // Quiet suppresses a success entry per key; only the failures come + // back, which is all this call reports. + Quiet: true, + Objects: make([]object, len(targets)), + } + for i, t := range targets { + doc.Objects[i] = object(t) + } + + body, err := xml.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("encoding batch delete: %w", err) + } + + var result struct { + Errors []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` + } + req := request{ + method: http.MethodPost, + url: c.url(bucket, "", url.Values{"delete": {""}}), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + header: map[string]string{ + "Content-Type": "application/xml", + // AWS refuses a batch delete without it; see contentMD5. + "Content-MD5": contentMD5(body), + }, + } + err = c.do(ctx, req, func(resp *http.Response) error { + payload, readErr := io.ReadAll(io.LimitReader(resp.Body, errorBodyLimit)) + if readErr != nil { + return fmt.Errorf("reading batch delete result: %w", readErr) + } + return xml.Unmarshal(payload, &result) + }) + if err != nil { + return nil, fmt.Errorf("deleting %d objects from %s: %w", len(targets), bucket, err) + } + + var failures []DeleteError + for _, e := range result.Errors { + failures = append(failures, DeleteError{ + Key: e.Key, VersionID: e.VersionID, Code: e.Code, Message: e.Message, + }) + } + return failures, nil +} diff --git a/internal/s3/errors_test.go b/internal/s3/errors_test.go index e135e16..2913b4e 100644 --- a/internal/s3/errors_test.go +++ b/internal/s3/errors_test.go @@ -76,7 +76,7 @@ func TestNewAPIErrorSynthesisesNoSuchKey(t *testing.T) { w.WriteHeader(http.StatusNotFound) }) - _, err := c.HeadObject(context.Background(), "db-backups", "missing") + _, err := c.HeadObject(context.Background(), "db-backups", "missing", "") if !IsNotFound(err) { t.Fatalf("a bodiless 404 was not recognised as not-found: %v", err) } @@ -189,7 +189,7 @@ func TestGetObjectCopyError(t *testing.T) { _, _ = w.Write([]byte("some bytes")) }) - _, err := c.GetObject(context.Background(), "db-backups", "key", failWriter{}) + _, err := c.GetObject(context.Background(), "db-backups", "key", "", failWriter{}) if err == nil { t.Fatal("a failing writer was reported as success") } @@ -225,7 +225,7 @@ func TestDoTransportError(t *testing.T) { t.Fatal(err) } - _, err = c.HeadObject(context.Background(), "db-backups", "key") + _, err = c.HeadObject(context.Background(), "db-backups", "key", "") if err == nil { t.Fatal("an unreachable endpoint was reported as success") } @@ -244,7 +244,7 @@ func TestHeadObjectUsesContentLengthHeader(t *testing.T) { w.WriteHeader(http.StatusOK) }) - info, err := c.HeadObject(context.Background(), "db-backups", "key") + info, err := c.HeadObject(context.Background(), "db-backups", "key", "") if err != nil { t.Fatal(err) } diff --git a/internal/s3/integrity.go b/internal/s3/integrity.go new file mode 100644 index 0000000..a40139e --- /dev/null +++ b/internal/s3/integrity.go @@ -0,0 +1,20 @@ +package s3 + +import ( + "crypto/md5" //nolint:gosec // G501: not a security choice — see contentMD5 + "encoding/base64" +) + +// contentMD5 renders the Content-MD5 header S3 requires on the request bodies +// that carry a list of operations (DeleteObjects) and accepts on the small +// configuration PUTs (PutBucketVersioning). +// +// MD5 is not a security decision here and no alternative is available: the +// algorithm is fixed by the S3 protocol, the header exists so the server can +// reject a body corrupted in transit, and the request is already authenticated +// by a SHA-256 SigV4 signature over the same bytes. Omitting the header makes +// AWS refuse a batch delete outright. +func contentMD5(body []byte) string { + sum := md5.Sum(body) //nolint:gosec // G401: protocol-mandated integrity check, not authentication + return base64.StdEncoding.EncodeToString(sum[:]) +} diff --git a/internal/s3/lifecycle_test.go b/internal/s3/lifecycle_test.go new file mode 100644 index 0000000..1169df2 --- /dev/null +++ b/internal/s3/lifecycle_test.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "testing" +) + +func TestCreateBucketSendsLocationConstraint(t *testing.T) { + var gotMethod, gotPath, gotBody string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + body, _ := io.ReadAll(r.Body) + gotMethod, gotPath, gotBody = r.Method, r.URL.Path, string(body) + w.WriteHeader(http.StatusOK) + }) + + if err := client.CreateBucket(context.Background(), "scratch"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut { + t.Errorf("method = %q, want PUT", gotMethod) + } + if gotPath != "/scratch" { + t.Errorf("path = %q, want /scratch", gotPath) + } + // The signing region is the location constraint: a request signed for one + // region is not served by another, so there is nothing else it could be. + want := "garage" + if gotBody != want { + t.Errorf("body = %q, want %q", gotBody, want) + } +} + +// us-east-1 is the one region AWS rejects a location constraint for, so the +// body must be absent rather than naming it. +func TestCreateBucketOmitsBodyForUSEast1(t *testing.T) { + var gotBody string + var gotLen int64 + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + gotBody, gotLen = string(body), r.ContentLength + w.WriteHeader(http.StatusOK) + }) + client.cfg.Region = usEast1 + + if err := client.CreateBucket(context.Background(), "scratch"); err != nil { + t.Fatal(err) + } + if gotBody != "" { + t.Errorf("body = %q, want empty", gotBody) + } + if gotLen > 0 { + t.Errorf("Content-Length = %d, want 0", gotLen) + } +} + +func TestCreateBucketAlreadyOwnedReportsCode(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprint(w, `BucketAlreadyOwnedByYou`+ + `Bucket already exists`) + }) + + err := client.CreateBucket(context.Background(), "scratch") + if err == nil { + t.Fatal("expected an error") + } + if got := ErrorCode(err); got != "BucketAlreadyOwnedByYou" { + t.Errorf("ErrorCode = %q, want BucketAlreadyOwnedByYou", got) + } +} + +func TestDeleteBucket(t *testing.T) { + var gotMethod, gotPath string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusNoContent) + }) + + if err := client.DeleteBucket(context.Background(), "scratch"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodDelete || gotPath != "/scratch" { + t.Errorf("request = %s %s, want DELETE /scratch", gotMethod, gotPath) + } +} + +func TestDeleteBucketNotEmptyReportsCode(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = fmt.Fprint(w, `BucketNotEmptynope`) + }) + + err := client.DeleteBucket(context.Background(), "scratch") + if got := ErrorCode(err); got != "BucketNotEmpty" { + t.Errorf("ErrorCode = %q, want BucketNotEmpty (err = %v)", got, err) + } +} + +func TestDeleteObject(t *testing.T) { + var gotMethod, gotPath string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusNoContent) + }) + + if err := client.DeleteObject(context.Background(), "db-backups", "dump.sql.gz", ""); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodDelete || gotPath != "/db-backups/dump.sql.gz" { + t.Errorf("request = %s %s, want DELETE /db-backups/dump.sql.gz", gotMethod, gotPath) + } +} + +// ErrorCode must not claim a code for an error that never came from S3. +func TestErrorCodeOnForeignError(t *testing.T) { + if got := ErrorCode(io.EOF); got != "" { + t.Errorf("ErrorCode(io.EOF) = %q, want empty", got) + } + if got := ErrorCode(nil); got != "" { + t.Errorf("ErrorCode(nil) = %q, want empty", got) + } +} + +// ListObjectsFunc must follow continuation tokens and stop the walk the moment +// the limit is reached, without asking for a page it cannot use. +func TestListObjectsFuncPagesAndCapsLimit(t *testing.T) { + var maxKeys []string + client := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + maxKeys = append(maxKeys, r.URL.Query().Get("max-keys")) + if r.URL.Query().Get("continuation-token") == "" { + _, _ = fmt.Fprint(w, `true + tok + a1 + b2 + `) + return + } + _, _ = fmt.Fprint(w, `false + c4 + `) + }) + + var keys []string + var sum int64 + err := client.ListObjectsFunc(context.Background(), "b", ListOptions{Limit: 3}, func(o Object) error { + keys = append(keys, o.Key) + sum += o.Size + return nil + }) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(keys, ","); got != "a,b,c" { + t.Errorf("keys = %q, want a,b,c", got) + } + if sum != 7 { + t.Errorf("sum = %d, want 7", sum) + } + // Page two may ask for at most the one key still wanted. + if len(maxKeys) != 2 { + t.Fatalf("requests = %d, want 2", len(maxKeys)) + } + if n, _ := strconv.Atoi(maxKeys[1]); n != 1 { + t.Errorf("second max-keys = %q, want 1", maxKeys[1]) + } +} + +// An error from the callback stops the walk and reaches the caller unwrapped, +// which is what lets a delete-as-you-list abort on the first failure. +func TestListObjectsFuncPropagatesCallbackError(t *testing.T) { + client := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + ab + `) + }) + + seen := 0 + err := client.ListObjectsFunc(context.Background(), "b", ListOptions{}, func(Object) error { + seen++ + return io.ErrUnexpectedEOF + }) + if err != io.ErrUnexpectedEOF { //nolint:errorlint // identity is the assertion + t.Errorf("err = %v, want io.ErrUnexpectedEOF", err) + } + if seen != 1 { + t.Errorf("callback ran %d times, want 1", seen) + } +} diff --git a/internal/s3/multipart.go b/internal/s3/multipart.go new file mode 100644 index 0000000..f9a9579 --- /dev/null +++ b/internal/s3/multipart.go @@ -0,0 +1,389 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" +) + +// Multipart upload. Two things force it, and PutObject can do neither: +// +// - the single-part ceiling. A signed PUT carries the whole object in one +// request body, and every S3 implementation caps that at 5 GiB, so a +// database dump past that size simply could not be uploaded. +// - an unseekable source. SigV4 signs a hash of the payload, so PutObject has +// to read its body twice and therefore demands an io.ReadSeeker. A pipe +// ("mysqldump | koc s3 upload -") cannot provide one. Multipart hashes each +// part separately, so the source only ever has to be read forwards. +const ( + // MinPartSize is the floor S3 puts on every part but the last. + MinPartSize = 5 << 20 + + // DefaultPartSize is the part size koc uses when none is given. Memory cost + // is PartSize × Concurrency, so this pairs with DefaultConcurrency for a + // 64 MiB working set — a deliberate choice for the air-gapped cluster nodes + // koc runs on, where s5cmd's 50 MiB × 5 would be a quarter of a gigabyte. + DefaultPartSize = 16 << 20 + + // DefaultConcurrency is how many parts are uploaded at once by default. + DefaultConcurrency = 4 + + // maxParts is the protocol's limit on parts per upload. With the default + // part size that caps one object at 160 GiB; --part-size raises it. + maxParts = 10000 +) + +// UploadOptions configures PutObjectStream. +type UploadOptions struct { + ContentType string + + // PartSize is the size of each part but the last (0 = DefaultPartSize). + // Values below MinPartSize are rejected rather than silently raised, so a + // "--part-size 1" does not quietly become something else. + PartSize int64 + + // Concurrency is how many parts travel at once (0 = DefaultConcurrency). + Concurrency int + + // Size is the source's length when it is known, or -1 when it is not (a + // pipe). It only selects the strategy: a known size at or below PartSize + // goes out as a single PUT, which is one request instead of three. + Size int64 +} + +// withDefaults validates the options and fills the zero values in. +func (o UploadOptions) withDefaults() (UploadOptions, error) { + if o.PartSize == 0 { + o.PartSize = DefaultPartSize + } + if o.PartSize < MinPartSize { + return o, fmt.Errorf("part size must be at least %d bytes (S3's minimum), got %d", MinPartSize, o.PartSize) + } + if o.Concurrency == 0 { + o.Concurrency = DefaultConcurrency + } + if o.Concurrency < 1 { + return o, fmt.Errorf("concurrency must be at least 1, got %d", o.Concurrency) + } + if o.Size == 0 { + // A zero-length object is legitimate and must not be mistaken for + // "unknown"; callers say unknown with a negative value. + o.Size = 0 + } + return o, nil +} + +// PutObjectStream uploads from r, choosing the single-PUT or multipart path by +// what opts.Size says about the source. +// +// A multipart upload that fails is aborted, so a half-written object never +// becomes visible and the parts already stored stop accruing cost. If the abort +// itself fails the returned error says so — the upload ID is then the operator's +// to clean up. +func (c *Client) PutObjectStream(ctx context.Context, bucket, key string, r io.Reader, + opts UploadOptions) (*ObjectInfo, error) { + opts, err := opts.withDefaults() + if err != nil { + return nil, err + } + + // A source whose length is known and fits one part is cheaper as a plain + // PUT, but only if it is seekable — SigV4 has to hash it before sending. + if seeker, ok := r.(io.ReadSeeker); ok && opts.Size >= 0 && opts.Size <= opts.PartSize { + return c.PutObject(ctx, bucket, key, seeker, opts.Size, opts.ContentType) + } + + uploadID, err := c.createMultipartUpload(ctx, bucket, key, opts.ContentType) + if err != nil { + return nil, err + } + + parts, err := c.uploadParts(ctx, bucket, key, uploadID, r, opts) + if err != nil { + if abortErr := c.abortMultipartUpload(ctx, bucket, key, uploadID); abortErr != nil { + return nil, fmt.Errorf("%w (and aborting upload %s failed: %w)", err, uploadID, abortErr) + } + return nil, err + } + return c.completeMultipartUpload(ctx, bucket, key, uploadID, parts, opts.ContentType) +} + +// completedPart is one finished part: S3 wants the number and the ETag back in +// the completion body, in ascending order. +type completedPart struct { + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + size int64 +} + +// createMultipartUpload opens an upload and returns its ID. +func (c *Client) createMultipartUpload(ctx context.Context, bucket, key, contentType string) (string, error) { + hdr := map[string]string{} + if contentType != "" { + hdr["Content-Type"] = contentType + } + + var result struct { + UploadID string `xml:"UploadId"` + } + req := request{ + method: http.MethodPost, + url: c.url(bucket, key, url.Values{"uploads": {""}}), + payloadHash: emptySHA256, + header: hdr, + } + err := c.do(ctx, req, func(resp *http.Response) error { + return xml.NewDecoder(resp.Body).Decode(&result) + }) + if err != nil { + return "", fmt.Errorf("starting multipart upload of %s/%s: %w", bucket, key, err) + } + if result.UploadID == "" { + return "", fmt.Errorf("starting multipart upload of %s/%s: server returned no upload ID", bucket, key) + } + return result.UploadID, nil +} + +// partUploader carries the state uploadParts shares with its workers. It is a +// struct rather than a pile of closure variables because the read loop, the +// worker body and the failure path all touch the same four fields, and a +// mutex-guarded field is easier to reason about when its lock is next to it. +type partUploader struct { + client *Client + bucket string + key string + upload string + + // slots bounds how many parts are in flight, and with them how many + // part-sized buffers are live: the working set stays PartSize × + // Concurrency however large the object is. + slots chan struct{} + wg sync.WaitGroup + cancel context.CancelFunc + + mu sync.Mutex + parts []completedPart + err error +} + +// fail records the first error and stops the other workers: their parts are +// about to be aborted anyway, and a failing upload should not keep pushing +// bytes at the server. +func (u *partUploader) fail(err error) { + u.mu.Lock() + defer u.mu.Unlock() + if u.err == nil { + u.err = err + u.cancel() + } +} + +// send uploads one part in the background, blocking first until a slot frees +// up. It reports false if the upload has already failed, which ends the read +// loop. +func (u *partUploader) send(ctx context.Context, number int, body []byte) bool { + select { + case u.slots <- struct{}{}: + case <-ctx.Done(): + u.fail(ctx.Err()) + return false + } + + u.wg.Add(1) + go func() { + defer u.wg.Done() + defer func() { <-u.slots }() + + etag, err := u.client.uploadPart(ctx, u.bucket, u.key, u.upload, number, body) + if err != nil { + u.fail(err) + return + } + u.mu.Lock() + u.parts = append(u.parts, completedPart{PartNumber: number, ETag: etag, size: int64(len(body))}) + u.mu.Unlock() + }() + return true +} + +// uploadParts reads r into part-sized buffers and uploads them with up to +// opts.Concurrency workers. +// +// Reading is sequential and single-threaded because the source may be a pipe +// and a pipe can only be read forwards; only the uploads overlap. +func (c *Client) uploadParts(ctx context.Context, bucket, key, uploadID string, + r io.Reader, opts UploadOptions) ([]completedPart, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + u := &partUploader{ + client: c, bucket: bucket, key: key, upload: uploadID, + slots: make(chan struct{}, opts.Concurrency), + cancel: cancel, + } + + for number := 1; ; number++ { + if number > maxParts { + u.fail(fmt.Errorf("object needs more than %d parts at a part size of %d bytes; raise --part-size", + maxParts, opts.PartSize)) + break + } + + buf := make([]byte, opts.PartSize) + n, readErr := io.ReadFull(r, buf) + if n > 0 && !u.send(ctx, number, buf[:n]) { + break + } + if readErr != nil { + // ErrUnexpectedEOF is the short final part, not a failure. + if !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { + u.fail(fmt.Errorf("reading upload source: %w", readErr)) + } + break + } + } + + u.wg.Wait() + if u.err != nil { + return nil, u.err + } + sort.Slice(u.parts, func(i, j int) bool { return u.parts[i].PartNumber < u.parts[j].PartNumber }) + return u.parts, nil +} + +// uploadPart sends one part and returns its ETag. +func (c *Client) uploadPart(ctx context.Context, bucket, key, uploadID string, + number int, body []byte) (string, error) { + q := url.Values{"partNumber": {strconv.Itoa(number)}, "uploadId": {uploadID}} + var etag string + req := request{ + method: http.MethodPut, + url: c.url(bucket, key, q), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + } + err := c.do(ctx, req, func(resp *http.Response) error { + etag = resp.Header.Get("ETag") + if etag == "" { + return fmt.Errorf("part %d: server returned no ETag", number) + } + return drainBody(resp) + }) + if err != nil { + return "", fmt.Errorf("uploading part %d of %s/%s: %w", number, bucket, key, err) + } + return etag, nil +} + +// completeMultipartUpload assembles the parts into the finished object. +func (c *Client) completeMultipartUpload(ctx context.Context, bucket, key, uploadID string, + parts []completedPart, contentType string) (*ObjectInfo, error) { + // An upload with no parts at all cannot be completed — S3 rejects an empty + // part list — so a zero-length source becomes a zero-length single PUT. + if len(parts) == 0 { + if err := c.abortMultipartUpload(ctx, bucket, key, uploadID); err != nil { + return nil, err + } + return c.PutObject(ctx, bucket, key, bytes.NewReader(nil), 0, contentType) + } + + body, err := xml.Marshal(struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []completedPart `xml:"Part"` + }{Parts: parts}) + if err != nil { + return nil, fmt.Errorf("encoding multipart completion: %w", err) + } + + var total int64 + for _, p := range parts { + total += p.size + } + + var result struct { + ETag string `xml:"ETag"` + } + req := request{ + method: http.MethodPost, + url: c.url(bucket, key, url.Values{"uploadId": {uploadID}}), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + header: map[string]string{"Content-Type": "application/xml"}, + } + // A completion can fail *inside* a 200 response: S3 keeps the connection + // open while it assembles the object and then writes an document. + // Decoding into a struct that only knows about ETag would swallow it. + err = c.do(ctx, req, func(resp *http.Response) error { + payload, readErr := io.ReadAll(io.LimitReader(resp.Body, errorBodyLimit)) + if readErr != nil { + return fmt.Errorf("reading multipart completion: %w", readErr) + } + if code := errorCodeInBody(payload); code != "" { + return &APIError{ + StatusCode: resp.StatusCode, + Code: code, + Message: "multipart completion was refused after the request was accepted", + Method: http.MethodPost, + Path: req.url.Path, + } + } + return xml.Unmarshal(payload, &result) + }) + if err != nil { + return nil, fmt.Errorf("completing multipart upload of %s/%s: %w", bucket, key, err) + } + + return &ObjectInfo{ + Bucket: bucket, + Key: key, + Size: total, + ETag: strings.Trim(result.ETag, `"`), + ContentType: contentType, + }, nil +} + +// abortMultipartUpload discards an upload and the parts already stored. +func (c *Client) abortMultipartUpload(ctx context.Context, bucket, key, uploadID string) error { + // The parent context may already be cancelled — that is the usual reason + // this is being called — so the abort gets a context of its own, or it + // would fail without being sent and leak the parts. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortTimeout) + defer cancel() + + req := request{ + method: http.MethodDelete, + url: c.url(bucket, key, url.Values{"uploadId": {uploadID}}), + payloadHash: emptySHA256, + } + if err := c.do(ctx, req, drainBody); err != nil { + return fmt.Errorf("aborting multipart upload of %s/%s: %w", bucket, key, err) + } + return nil +} + +// errorCodeInBody reports the of an S3 document, or "" if the +// payload is not one. +func errorCodeInBody(payload []byte) string { + var doc struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + } + if xml.Unmarshal(payload, &doc) != nil { + return "" + } + if doc.Code == "" && doc.XMLName.Local == "Error" { + return "InternalError" + } + return doc.Code +} diff --git a/internal/s3/multipart_test.go b/internal/s3/multipart_test.go new file mode 100644 index 0000000..22c71d0 --- /dev/null +++ b/internal/s3/multipart_test.go @@ -0,0 +1,275 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "testing" +) + +// multipartRecorder is a mock S3 that speaks the three-call multipart protocol +// and records what it was sent. +type multipartRecorder struct { + mu sync.Mutex + + created int + aborted int + complete string // the completion body + parts map[int]string + failPart int // a part number to refuse, 0 for none +} + +func newMultipartRecorder() *multipartRecorder { + return &multipartRecorder{parts: map[int]string{}} +} + +func (m *multipartRecorder) handler(t *testing.T) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + body, _ := io.ReadAll(r.Body) + + m.mu.Lock() + defer m.mu.Unlock() + switch { + case r.Method == http.MethodPost && q.Has("uploads"): + m.created++ + _, _ = fmt.Fprint(w, `up-1`) + case r.Method == http.MethodPut && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + if n == m.failPart { + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `AccessDenied`) + return + } + m.parts[n] = string(body) + w.Header().Set("ETag", fmt.Sprintf(`"etag-%d"`, n)) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && q.Has("uploadId"): + m.complete = string(body) + _, _ = fmt.Fprint(w, `"final-etag"`) + case r.Method == http.MethodDelete && q.Has("uploadId"): + m.aborted++ + w.WriteHeader(http.StatusNoContent) + default: + // A plain PUT: the single-part path. + m.parts[0] = string(body) + w.Header().Set("ETag", `"single"`) + w.WriteHeader(http.StatusOK) + } + } +} + +// unseekable hides the io.Seeker a bytes.Reader would otherwise offer, which is +// what a pipe looks like to the client. +type unseekable struct{ r io.Reader } + +func (u unseekable) Read(p []byte) (int, error) { return u.r.Read(p) } + +// A source whose length is unknown — stdin — must take the multipart path even +// when it is tiny, because a single signed PUT has to hash its body first and a +// pipe cannot be rewound. +func TestPutObjectStreamUsesMultipartForAnUnknownSize(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + obj, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader("small")}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if rec.created != 1 { + t.Errorf("multipart uploads created = %d, want 1", rec.created) + } + if got := rec.parts[1]; got != "small" { + t.Errorf("part 1 = %q, want small", got) + } + if obj.ETag != "final-etag" { + t.Errorf("ETag = %q, want final-etag", obj.ETag) + } + if obj.Size != 5 { + t.Errorf("Size = %d, want 5", obj.Size) + } +} + +// A known, small, seekable source is one request instead of three. +func TestPutObjectStreamUsesASinglePutWhenItFits(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + obj, err := c.PutObjectStream(context.Background(), "b", "k", + bytes.NewReader([]byte("payload")), + UploadOptions{Size: 7, PartSize: MinPartSize, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if rec.created != 0 { + t.Errorf("a multipart upload was started for a %d-byte object", 7) + } + if obj.ETag != "single" { + t.Errorf("ETag = %q, want single", obj.ETag) + } +} + +// Several parts must arrive whole, in the right pieces, and be listed in +// ascending order in the completion body — S3 rejects any other order. +func TestPutObjectStreamSplitsAndOrdersParts(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + // Three parts: two full and a short last one. + payload := strings.Repeat("a", MinPartSize) + strings.Repeat("b", MinPartSize) + "tail" + obj, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader(payload)}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 3}) + if err != nil { + t.Fatal(err) + } + if got := len(rec.parts); got != 3 { + t.Fatalf("parts = %d, want 3", got) + } + if len(rec.parts[1]) != MinPartSize || len(rec.parts[2]) != MinPartSize || rec.parts[3] != "tail" { + t.Errorf("part sizes = %d/%d/%q, want two full parts and a short tail", + len(rec.parts[1]), len(rec.parts[2]), rec.parts[3]) + } + if obj.Size != int64(len(payload)) { + t.Errorf("Size = %d, want %d", obj.Size, len(payload)) + } + // Reassembled, the parts are the original stream. + var nums []int + for n := range rec.parts { + nums = append(nums, n) + } + sort.Ints(nums) + var joined strings.Builder + for _, n := range nums { + joined.WriteString(rec.parts[n]) + } + if joined.String() != payload { + t.Error("the parts do not reassemble into the source stream") + } + // The completion body must name the parts in ascending order. + if !strings.Contains(rec.complete, "1") || + strings.Index(rec.complete, "1") > + strings.Index(rec.complete, "3") { + t.Errorf("completion body is not in ascending part order:\n%s", rec.complete) + } +} + +// A failed part must abort the upload: a half-written object must never become +// visible, and the stored parts must stop costing money. +func TestPutObjectStreamAbortsOnFailure(t *testing.T) { + rec := newMultipartRecorder() + rec.failPart = 2 + c := newTestClient(t, rec.handler(t)) + + payload := strings.Repeat("a", MinPartSize) + strings.Repeat("b", MinPartSize) + _, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader(payload)}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err == nil { + t.Fatal("a refused part was reported as success") + } + if !strings.Contains(err.Error(), "part 2") { + t.Errorf("error %q does not name the failing part", err) + } + if rec.aborted != 1 { + t.Errorf("aborts = %d, want 1 — the parts already stored would otherwise leak", rec.aborted) + } + if rec.complete != "" { + t.Error("the upload was completed despite a failed part") + } +} + +// S3 can refuse a completion inside a 200 response: it holds the connection +// open while it assembles the object, then writes an document. Decoding +// only the fields we want would swallow it and report a successful upload. +func TestPutObjectStreamDetectsAnErrorInsideA200(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case q.Has("uploads"): + _, _ = fmt.Fprint(w, `up-1`) + case q.Has("partNumber"): + w.Header().Set("ETag", `"e"`) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, `InternalErrorassembly failed`) + } + }) + + _, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader("payload")}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err == nil { + t.Fatal("an inside a 200 was reported as a successful upload") + } + if got := ErrorCode(err); got != "InternalError" { + t.Errorf("ErrorCode = %q, want InternalError", got) + } +} + +// A zero-length stream cannot be completed as a multipart upload — S3 rejects +// an empty part list — so it has to fall back to a plain PUT. +func TestPutObjectStreamHandlesAnEmptyStream(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + obj, err := c.PutObjectStream(context.Background(), "b", "k", + unseekable{strings.NewReader("")}, + UploadOptions{Size: -1, PartSize: MinPartSize, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if obj.Size != 0 { + t.Errorf("Size = %d, want 0", obj.Size) + } + if rec.aborted != 1 { + t.Errorf("aborts = %d, want the empty upload discarded", rec.aborted) + } + if rec.complete != "" { + t.Error("an empty part list was sent to CompleteMultipartUpload") + } +} + +// A part size the protocol forbids must be refused before anything is uploaded, +// not silently raised to the minimum. +func TestUploadOptionsRejectATooSmallPartSize(t *testing.T) { + rec := newMultipartRecorder() + c := newTestClient(t, rec.handler(t)) + + _, err := c.PutObjectStream(context.Background(), "b", "k", strings.NewReader("x"), + UploadOptions{Size: -1, PartSize: 1024}) + if err == nil || !strings.Contains(err.Error(), "at least") { + t.Errorf("err = %v, want it to name S3's minimum part size", err) + } + if rec.created != 0 { + t.Error("an upload was started despite invalid options") + } +} + +func TestUploadOptionsDefaults(t *testing.T) { + got, err := UploadOptions{Size: -1}.withDefaults() + if err != nil { + t.Fatal(err) + } + if got.PartSize != DefaultPartSize { + t.Errorf("PartSize = %d, want %d", got.PartSize, DefaultPartSize) + } + if got.Concurrency != DefaultConcurrency { + t.Errorf("Concurrency = %d, want %d", got.Concurrency, DefaultConcurrency) + } + if _, err := (UploadOptions{Concurrency: -1}).withDefaults(); err == nil { + t.Error("a negative concurrency was accepted") + } +} diff --git a/internal/s3/presign.go b/internal/s3/presign.go new file mode 100644 index 0000000..201d416 --- /dev/null +++ b/internal/s3/presign.go @@ -0,0 +1,97 @@ +package s3 + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// Presigned URLs. The credential moves out of the Authorization header and into +// the query string, which makes the whole request a URL anyone can fetch until +// it expires — the way to hand a colleague one backup without handing them a +// key. No request is made: this is pure local computation, so it works offline +// and leaves no trace on the store. +const ( + // MaxPresignExpiry is SigV4's ceiling on a presigned URL's lifetime. + MaxPresignExpiry = 7 * 24 * time.Hour + + // unsignedPayload replaces the body hash: the signature cannot cover a body + // the signer never sees, and for a GET there is none. + unsignedPayload = "UNSIGNED-PAYLOAD" +) + +// PresignGetObject returns a URL that fetches the object, valid for expiry. +// versionID selects a specific version, or "" for the current one. +// +// Anyone holding the URL can read the object until it expires, so treat one +// like a password: it is a bearer credential scoped to a single key. +func (c *Client) PresignGetObject(bucket, key, versionID string, expiry time.Duration) (string, error) { + return c.presign(http.MethodGet, bucket, key, versionQuery(versionID), expiry) +} + +// PresignPutObject returns a URL that uploads to the key, valid for expiry. The +// holder can write that one key and nothing else — which is how a machine with +// no S3 credentials at all delivers a dump. +func (c *Client) PresignPutObject(bucket, key string, expiry time.Duration) (string, error) { + return c.presign(http.MethodPut, bucket, key, nil, expiry) +} + +// presign builds a query-signed URL for one method/key pair. +func (c *Client) presign(method, bucket, key string, extra url.Values, expiry time.Duration) (string, error) { + if c.cfg.Anonymous { + return "", errors.New("cannot presign without credentials (--s3-anonymous is set)") + } + if key == "" { + return "", errors.New("presigning needs an object key") + } + if expiry <= 0 { + return "", fmt.Errorf("expiry must be positive, got %s", expiry) + } + if expiry > MaxPresignExpiry { + return "", fmt.Errorf("expiry must be at most %s (SigV4's limit), got %s", MaxPresignExpiry, expiry) + } + + utc := c.now().UTC() + amzDate, dateStamp := utc.Format(amzDateFormat), utc.Format(dateStampFormat) + scope := strings.Join([]string{dateStamp, c.cfg.Region, service, terminacc}, "/") + + q := url.Values{} + for k, vs := range extra { + q[k] = vs + } + q.Set("X-Amz-Algorithm", algorithm) + q.Set("X-Amz-Credential", c.cfg.AccessKey+"/"+scope) + q.Set("X-Amz-Date", amzDate) + q.Set("X-Amz-Expires", strconv.Itoa(int(expiry.Seconds()))) + // Only host is signed: a browser or curl sends nothing else predictable, + // and an unsigned header is one the holder cannot be forced to reproduce. + q.Set("X-Amz-SignedHeaders", "host") + + u := c.url(bucket, key, q) + + canonicalRequest := strings.Join([]string{ + method, + canonicalURI(u), + u.RawQuery, // already canonical: c.url renders it with canonicalQuery + "host:" + u.Host + "\n", + "host", + unsignedPayload, + }, "\n") + + stringToSign := strings.Join([]string{ + algorithm, + amzDate, + scope, + hexSHA256([]byte(canonicalRequest)), + }, "\n") + + sig := hmacSHA256(signingKey(c.cfg.SecretKey, dateStamp, c.cfg.Region), stringToSign) + // Appended rather than added to q before signing: the signature is not part + // of what it covers, and re-rendering the query would reorder the pairs. + u.RawQuery += "&X-Amz-Signature=" + hexEncode(sig) + return u.String(), nil +} diff --git a/internal/s3/presign_test.go b/internal/s3/presign_test.go new file mode 100644 index 0000000..3e1c5e7 --- /dev/null +++ b/internal/s3/presign_test.go @@ -0,0 +1,125 @@ +package s3 + +import ( + "strings" + "testing" + "time" +) + +// presignClient builds a client with fixed credentials and a frozen clock, so a +// presigned URL is reproducible and can be compared against a golden value. +func presignClient(t *testing.T) *Client { + t.Helper() + c, err := New(Config{ + Endpoint: "https://s3.example.com", + Region: "garage", + AccessKey: "GKtest", + SecretKey: "secret", + PathStyle: true, + }) + if err != nil { + t.Fatal(err) + } + c.now = func() time.Time { return time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) } + return c +} + +// The golden signatures come from an independent SigV4 query-signing +// implementation written from the AWS specification ("Authenticating Requests: +// Using Query Parameters"), not from this package. A URL koc computes is only +// useful if a *server* accepts it, so agreeing with a second implementation is +// the property worth asserting — a test that re-derived the value with these +// same functions would pass however wrong they both were. +func TestPresignGetObjectMatchesTheSpec(t *testing.T) { + c := presignClient(t) + + // A space in the key is the interesting case: net/url would encode it as + // "+" in a query and leave it alone in a path, either of which breaks the + // signature. + got, err := c.PresignGetObject("db-backups", "dump 1.sql.gz", "", time.Hour) + if err != nil { + t.Fatal(err) + } + const want = "https://s3.example.com/db-backups/dump%201.sql.gz?" + + "X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Credential=GKtest%2F20260914%2Fgarage%2Fs3%2Faws4_request" + + "&X-Amz-Date=20260914T120000Z" + + "&X-Amz-Expires=3600" + + "&X-Amz-SignedHeaders=host" + + "&X-Amz-Signature=8190674d81d0695616b83fa8f54b73ba0d8fa202bc6b505ce52ba8b516771afc" + if got != want { + t.Errorf("presigned URL\n got %s\nwant %s", got, want) + } +} + +func TestPresignPutObjectMatchesTheSpec(t *testing.T) { + c := presignClient(t) + + got, err := c.PresignPutObject("db-backups", "incoming.bin", 15*time.Minute) + if err != nil { + t.Fatal(err) + } + const wantSig = "X-Amz-Signature=1b17213576b7451158c6054284e65dfdea8ea1c046a947af8d8993896feedf7d" + if !strings.HasSuffix(got, wantSig) { + t.Errorf("presigned PUT URL %s does not end with the expected signature", got) + } + if !strings.Contains(got, "X-Amz-Expires=900") { + t.Errorf("presigned PUT URL %s does not carry the 15m expiry", got) + } +} + +// A version ID is an ordinary query parameter, so it has to be signed with the +// rest — in canonical (sorted) order, which puts it after the X-Amz-* names. +func TestPresignGetObjectWithVersion(t *testing.T) { + c := presignClient(t) + + got, err := c.PresignGetObject("db-backups", "dump.sql.gz", "v2", time.Hour) + if err != nil { + t.Fatal(err) + } + const wantTail = "&X-Amz-SignedHeaders=host&versionId=v2" + + "&X-Amz-Signature=5d68330c6f95ebef61e3e373add6affbd8c3919a94a83b895c2360e6ee1630cd" + if !strings.HasSuffix(got, wantTail) { + t.Errorf("presigned URL %s does not end with %s", got, wantTail) + } +} + +func TestPresignRejectsBadExpiry(t *testing.T) { + c := presignClient(t) + + for _, tc := range []struct { + name string + expiry time.Duration + want string + }{ + {"zero", 0, "must be positive"}, + {"negative", -time.Second, "must be positive"}, + {"past the ceiling", MaxPresignExpiry + time.Second, "at most"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := c.PresignGetObject("b", "k", "", tc.expiry); err == nil || + !strings.Contains(err.Error(), tc.want) { + t.Errorf("err = %v, want it to mention %q", err, tc.want) + } + }) + } +} + +// Anonymous mode has no secret to sign with, so presigning must say so rather +// than emit a URL signed with an empty key. +func TestPresignRefusedWhenAnonymous(t *testing.T) { + c, err := New(Config{Endpoint: "https://s3.example.com", Anonymous: true}) + if err != nil { + t.Fatal(err) + } + if _, err := c.PresignGetObject("b", "k", "", time.Hour); err == nil || + !strings.Contains(err.Error(), "anonymous") { + t.Errorf("err = %v, want it to name --s3-anonymous", err) + } +} + +func TestPresignNeedsAKey(t *testing.T) { + if _, err := presignClient(t).PresignGetObject("b", "", "", time.Hour); err == nil { + t.Error("presigning a bucket with no key was accepted") + } +} diff --git a/internal/s3/retry.go b/internal/s3/retry.go new file mode 100644 index 0000000..6d82b2a --- /dev/null +++ b/internal/s3/retry.go @@ -0,0 +1,91 @@ +package s3 + +import ( + "context" + "errors" + "net/http" + "time" +) + +// Retry policy. A koc S3 call can be a multi-gigabyte transfer over a cluster +// network, where a reset connection or a momentary 503 from one gateway node is +// routine rather than exceptional — s5cmd retries ten times by default for the +// same reason. Without this a transient blip failed the whole command and, on an +// upload, left the object half-written. +const ( + // retryBaseDelay is the first backoff interval; each further attempt doubles + // it up to retryMaxDelay. + retryBaseDelay = 200 * time.Millisecond + retryMaxDelay = 10 * time.Second +) + +// retryableCodes are the S3 error codes that mean "ask again", as opposed to +// "you asked wrong". SlowDown and RequestTimeout are the two a healthy store +// still emits under load. RequestTimeTooSkewed is deliberately absent: it is a +// clock problem, and repeating the request cannot fix it. +var retryableCodes = map[string]bool{ + "InternalError": true, + "ServiceUnavailable": true, + "SlowDown": true, + "RequestTimeout": true, +} + +// nonRetryableError marks an error raised after the response headers were accepted, +// i.e. by the sink while it consumed the body. Those are never replayed: the +// sink has already written some of the object to a file or to stdout, and a +// second attempt would append a duplicate prefix rather than resume. +type nonRetryableError struct{ err error } + +func (e *nonRetryableError) Error() string { return e.err.Error() } +func (e *nonRetryableError) Unwrap() error { return e.err } + +// retryable reports whether err is worth another attempt. +func retryable(err error) bool { + var sink *nonRetryableError + if errors.As(err, &sink) { + return false + } + + var ae *APIError + if !errors.As(err, &ae) { + // Not an answer from S3 at all: a dial failure, a reset, or EOF before + // the headers arrived. Nothing was applied, so it is safe to repeat. + return true + } + if retryableCodes[ae.Code] { + return true + } + switch ae.StatusCode { + case http.StatusRequestTimeout, http.StatusTooManyRequests: + return true + } + return ae.StatusCode >= http.StatusInternalServerError +} + +// backoff waits before attempt n (1 = the first retry), or returns the +// context's error if the caller gave up first. The delay carries a little +// jitter so a pool of --concurrency workers that all hit the same 503 do not +// all come back in the same millisecond; it is read off the clock rather than +// from math/rand because spreading requests is the only property needed and an +// unseeded PRNG in a short-lived CLI gives no better one. +func (c *Client) backoff(ctx context.Context, n int) error { + d := c.retryBase + for range n - 1 { + if d *= 2; d >= retryMaxDelay { + d = retryMaxDelay + break + } + } + if spread := int64(d / 4); spread > 0 { + d += time.Duration(c.now().UnixNano() % spread) + } + + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} diff --git a/internal/s3/retry_test.go b/internal/s3/retry_test.go new file mode 100644 index 0000000..f1538e3 --- /dev/null +++ b/internal/s3/retry_test.go @@ -0,0 +1,234 @@ +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +// retryClient is newTestClient with retries on and a backoff short enough that +// a test does not spend real time asleep. +func retryClient(t *testing.T, retries int, h http.HandlerFunc) *Client { + t.Helper() + c := newTestClient(t, h) + c.cfg.MaxRetries = retries + c.retryBase = time.Microsecond + return c +} + +func TestRetryableClassification(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"transport failure", errors.New("dial tcp: connection refused"), true}, + {"500", &APIError{StatusCode: 500}, true}, + {"503 SlowDown", &APIError{StatusCode: 503, Code: "SlowDown"}, true}, + {"429", &APIError{StatusCode: 429}, true}, + {"408", &APIError{StatusCode: 408}, true}, + {"403", &APIError{StatusCode: 403, Code: "AccessDenied"}, false}, + {"404", &APIError{StatusCode: 404, Code: "NoSuchKey"}, false}, + {"clock skew is not retryable", &APIError{StatusCode: 403, Code: "RequestTimeTooSkewed"}, false}, + {"a sink failure is never replayed", &nonRetryableError{err: errors.New("short write")}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := retryable(tc.err); got != tc.want { + t.Errorf("retryable(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +func TestDoRetriesA503ThenSucceeds(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 3, func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = fmt.Fprint(w, `SlowDown`) + return + } + _, _ = fmt.Fprint(w, ``) + }) + + if _, err := c.ListBuckets(context.Background()); err != nil { + t.Fatalf("a retryable failure was not retried: %v", err) + } + if got := calls.Load(); got != 3 { + t.Errorf("attempts = %d, want 3", got) + } +} + +// The retry budget is finite: past it the last error reaches the caller. +func TestDoGivesUpAfterMaxRetries(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 2, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusInternalServerError) + }) + + if _, err := c.ListBuckets(context.Background()); err == nil { + t.Fatal("expected the failure to be reported once the budget ran out") + } + if got := calls.Load(); got != 3 { + t.Errorf("attempts = %d, want 1 try + 2 retries", got) + } +} + +func TestDoDoesNotRetryA403(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 5, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(w, `AccessDenied`) + }) + + if _, err := c.ListBuckets(context.Background()); err == nil { + t.Fatal("expected an error") + } + if got := calls.Load(); got != 1 { + t.Errorf("attempts = %d, want 1 — a 403 is not worth repeating", got) + } +} + +// A retried PUT has to send the same bytes again, which is the whole reason +// request.body is an io.ReadSeeker. +func TestDoRewindsTheBodyOnRetry(t *testing.T) { + var bodies []string + var calls atomic.Int32 + c := retryClient(t, 1, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(body)) + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set("ETag", `"abc"`) + w.WriteHeader(http.StatusOK) + }) + + payload := "the whole dump" + obj, err := c.PutObject(context.Background(), "b", "k", strings.NewReader(payload), + int64(len(payload)), "application/octet-stream") + if err != nil { + t.Fatal(err) + } + if obj.ETag != "abc" { + t.Errorf("ETag = %q, want abc", obj.ETag) + } + if len(bodies) != 2 || bodies[0] != payload || bodies[1] != payload { + t.Errorf("bodies = %q, want the same payload twice", bodies) + } +} + +// A body read from the caller's current offset must rewind to *that* offset, not +// to byte zero: an upload of one slice of a file would otherwise re-send the +// wrong bytes. +func TestDoRewindsToTheCallerOffset(t *testing.T) { + var bodies []string + var calls atomic.Int32 + c := retryClient(t, 1, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(body)) + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + }) + + src := bytes.NewReader([]byte("SKIPMEpayload")) + if _, err := src.Seek(6, io.SeekStart); err != nil { + t.Fatal(err) + } + if _, err := c.PutObject(context.Background(), "b", "k", src, 7, ""); err != nil { + t.Fatal(err) + } + for i, got := range bodies { + if got != "payload" { + t.Errorf("attempt %d sent %q, want payload", i+1, got) + } + } +} + +// A failure raised while the sink was consuming the body must not be replayed: +// the bytes it already wrote to a file or to stdout cannot be taken back. +func TestDoDoesNotRetryASinkFailure(t *testing.T) { + var calls atomic.Int32 + c := retryClient(t, 5, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + _, _ = w.Write([]byte("payload")) + }) + + _, err := c.GetObject(context.Background(), "b", "k", "", failWriter{}) + if err == nil { + t.Fatal("expected the sink failure to be reported") + } + if got := calls.Load(); got != 1 { + t.Errorf("attempts = %d, want 1 — a partly written destination cannot be replayed", got) + } + // The caller must see its own error, not the internal marker. + var marker *nonRetryableError + if errors.As(err, &marker) { + t.Errorf("error %v still carries the internal retry marker", err) + } +} + +// A cancelled context ends the loop instead of sleeping out the budget. +func TestDoStopsOnContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + var calls atomic.Int32 + c := retryClient(t, 5, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + cancel() + w.WriteHeader(http.StatusServiceUnavailable) + }) + + if _, err := c.ListBuckets(ctx); err == nil { + t.Fatal("expected an error") + } + if got := calls.Load(); got != 1 { + t.Errorf("attempts = %d, want 1 once the caller gave up", got) + } +} + +// Anonymous mode must send no credential at all — that is the point of it. +func TestAnonymousSendsNoSignature(t *testing.T) { + var auth, sha string + srv := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + auth, sha = r.Header.Get("Authorization"), r.Header.Get("X-Amz-Content-Sha256") + _, _ = fmt.Fprint(w, `false`) + }) + + c, err := New(Config{Endpoint: srv.Endpoint(), Anonymous: true, PathStyle: true}) + if err != nil { + t.Fatal(err) + } + if _, err := c.ListObjects(context.Background(), "public", ListOptions{}); err != nil { + t.Fatal(err) + } + if auth != "" { + t.Errorf("Authorization = %q, want none", auth) + } + if sha != "" { + t.Errorf("X-Amz-Content-Sha256 = %q, want none", sha) + } +} + +// Anonymous mode is the only one that needs no credentials; every other mode +// must still insist on them. +func TestNewRequiresCredentialsUnlessAnonymous(t *testing.T) { + if _, err := New(Config{Endpoint: "https://s3.example.com"}); err == nil { + t.Error("a client with no credentials was accepted") + } + if _, err := New(Config{Endpoint: "https://s3.example.com", Anonymous: true}); err != nil { + t.Errorf("anonymous client rejected: %v", err) + } +} diff --git a/internal/s3/s3.go b/internal/s3/s3.go index db942af..9e92832 100644 --- a/internal/s3/s3.go +++ b/internal/s3/s3.go @@ -13,6 +13,7 @@ package s3 import ( + "bytes" "context" "crypto/sha256" "crypto/tls" @@ -69,6 +70,14 @@ type Config struct { // cap. A wedged endpoint is still caught by responseHeaderTimeout. Timeout time.Duration + // MaxRetries is how many further attempts a retryable failure gets (0 = + // none). See retry.go for what counts as retryable. + MaxRetries int + + // Anonymous sends requests unsigned, for a bucket granted to everyone. It + // is the only mode in which no credentials are required. + Anonymous bool + Debug bool } @@ -80,6 +89,9 @@ type Client struct { // now is the signing clock, overridden in tests. now func() time.Time + // retryBase is the first backoff interval, overridden in tests so a retry + // case does not spend real time asleep. + retryBase time.Duration } // New validates the config and builds the client. It performs no network I/O: @@ -88,8 +100,8 @@ func New(cfg Config) (*Client, error) { if cfg.Endpoint == "" { return nil, errors.New("S3 endpoint is required (--s3-endpoint / AWS_ENDPOINT_URL)") } - if cfg.AccessKey == "" || cfg.SecretKey == "" { - return nil, errors.New("S3 credentials are required (--s3-access-key/--s3-secret-key, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY, or --s3-creds-from-ns)") + if !cfg.Anonymous && (cfg.AccessKey == "" || cfg.SecretKey == "") { + return nil, errors.New("S3 credentials are required (--s3-access-key/--s3-secret-key, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY, or --s3-creds-from-ns); pass --s3-anonymous for a publicly readable bucket") } if cfg.Region == "" { cfg.Region = DefaultRegion @@ -117,10 +129,11 @@ func New(cfg Config) (*Client, error) { } return &Client{ - cfg: cfg, - hc: newHTTPClient(tlsCfg, cfg.Timeout), - base: base, - now: time.Now, + cfg: cfg, + hc: newHTTPClient(tlsCfg, cfg.Timeout), + base: base, + now: time.Now, + retryBase: retryBaseDelay, }, nil } @@ -190,6 +203,18 @@ func IsNotFound(err error) bool { return ae.StatusCode == http.StatusNotFound } +// ErrorCode returns the S3 error code carried by err ("BucketNotEmpty", +// "BucketAlreadyOwnedByYou"), or "" if err is not an S3 API error. Callers use +// it to turn a specific server answer into advice; matching on the message +// would break with the server's wording. +func ErrorCode(err error) string { + var ae *APIError + if !errors.As(err, &ae) { + return "" + } + return ae.Code +} + // Bucket is one entry of a ListBuckets result. type Bucket struct { Name string @@ -203,6 +228,39 @@ type Object struct { LastModified time.Time ETag string StorageClass string + + // IsPrefix marks a CommonPrefixes entry rather than a key: what a + // delimited listing reports in place of everything below it. Only Key is + // meaningful on one. + IsPrefix bool + + // VersionID, IsLatest and DeleteMarker are filled only by a versioned + // listing (ListOptions.Versions). A store without versioning — Garage has + // none — never sets them. + VersionID string + IsLatest bool + DeleteMarker bool +} + +// ListOptions narrows a listing. The zero value lists every current object in +// the bucket. +type ListOptions struct { + // Prefix restricts the listing to keys starting with it. + Prefix string + + // Delimiter collapses everything after the next occurrence of it into a + // single CommonPrefixes entry, which is how "/" turns a flat keyspace into + // one directory level. Such entries arrive as Objects with IsPrefix set. + Delimiter string + + // Limit caps the number of entries reported (0 = no cap). It is a hard + // result cap, not merely a page size — the same rule the rest of koc's + // --limit flags follow. + Limit int + + // Versions lists every version and delete marker of each key instead of + // only the current object, via the ?versions endpoint. + Versions bool } // ObjectInfo describes a single object, as returned by HeadObject (and by @@ -214,6 +272,7 @@ type ObjectInfo struct { LastModified time.Time ETag string ContentType string + VersionID string // set only by a store with versioning on Metadata map[string]string // x-amz-meta-*, with the prefix stripped } @@ -240,6 +299,56 @@ func (c *Client) ListBuckets(ctx context.Context) ([]Bucket, error) { return out, nil } +// usEast1 is the one region whose name must NOT appear in a +// CreateBucketConfiguration: AWS treats it as the default and rejects the body +// that names it. +const usEast1 = "us-east-1" + +// createBucketConfiguration is the CreateBucket request body. It exists because +// every region other than us-east-1 requires the bucket's location to be stated +// explicitly, and Garage validates the value against its configured region. +type createBucketConfiguration struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + LocationConstraint string `xml:"LocationConstraint"` +} + +// CreateBucket creates a bucket. The location constraint is the signing region, +// which is necessarily the right answer: a request signed for region R is only +// accepted by a store serving R. +// +// Creating a bucket the credentials already own is not an error on AWS outside +// us-east-1 either — it answers BucketAlreadyOwnedByYou, which the caller can +// recognise with ErrorCode. +func (c *Client) CreateBucket(ctx context.Context, bucket string) error { + req := request{method: http.MethodPut, url: c.url(bucket, "", nil), payloadHash: emptySHA256} + + if region := c.cfg.Region; region != "" && region != usEast1 { + body, err := xml.Marshal(createBucketConfiguration{LocationConstraint: region}) + if err != nil { + return fmt.Errorf("encoding create-bucket body: %w", err) + } + req.body = bytes.NewReader(body) + req.payloadHash = hexSHA256(body) + req.size = int64(len(body)) + req.header = map[string]string{"Content-Type": "application/xml"} + } + + return c.do(ctx, req, drainBody) +} + +// DeleteBucket removes an empty bucket. A bucket with objects in it is refused +// by the server with BucketNotEmpty; nothing is deleted implicitly. +func (c *Client) DeleteBucket(ctx context.Context, bucket string) error { + return c.do(ctx, request{method: http.MethodDelete, url: c.url(bucket, "", nil), payloadHash: emptySHA256}, drainBody) +} + +// drainBody is the sink for a call whose reply carries nothing the caller wants. +// The body is still read so the connection goes back to the pool. +func drainBody(resp *http.Response) error { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, errorBodyLimit)) + return nil +} + // listObjectsPage is one ListObjectsV2 response. type listObjectsPage struct { IsTruncated bool `xml:"IsTruncated"` @@ -251,47 +360,104 @@ type listObjectsPage struct { ETag string `xml:"ETag"` StorageClass string `xml:"StorageClass"` } `xml:"Contents"` + CommonPrefixes []struct { + Prefix string `xml:"Prefix"` + } `xml:"CommonPrefixes"` } // ListObjects lists objects in a bucket, following continuation tokens until the // server says there are no more. limit caps the number of objects returned (0 = // no cap) and is applied as a hard result cap, not merely as a page size — the // same rule the rest of koc's --limit flags follow. -func (c *Client) ListObjects(ctx context.Context, bucket, prefix string, limit int) ([]Object, error) { +func (c *Client) ListObjects(ctx context.Context, bucket string, opts ListOptions) ([]Object, error) { var out []Object - token := "" - for { - q := url.Values{"list-type": {"2"}, "max-keys": {strconv.Itoa(pageSize(limit, len(out)))}} - if prefix != "" { - q.Set("prefix", prefix) - } - if token != "" { - q.Set("continuation-token", token) - } + err := c.ListObjectsFunc(ctx, bucket, opts, func(o Object) error { + out = append(out, o) + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +// ListObjectsFunc is ListObjects without materialising the result: fn is called +// once per key, in listing order, and an error from it stops the walk and is +// returned as-is. Callers that only fold over the listing (summing sizes, +// deleting as they go) use this so a bucket with a million keys costs one page +// of memory instead of all of them. +func (c *Client) ListObjectsFunc(ctx context.Context, bucket string, opts ListOptions, fn func(Object) error) error { + if opts.Versions { + return c.listObjectVersions(ctx, bucket, opts, fn) + } + token, seen := "", 0 + for { var page listObjectsPage - if err := c.getXML(ctx, c.url(bucket, "", q), &page); err != nil { - return nil, err + u := c.url(bucket, "", listObjectsQuery(opts, token, seen)) + if err := c.getXML(ctx, u, &page); err != nil { + return err } - for _, o := range page.Contents { - out = append(out, Object{ - Key: o.Key, - Size: o.Size, - LastModified: parseS3Time(o.LastModified), - ETag: strings.Trim(o.ETag, `"`), - StorageClass: o.StorageClass, - }) - if limit > 0 && len(out) >= limit { - return out, nil - } + + done, err := emitPage(&page, opts.Limit, &seen, fn) + if err != nil || done { + return err } if !page.IsTruncated || page.NextContinuationToken == "" { - return out, nil + return nil } token = page.NextContinuationToken } } +// listObjectsQuery renders one ListObjectsV2 request's parameters. +func listObjectsQuery(opts ListOptions, token string, seen int) url.Values { + q := url.Values{"list-type": {"2"}, "max-keys": {strconv.Itoa(pageSize(opts.Limit, seen))}} + if opts.Prefix != "" { + q.Set("prefix", opts.Prefix) + } + if opts.Delimiter != "" { + q.Set("delimiter", opts.Delimiter) + } + if token != "" { + q.Set("continuation-token", token) + } + return q +} + +// emitPage hands one page's entries to fn, counting them against limit. It +// reports done once the limit is reached, so the caller stops paging. +// +// CommonPrefixes come first so a delimited listing reads like a directory: the +// subtrees, then the keys at this level. +func emitPage(page *listObjectsPage, limit int, seen *int, fn func(Object) error) (done bool, err error) { + for _, p := range page.CommonPrefixes { + if err := fn(Object{Key: p.Prefix, IsPrefix: true}); err != nil { + return false, err + } + if *seen++; limit > 0 && *seen >= limit { + return true, nil + } + } + for _, o := range page.Contents { + err := fn(Object{ + Key: o.Key, + Size: o.Size, + LastModified: parseS3Time(o.LastModified), + ETag: strings.Trim(o.ETag, `"`), + StorageClass: o.StorageClass, + IsLatest: true, + }) + if err != nil { + return false, err + } + if *seen++; limit > 0 && *seen >= limit { + return true, nil + } + } + return false, nil +} + // pageSize asks for a full page unless a limit means fewer keys are wanted. func pageSize(limit, have int) int { if limit <= 0 || limit-have > maxKeysPerPage { @@ -300,12 +466,30 @@ func pageSize(limit, have int) int { return limit - have } -// HeadObject fetches an object's metadata without its body. -func (c *Client) HeadObject(ctx context.Context, bucket, key string) (*ObjectInfo, error) { +// versionQuery addresses one specific version of a key, or the current one when +// versionID is empty. +func versionQuery(versionID string) url.Values { + if versionID == "" { + return nil + } + return url.Values{"versionId": {versionID}} +} + +// HeadBucket reports whether the bucket exists and the credentials may reach +// it, without listing anything. It is the cheapest possible probe: a bodiless +// request that costs the server no listing work. +func (c *Client) HeadBucket(ctx context.Context, bucket string) error { + return c.do(ctx, request{method: http.MethodHead, url: c.url(bucket, "", nil), payloadHash: emptySHA256}, drainBody) +} + +// HeadObject fetches an object's metadata without its body. versionID selects a +// specific version, or "" for the current one. +func (c *Client) HeadObject(ctx context.Context, bucket, key, versionID string) (*ObjectInfo, error) { var info *ObjectInfo - err := c.do(ctx, request{method: http.MethodHead, url: c.url(bucket, key, nil), payloadHash: emptySHA256}, + err := c.do(ctx, request{method: http.MethodHead, url: c.url(bucket, key, versionQuery(versionID)), payloadHash: emptySHA256}, func(resp *http.Response) error { info = objectInfoFromHeader(bucket, key, resp) + info.VersionID = resp.Header.Get("x-amz-version-id") return nil }) if err != nil { @@ -316,9 +500,9 @@ func (c *Client) HeadObject(ctx context.Context, bucket, key string) (*ObjectInf // GetObject streams an object's body to w and returns the number of bytes // written. -func (c *Client) GetObject(ctx context.Context, bucket, key string, w io.Writer) (int64, error) { +func (c *Client) GetObject(ctx context.Context, bucket, key, versionID string, w io.Writer) (int64, error) { var n int64 - err := c.do(ctx, request{method: http.MethodGet, url: c.url(bucket, key, nil), payloadHash: emptySHA256}, + err := c.do(ctx, request{method: http.MethodGet, url: c.url(bucket, key, versionQuery(versionID)), payloadHash: emptySHA256}, func(resp *http.Response) error { var cerr error n, cerr = io.Copy(w, resp.Body) @@ -330,12 +514,21 @@ func (c *Client) GetObject(ctx context.Context, bucket, key string, w io.Writer) return n, err } -// PutObject uploads body as a single part. The reader must be seekable because -// SigV4 signs a hash of the payload: the body is read once to hash it, then -// rewound and sent. That rules out streaming from a pipe, and is the reason -// there is no multipart support here — for the backup-sized objects koc moves, -// a single signed PUT is enough, and an S3 server's own single-part ceiling -// (5 GiB) applies. +// DeleteObject removes one object. S3 answers 204 for a key that was never +// there, so a delete is idempotent and "no such key" is not reported — the +// caller asked for the key to be gone, and it is. +func (c *Client) DeleteObject(ctx context.Context, bucket, key, versionID string) error { + return c.do(ctx, request{method: http.MethodDelete, url: c.url(bucket, key, versionQuery(versionID)), payloadHash: emptySHA256}, drainBody) +} + +// PutObject uploads body as a single request. The reader must be seekable +// because SigV4 signs a hash of the payload: the body is read once to hash it, +// then rewound and sent — which is also what lets a retry replay it. +// +// That rules out a pipe, and the server's single-part ceiling (5 GiB +// everywhere) applies. Callers that cannot promise either use +// PutObjectStream, which picks this path when the source is seekable and fits +// one part and goes multipart otherwise. func (c *Client) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker, size int64, contentType string) (*ObjectInfo, error) { hash, err := hashSeeker(body) if err != nil { @@ -466,22 +659,89 @@ func (c *Client) getXML(ctx context.Context, u *url.URL, out any) error { type request struct { method string url *url.URL - body io.Reader // nil for GET/HEAD + // body is seekable rather than a plain io.Reader so a retried attempt can + // replay it — SigV4 already requires the payload to be readable twice, so + // this costs nothing extra. + body io.ReadSeeker // nil for GET/HEAD // payloadHash is the SigV4 hash of body; emptySHA256 when there is none. payloadHash string size int64 // Content-Length; only read when body != nil header map[string]string // extra request headers, e.g. Content-Type } -// do signs and performs one request, then hands the still-open response to sink. -// Owning the response lifecycle here (rather than returning it) keeps every body -// closed on every path, including the error ones. +// do performs a request, retrying a retryable failure up to cfg.MaxRetries +// times with exponential backoff. An error raised by sink is never retried — +// see nonRetryableError. // // Bodies are never logged, even under --debug: a request body is object data and // a response body can be too, while the headers carry the signature. Method, // path and status are enough to debug a 403. func (c *Client) do(ctx context.Context, r request, sink func(*http.Response) error) error { - req, err := http.NewRequestWithContext(ctx, r.method, r.url.String(), r.body) + start, err := bodyOffset(r.body) + if err != nil { + return err + } + + for attempt := 0; ; attempt++ { + if attempt > 0 { + if err := c.backoff(ctx, attempt); err != nil { + return err + } + if r.body != nil { + if _, err := r.body.Seek(start, io.SeekStart); err != nil { + return fmt.Errorf("rewinding request body for retry: %w", err) + } + } + } + + err := c.attempt(ctx, r, sink) + if err == nil { + return nil + } + if attempt >= c.cfg.MaxRetries || ctx.Err() != nil || !retryable(err) { + return unwrapSink(err) + } + if c.cfg.Debug { + fmt.Fprintf(os.Stderr, "s3: %s %s failed (%v); retrying\n", r.method, r.url.RequestURI(), err) + } + } +} + +// bodyOffset records where a replayable body starts, so a retry rewinds to the +// caller's position rather than to byte zero of, say, an open file. +func bodyOffset(body io.ReadSeeker) (int64, error) { + if body == nil { + return 0, nil + } + at, err := body.Seek(0, io.SeekCurrent) + if err != nil { + return 0, fmt.Errorf("seeking request body: %w", err) + } + return at, nil +} + +// unwrapSink strips the marker retryable() keys off, so callers see the error +// the sink actually returned. +func unwrapSink(err error) error { + var sink *nonRetryableError + if errors.As(err, &sink) { + return sink.err + } + return err +} + +// attempt performs one signed request, then hands the still-open response to +// sink. Owning the response lifecycle here (rather than returning it) keeps +// every body closed on every path, including the error ones. +func (c *Client) attempt(ctx context.Context, r request, sink func(*http.Response) error) error { + // A nil io.ReadSeeker in an interface-typed argument is not a nil + // io.Reader, and net/http treats the difference as "body of unknown + // length" — so the nil case has to be passed explicitly. + var rc io.Reader + if r.body != nil { + rc = r.body + } + req, err := http.NewRequestWithContext(ctx, r.method, r.url.String(), rc) if err != nil { return err } @@ -491,7 +751,9 @@ func (c *Client) do(ctx context.Context, r request, sink func(*http.Response) er if r.body != nil { req.ContentLength = r.size } - c.sign(req, r.payloadHash, c.now()) + if !c.cfg.Anonymous { + c.sign(req, r.payloadHash, c.now()) + } resp, err := c.hc.Do(req) if err != nil { @@ -505,7 +767,10 @@ func (c *Client) do(ctx context.Context, r request, sink func(*http.Response) er if resp.StatusCode < 200 || resp.StatusCode >= 300 { return newAPIError(r.method, r.url.Path, resp) } - return sink(resp) + if err := sink(resp); err != nil { + return &nonRetryableError{err: err} + } + return nil } // newAPIError builds an APIError from a failed response, parsing S3's XML error diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index 4f5a33e..1ecb422 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -121,7 +121,7 @@ func TestListObjectsPaging(t *testing.T) { `) }) - objs, err := c.ListObjects(context.Background(), "db-backups", "e2e-", 0) + objs, err := c.ListObjects(context.Background(), "db-backups", ListOptions{Prefix: "e2e-"}) if err != nil { t.Fatal(err) } @@ -159,7 +159,7 @@ func TestListObjectsLimit(t *testing.T) { `) }) - objs, err := c.ListObjects(context.Background(), "b", "", 1) + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Limit: 1}) if err != nil { t.Fatal(err) } @@ -188,7 +188,7 @@ func TestHeadObject(t *testing.T) { w.WriteHeader(http.StatusOK) }) - info, err := c.HeadObject(context.Background(), "db-backups", "e2e-a.sql.gz") + info, err := c.HeadObject(context.Background(), "db-backups", "e2e-a.sql.gz", "") if err != nil { t.Fatal(err) } @@ -216,7 +216,7 @@ func TestGetObject(t *testing.T) { }) var buf bytes.Buffer - n, err := c.GetObject(context.Background(), "db-backups", "dir/a b.txt", &buf) + n, err := c.GetObject(context.Background(), "db-backups", "dir/a b.txt", "", &buf) if err != nil { t.Fatal(err) } @@ -296,7 +296,7 @@ func TestAPIError(t *testing.T) { `/db-backups/garage`)) }) - _, err := c.ListObjects(context.Background(), "db-backups", "", 0) + _, err := c.ListObjects(context.Background(), "db-backups", ListOptions{}) var apiErr *APIError if !asAPIError(err, &apiErr) { t.Fatalf("err = %v (%T), want *APIError", err, err) @@ -311,7 +311,7 @@ func TestAPIError(t *testing.T) { t.Error("AccessDenied must not read as not-found") } - _, err = c.HeadObject(context.Background(), "db-backups", "missing") + _, err = c.HeadObject(context.Background(), "db-backups", "missing", "") if !IsNotFound(err) { t.Errorf("bodiless 404 = %v, want IsNotFound", err) } diff --git a/internal/s3/sign.go b/internal/s3/sign.go index ed471ee..45316da 100644 --- a/internal/s3/sign.go +++ b/internal/s3/sign.go @@ -204,3 +204,7 @@ func hexSHA256(b []byte) string { sum := sha256.Sum256(b) return hex.EncodeToString(sum[:]) } + +// hexEncode names encoding/hex at the one call site outside this file, so +// presign.go does not import it only to render a signature. +func hexEncode(b []byte) string { return hex.EncodeToString(b) } diff --git a/internal/s3/transport.go b/internal/s3/transport.go index 8d5e6d1..4ed3972 100644 --- a/internal/s3/transport.go +++ b/internal/s3/transport.go @@ -19,6 +19,12 @@ const ( // maxRedirects caps a redirect chain. See sameHostRedirect for what travels // with one. maxRedirects = 5 + + // abortTimeout bounds the cleanup request a failed multipart upload sends. + // It runs on a fresh context — the caller's is usually already cancelled — + // so it needs a deadline of its own or a wedged endpoint would hang the + // error path. + abortTimeout = 30 * time.Second ) // credentialHeaders are dropped when a redirect leaves the origin host. Go's own diff --git a/internal/s3/versioning.go b/internal/s3/versioning.go new file mode 100644 index 0000000..1a757f8 --- /dev/null +++ b/internal/s3/versioning.go @@ -0,0 +1,185 @@ +package s3 + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" +) + +// Object versioning. Note that Garage — koc's primary target — does not +// implement it: its GetBucketVersioning is a stub that always answers "not +// enabled", and enabling it fails. These calls are here for the other stores +// koc can be pointed at (AWS, Ceph RGW, MinIO), and for the operator who needs +// to confirm which of the two they are talking to. +const ( + // VersioningEnabled and VersioningSuspended are the two states S3 defines. + // A bucket that was never configured reports neither, which this package + // renders as VersioningUnversioned. + VersioningEnabled = "Enabled" + VersioningSuspended = "Suspended" + VersioningUnversioned = "Unversioned" +) + +// versioningConfiguration is the body of both versioning calls. +type versioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status,omitempty"` +} + +// GetBucketVersioning reports the bucket's versioning state, normalising the +// "never configured" empty answer to VersioningUnversioned so a caller never +// has to special-case the empty string. +func (c *Client) GetBucketVersioning(ctx context.Context, bucket string) (string, error) { + var cfg versioningConfiguration + if err := c.getXML(ctx, c.url(bucket, "", url.Values{"versioning": {""}}), &cfg); err != nil { + return "", err + } + if cfg.Status == "" { + return VersioningUnversioned, nil + } + return cfg.Status, nil +} + +// SetBucketVersioning enables or suspends versioning. status must be +// VersioningEnabled or VersioningSuspended: S3 has no call that returns a +// bucket to never-versioned, which is why VersioningUnversioned is not accepted +// here. +func (c *Client) SetBucketVersioning(ctx context.Context, bucket, status string) error { + switch status { + case VersioningEnabled, VersioningSuspended: + default: + return fmt.Errorf("versioning status must be %q or %q, not %q", + VersioningEnabled, VersioningSuspended, status) + } + + body, err := xml.Marshal(versioningConfiguration{Status: status}) + if err != nil { + return fmt.Errorf("encoding versioning body: %w", err) + } + return c.do(ctx, request{ + method: http.MethodPut, + url: c.url(bucket, "", url.Values{"versioning": {""}}), + body: bytes.NewReader(body), + payloadHash: hexSHA256(body), + size: int64(len(body)), + header: map[string]string{ + "Content-Type": "application/xml", + "Content-MD5": contentMD5(body), + }, + }, drainBody) +} + +// listVersionsPage is one ListObjectVersions response. +type listVersionsPage struct { + IsTruncated bool `xml:"IsTruncated"` + NextKeyMarker string `xml:"NextKeyMarker"` + NextVersionIDMarker string `xml:"NextVersionIdMarker"` + Versions []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + Size int64 `xml:"Size"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + StorageClass string `xml:"StorageClass"` + } `xml:"Version"` + DeleteMarkers []struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified string `xml:"LastModified"` + } `xml:"DeleteMarker"` + CommonPrefixes []struct { + Prefix string `xml:"Prefix"` + } `xml:"CommonPrefixes"` +} + +// listObjectVersions walks the ?versions endpoint, which pages on a key marker +// plus a version-id marker rather than a continuation token. +// +// A page's and elements interleave on the wire, and +// decoding them into two slices loses that order, so each page is re-sorted by +// key and then newest-first. The result is the presentation a reader wants +// anyway, and it is deterministic, which the wire order across a page boundary +// is not. +func (c *Client) listObjectVersions(ctx context.Context, bucket string, opts ListOptions, + fn func(Object) error) error { + keyMarker, versionMarker, seen := "", "", 0 + for { + q := url.Values{"versions": {""}, "max-keys": {strconv.Itoa(pageSize(opts.Limit, seen))}} + if opts.Prefix != "" { + q.Set("prefix", opts.Prefix) + } + if opts.Delimiter != "" { + q.Set("delimiter", opts.Delimiter) + } + if keyMarker != "" { + q.Set("key-marker", keyMarker) + } + if versionMarker != "" { + q.Set("version-id-marker", versionMarker) + } + + var page listVersionsPage + if err := c.getXML(ctx, c.url(bucket, "", q), &page); err != nil { + return err + } + for _, obj := range versionEntries(&page) { + if err := fn(obj); err != nil { + return err + } + if seen++; opts.Limit > 0 && seen >= opts.Limit { + return nil + } + } + if !page.IsTruncated || (page.NextKeyMarker == "" && page.NextVersionIDMarker == "") { + return nil + } + keyMarker, versionMarker = page.NextKeyMarker, page.NextVersionIDMarker + } +} + +// versionEntries flattens one page into the order described on +// listObjectVersions. +func versionEntries(page *listVersionsPage) []Object { + out := make([]Object, 0, len(page.Versions)+len(page.DeleteMarkers)+len(page.CommonPrefixes)) + for _, p := range page.CommonPrefixes { + out = append(out, Object{Key: p.Prefix, IsPrefix: true}) + } + for _, v := range page.Versions { + out = append(out, Object{ + Key: v.Key, + Size: v.Size, + LastModified: parseS3Time(v.LastModified), + ETag: strings.Trim(v.ETag, `"`), + StorageClass: v.StorageClass, + VersionID: v.VersionID, + IsLatest: v.IsLatest, + }) + } + for _, d := range page.DeleteMarkers { + out = append(out, Object{ + Key: d.Key, + LastModified: parseS3Time(d.LastModified), + VersionID: d.VersionID, + IsLatest: d.IsLatest, + DeleteMarker: true, + }) + } + + prefixes := len(page.CommonPrefixes) + sort.SliceStable(out[prefixes:], func(i, j int) bool { + a, b := out[prefixes+i], out[prefixes+j] + if a.Key != b.Key { + return a.Key < b.Key + } + return a.LastModified.After(b.LastModified) + }) + return out +} diff --git a/internal/s3/versioning_test.go b/internal/s3/versioning_test.go new file mode 100644 index 0000000..8366e96 --- /dev/null +++ b/internal/s3/versioning_test.go @@ -0,0 +1,260 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" +) + +// A bucket that was never configured answers an empty document, which must be +// normalised rather than handed to the caller as an empty string. +func TestGetBucketVersioning(t *testing.T) { + for _, tc := range []struct { + name, body, want string + }{ + {"enabled", `Enabled`, VersioningEnabled}, + {"suspended", `Suspended`, VersioningSuspended}, + {"never configured", ``, VersioningUnversioned}, + } { + t.Run(tc.name, func(t *testing.T) { + var hasParam bool + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + hasParam = r.URL.Query().Has("versioning") + _, _ = fmt.Fprint(w, tc.body) + }) + + got, err := c.GetBucketVersioning(context.Background(), "b") + if err != nil { + t.Fatal(err) + } + if !hasParam { + t.Error("the request did not carry ?versioning") + } + if got != tc.want { + t.Errorf("status = %q, want %q", got, tc.want) + } + }) + } +} + +func TestSetBucketVersioning(t *testing.T) { + var gotMethod, gotBody, gotMD5 string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + assertSigned(t, r) + body, _ := io.ReadAll(r.Body) + gotMethod, gotBody, gotMD5 = r.Method, string(body), r.Header.Get("Content-MD5") + w.WriteHeader(http.StatusOK) + }) + + if err := c.SetBucketVersioning(context.Background(), "b", VersioningEnabled); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPut { + t.Errorf("method = %q, want PUT", gotMethod) + } + want := "Enabled" + if gotBody != want { + t.Errorf("body = %q, want %q", gotBody, want) + } + if gotMD5 == "" { + t.Error("Content-MD5 was not sent") + } +} + +// S3 has no call that returns a bucket to never-versioned, so the constant that +// means it must not be accepted here. +func TestSetBucketVersioningRejectsAnImpossibleState(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an invalid status must be refused before any request") + w.WriteHeader(http.StatusOK) + }) + + if err := c.SetBucketVersioning(context.Background(), "b", VersioningUnversioned); err == nil { + t.Fatal("Unversioned was accepted as a settable status") + } +} + +// A versioned listing pages on a key marker plus a version-id marker, and its +// and elements have to be presented together per key, +// newest first, rather than as two separate runs. +func TestListObjectVersionsPagesAndOrders(t *testing.T) { + var queries []string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + queries = append(queries, q.Get("key-marker")+"/"+q.Get("version-id-marker")) + if !q.Has("versions") { + t.Error("the request did not carry ?versions") + } + if q.Get("key-marker") == "" { + _, _ = fmt.Fprint(w, `true + dumpv1 + dumpv2true + 202026-09-14T10:00:00.000Z + dumpv3 + 2026-09-14T11:00:00.000Z + `) + return + } + _, _ = fmt.Fprint(w, `false + dumpv110 + 2026-09-14T09:00:00.000Z + `) + }) + + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Versions: true}) + if err != nil { + t.Fatal(err) + } + if len(queries) != 2 || queries[1] != "dump/v1" { + t.Errorf("requests = %v, want the second to carry both markers", queries) + } + if len(objs) != 3 { + t.Fatalf("entries = %d, want 3", len(objs)) + } + // Within a page: same key, newest first — the delete marker (11:00) before + // the version it hides (10:00). + if !objs[0].DeleteMarker || objs[0].VersionID != "v3" { + t.Errorf("first entry = %+v, want the newest delete marker", objs[0]) + } + if objs[1].VersionID != "v2" || !objs[1].IsLatest { + t.Errorf("second entry = %+v, want version v2", objs[1]) + } + if objs[2].VersionID != "v1" || objs[2].Size != 10 { + t.Errorf("third entry = %+v, want version v1 from the second page", objs[2]) + } +} + +// A delimited listing reports subtrees as CommonPrefixes, which is what turns a +// flat keyspace into one directory level. +func TestListObjectsWithADelimiter(t *testing.T) { + var gotDelimiter string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotDelimiter = r.URL.Query().Get("delimiter") + _, _ = fmt.Fprint(w, `false + 2026/ + 2025/ + latest.sql.gz7 + `) + }) + + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Delimiter: "/"}) + if err != nil { + t.Fatal(err) + } + if gotDelimiter != "/" { + t.Errorf("delimiter = %q, want /", gotDelimiter) + } + if len(objs) != 3 { + t.Fatalf("entries = %d, want 2 prefixes and 1 key", len(objs)) + } + // Subtrees first, so a delimited listing reads like a directory. + if !objs[0].IsPrefix || objs[0].Key != "2026/" { + t.Errorf("first entry = %+v, want the 2026/ prefix", objs[0]) + } + if !objs[1].IsPrefix { + t.Errorf("second entry = %+v, want a prefix", objs[1]) + } + if objs[2].IsPrefix || objs[2].Key != "latest.sql.gz" { + t.Errorf("third entry = %+v, want the key at this level", objs[2]) + } +} + +// --limit counts prefixes as well as keys: they are both entries the caller +// asked to see at most N of. +func TestListObjectsLimitCountsPrefixes(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprint(w, `false + a/ + b/ + c + `) + }) + + objs, err := c.ListObjects(context.Background(), "b", ListOptions{Delimiter: "/", Limit: 2}) + if err != nil { + t.Fatal(err) + } + if len(objs) != 2 { + t.Errorf("entries = %d, want the limit honoured", len(objs)) + } +} + +func TestHeadBucket(t *testing.T) { + var gotMethod, gotPath string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gotMethod, gotPath = r.Method, r.URL.Path + w.WriteHeader(http.StatusOK) + }) + + if err := c.HeadBucket(context.Background(), "db-backups"); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodHead || gotPath != "/db-backups" { + t.Errorf("request = %s %s, want HEAD /db-backups", gotMethod, gotPath) + } +} + +// A HEAD reply has no body, so a missing bucket is a bare 404 and IsNotFound +// has to recognise it from the status alone. +func TestHeadBucketNotFound(t *testing.T) { + c := newTestClient(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + err := c.HeadBucket(context.Background(), "absent") + if err == nil || !IsNotFound(err) { + t.Errorf("err = %v, want a recognised not-found", err) + } +} + +// A version ID must reach the request as a query parameter on the object's URL. +func TestVersionedObjectAddressing(t *testing.T) { + var queries []string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.Query().Get("versionId")) + w.Header().Set("x-amz-version-id", "v9") + w.WriteHeader(http.StatusOK) + }) + + info, err := c.HeadObject(context.Background(), "b", "k", "v9") + if err != nil { + t.Fatal(err) + } + if info.VersionID != "v9" { + t.Errorf("VersionID = %q, want v9 from the response header", info.VersionID) + } + if err := c.DeleteObject(context.Background(), "b", "k", "v9"); err != nil { + t.Fatal(err) + } + if _, err := c.GetObject(context.Background(), "b", "k", "v9", io.Discard); err != nil { + t.Fatal(err) + } + for i, got := range queries { + if got != "v9" { + t.Errorf("request %d carried versionId=%q, want v9", i, got) + } + } + if len(queries) != 3 { + t.Errorf("requests = %d, want head/delete/get", len(queries)) + } +} + +// The current object must not carry a versionId parameter at all: some stores +// reject an empty one. +func TestUnversionedObjectAddressingSendsNoParameter(t *testing.T) { + var raw string + c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + raw = r.URL.RawQuery + w.WriteHeader(http.StatusOK) + }) + + if _, err := c.HeadObject(context.Background(), "b", "k", ""); err != nil { + t.Fatal(err) + } + if strings.Contains(raw, "versionId") { + t.Errorf("query = %q, want no versionId", raw) + } +}