diff --git a/.agents/skills/use-ocis-cli/SKILL.md b/.agents/skills/use-ocis-cli/SKILL.md index e77d972..a7c20e1 100644 --- a/.agents/skills/use-ocis-cli/SKILL.md +++ b/.agents/skills/use-ocis-cli/SKILL.md @@ -1,6 +1,6 @@ --- name: use-ocis-cli -description: Safely operate oCIS servers through the installed ocis command-line client. Use when an AI agent is asked to inspect, list, search, transfer, synchronize, share, restore, inspect activity history, manage notifications, or administer files, Spaces, shares, users, or groups in oCIS. Do not use for developing the ocis-cli source code. +description: Safely operate oCIS servers through the installed ocis command-line client. Use when an AI agent is asked to inspect, list, search, transfer, archive, synchronize, share, restore, inspect activity history, manage notifications, or administer files, Spaces, shares, users, or groups in oCIS. Do not use for developing the ocis-cli source code. --- # Use oCIS CLI @@ -52,6 +52,9 @@ protocol-level work. ## Choose the operation - Use `upload` or `download` for a one-time transfer. +- Use `archive download` when the user wants the server to package one or more + remote resources into a single ZIP or TAR file. Check `archive formats` and + run with `--dry-run` before a broad archive download. - Use `sync push`, `sync pull`, or `sync bidirectional` for directory trees that should be reconciled. Run a sync with `--dry-run` first. - Use `share received` or `share overview` to inspect shares. Never accept a diff --git a/.agents/skills/use-ocis-cli/references/commands.md b/.agents/skills/use-ocis-cli/references/commands.md index e8efea0..b31b953 100644 --- a/.agents/skills/use-ocis-cli/references/commands.md +++ b/.agents/skills/use-ocis-cli/references/commands.md @@ -53,6 +53,8 @@ same command. | `rm, remove` | Move a remote resource to trash unless the selected operation states otherwise. | | `upload` | Transfer a local file or directory to oCIS. | | `download` | Transfer a remote file or directory to the local filesystem. | +| `archive download` | Ask the server to package selected remote resources into one local ZIP or TAR archive. | +| `archive formats` | List archive formats and source limits advertised by the server. | | `du` | Summarize logical remote file sizes. | | `search, find` | Search remote files and directories. | | `batch` | Execute reviewed file operations supplied as JSONL. | diff --git a/.claude/skills/use-ocis-cli/SKILL.md b/.claude/skills/use-ocis-cli/SKILL.md index e77d972..a7c20e1 100644 --- a/.claude/skills/use-ocis-cli/SKILL.md +++ b/.claude/skills/use-ocis-cli/SKILL.md @@ -1,6 +1,6 @@ --- name: use-ocis-cli -description: Safely operate oCIS servers through the installed ocis command-line client. Use when an AI agent is asked to inspect, list, search, transfer, synchronize, share, restore, inspect activity history, manage notifications, or administer files, Spaces, shares, users, or groups in oCIS. Do not use for developing the ocis-cli source code. +description: Safely operate oCIS servers through the installed ocis command-line client. Use when an AI agent is asked to inspect, list, search, transfer, archive, synchronize, share, restore, inspect activity history, manage notifications, or administer files, Spaces, shares, users, or groups in oCIS. Do not use for developing the ocis-cli source code. --- # Use oCIS CLI @@ -52,6 +52,9 @@ protocol-level work. ## Choose the operation - Use `upload` or `download` for a one-time transfer. +- Use `archive download` when the user wants the server to package one or more + remote resources into a single ZIP or TAR file. Check `archive formats` and + run with `--dry-run` before a broad archive download. - Use `sync push`, `sync pull`, or `sync bidirectional` for directory trees that should be reconciled. Run a sync with `--dry-run` first. - Use `share received` or `share overview` to inspect shares. Never accept a diff --git a/.claude/skills/use-ocis-cli/references/commands.md b/.claude/skills/use-ocis-cli/references/commands.md index e8efea0..b31b953 100644 --- a/.claude/skills/use-ocis-cli/references/commands.md +++ b/.claude/skills/use-ocis-cli/references/commands.md @@ -53,6 +53,8 @@ same command. | `rm, remove` | Move a remote resource to trash unless the selected operation states otherwise. | | `upload` | Transfer a local file or directory to oCIS. | | `download` | Transfer a remote file or directory to the local filesystem. | +| `archive download` | Ask the server to package selected remote resources into one local ZIP or TAR archive. | +| `archive formats` | List archive formats and source limits advertised by the server. | | `du` | Summarize logical remote file sizes. | | `search, find` | Search remote files and directories. | | `batch` | Execute reviewed file operations supplied as JSONL. | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9f6aec8..2ea6838 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -13,6 +13,7 @@ internal/ command/ Cobra command tree and input validation app/ application use-case orchestration apperror/ stable error categories and exit-code mapping + archiver/ authenticated archive-download protocol client auth/ OIDC protocol implementation config/ persisted profile model and atomic storage credentials/ OS credential-service adapter @@ -69,6 +70,9 @@ without starting a subprocess. contains shared application wiring. - `internal/apperror`: classify usage, authentication, not-found, and conflict errors without coupling application services to Cobra. +- `internal/archiver`: validate same-origin server-advertised archive endpoints, + stream authenticated ZIP or TAR responses, and verify completed archives + before the application atomically installs them at a local destination. - `internal/activities`: query account-wide or resource-scoped activity history through the bounded oCIS Graph extension, preserving the localized message template and structured variables returned by the server. @@ -143,7 +147,7 @@ without starting a subprocess. authentication headers, metadata and checksum response mapping, and safe scalar custom-property `PROPFIND`/`PROPPATCH` operations. -Protocol-specific behavior belongs in dedicated `internal/activities`, `internal/auth`, +Protocol-specific behavior belongs in dedicated `internal/activities`, `internal/archiver`, `internal/auth`, `internal/eventstream`, `internal/federation`, `internal/graph`, `internal/notifications`, `internal/search`, `internal/sharing`, `internal/trash`, `internal/versions`, and `internal/webdav` adapters. Recursive local/remote diff --git a/Makefile b/Makefile index b99aa63..740f2db 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ check: fmt coverage: go run ./tools/covercheck -min $(COVERAGE_MIN) \ - activities app auth eventstream federation graph httpapi notifications retry search sharing sync trash transfer versions \ + activities app archiver auth eventstream federation graph httpapi notifications retry search sharing sync trash transfer versions \ webdav fmt: diff --git a/README.md b/README.md index f4e01e9..ea1932b 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,8 @@ The current profile is marked with `*` in `server list`. Run `ocis doctor [PROFILE]` to validate the config schema, operating-system credential service, authentication, advertised WebDAV capabilities, Spaces, -public-link support, and resumable-upload support. +public-link support, resumable uploads, archive downloads, and real-time event +support. ## Spaces @@ -653,6 +654,36 @@ directory. For example, `ocis download /demo ./ --recursive` creates `./demo/demo`. A destination that does not exist is created as the downloaded directory itself. +## Archive downloads + +Ask the oCIS server to package one or more files and directories into a single +ZIP or TAR download: + +```sh +ocis archive formats +ocis archive download /Documents /Photos/trip.jpg --output backup.zip +ocis archive download /Documents --output backup.tar --format tar +ocis archive download /Documents --output backup.zip --dry-run +``` + +The command is different from `download --recursive`: oCIS creates one archive +on the server and the CLI downloads that one stream. `archive formats` shows +the formats and source limits advertised by the selected server. The output +format is inferred from `.tar`; otherwise it defaults to ZIP. A conflicting +`.zip` or `.tar` extension is rejected. + +Before downloading, the CLI resolves every selected resource, walks selected +directories, and checks the advertised entry-count and logical-size limits. +Nested or duplicate selections are rejected so resources are not archived +twice. `--dry-run` performs that complete preflight without requesting an +archive or creating a local file. The server remains authoritative for access +permissions, including within a selected Space. + +Archive downloads use the same authenticated profile as other commands. The +CLI refuses a cross-origin archive endpoint advertised by a server, validates +the complete ZIP or TAR stream, and atomically installs the result. Existing +destinations are protected unless `--overwrite` is explicitly supplied. + ## One-way synchronization Reconcile complete directory trees from one authoritative source: diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 065f1b2..9f0a521 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -302,6 +302,10 @@ func TestDoctorValidatesProfileAndCapabilities(t *testing.T) { case request.URL.Path == "/ocs/v2.php/cloud/capabilities": writeAppOCS(writer, `{"capabilities":{ "core":{"support-sse":true}, + "files":{"archivers":[{ + "enabled":true,"version":"2.0.0","formats":["zip","tar"], + "archiver_url":"/archiver","max_num_files":"1000","max_size":"1000000" + }]}, "files_sharing":{"api_enabled":true,"public":{ "enabled":true,"password":{"enforced":false}, "expire_date":{"enabled":true} @@ -338,6 +342,7 @@ func TestDoctorValidatesProfileAndCapabilities(t *testing.T) { if !strings.Contains(rendered.String(), `"type": "diagnostic"`) || !strings.Contains(rendered.String(), `"DAV capabilities"`) || !strings.Contains(rendered.String(), `"public links"`) || + !strings.Contains(rendered.String(), `"archive downloads"`) || !strings.Contains(rendered.String(), `"real-time events"`) { t.Fatalf("output: %s", rendered.String()) } diff --git a/internal/app/archive_api.go b/internal/app/archive_api.go new file mode 100644 index 0000000..acee6a7 --- /dev/null +++ b/internal/app/archive_api.go @@ -0,0 +1,40 @@ +package app + +import "context" + +// ArchiveDownloadRequest describes one server-side archive download. +type ArchiveDownloadRequest struct { + Paths []string + Destination string + Format string + Overwrite bool + DryRun bool +} + +// RunArchiveDownloadWithOptions creates and downloads a server-side archive. +func RunArchiveDownloadWithOptions( + ctx context.Context, + request ArchiveDownloadRequest, + selectedProfile string, + options RunOptions, +) error { + return classifyProtocolError( + "archive download", + runArchiveDownload( + ctx, request, selectedProfile, options.normalized(), + ), + ) +} + +// RunArchiveFormatsWithOptions lists formats advertised by the selected +// server's preferred enabled archive service. +func RunArchiveFormatsWithOptions( + ctx context.Context, + selectedProfile string, + options RunOptions, +) error { + return classifyProtocolError( + "archive formats", + runArchiveFormats(ctx, selectedProfile, options.normalized()), + ) +} diff --git a/internal/app/archive_service.go b/internal/app/archive_service.go new file mode 100644 index 0000000..bcb4c49 --- /dev/null +++ b/internal/app/archive_service.go @@ -0,0 +1,546 @@ +package app + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/mzner/ocis-cli/internal/apperror" + archiveclient "github.com/mzner/ocis-cli/internal/archiver" + appoutput "github.com/mzner/ocis-cli/internal/output" + "github.com/mzner/ocis-cli/internal/sharing" + "github.com/mzner/ocis-cli/internal/transfer" + "golang.org/x/term" +) + +// ArchiveFormat reports one usable format and the limits shared by the +// selected archive service. +type ArchiveFormat struct { + Format string `json:"format"` + Version string `json:"version"` + MaxNumFiles int64 `json:"maxNumFiles,omitempty"` + MaxSize int64 `json:"maxSize,omitempty"` +} + +// ArchiveResource is one selected archive root. +type ArchiveResource struct { + Path string `json:"path"` + ResourceID string `json:"resourceId"` + Type string `json:"type"` +} + +// ArchiveResult describes archive preflight and completed download state. +type ArchiveResult struct { + Resources []ArchiveResource `json:"resources"` + Destination string `json:"destination"` + Format string `json:"format"` + Entries int64 `json:"entries"` + Files int64 `json:"files"` + Directories int64 `json:"directories"` + LogicalBytes int64 `json:"logicalBytes"` + ArchiveBytes int64 `json:"archiveBytes,omitempty"` + DryRun bool `json:"dryRun,omitempty"` +} + +func runArchiveFormats( + ctx context.Context, selectedProfile string, options RunOptions, +) error { + client, err := newClientWithOptions(ctx, selectedProfile, options) + if err != nil { + return err + } + capability, err := discoverArchiver(ctx, client) + if err != nil { + return err + } + formats := archiveFormats(capability) + if options.OutputMode != appoutput.Human { + return writeOutput(options, "archive-format", formats) + } + writer := tabwriter.NewWriter(options.Out, 0, 4, 2, ' ', 0) + if _, err := fmt.Fprintln( + writer, "FORMAT\tVERSION\tMAX ENTRIES\tMAX SOURCE BYTES", + ); err != nil { + return err + } + for _, value := range formats { + if _, err := fmt.Fprintf( + writer, "%s\t%s\t%s\t%s\n", value.Format, value.Version, + limitText(value.MaxNumFiles), limitText(value.MaxSize), + ); err != nil { + return err + } + } + return writer.Flush() +} + +func runArchiveDownload( + ctx context.Context, + request ArchiveDownloadRequest, + selectedProfile string, + options RunOptions, +) error { + if len(request.Paths) == 0 { + return archiveUsage("select at least one remote path") + } + if strings.TrimSpace(request.Destination) == "" { + return archiveUsage("--output is required") + } + if request.Destination == "-" { + return archiveUsage("archive output must be a local file, not stdout") + } + paths, err := normalizeArchivePaths(request.Paths) + if err != nil { + return archiveUsage(err.Error()) + } + format, err := resolveArchiveFormat(request.Format, request.Destination) + if err != nil { + return archiveUsage(err.Error()) + } + if err := validateArchiveDestination( + request.Destination, request.Overwrite, + ); err != nil { + return err + } + + client, err := newClientWithOptions(ctx, selectedProfile, options) + if err != nil { + return err + } + if err := client.selectSpace(options.Space); err != nil { + return err + } + capability, err := discoverArchiver(ctx, client) + if err != nil { + return err + } + if !containsArchiveFormat(capability.Formats, format) { + return archiveUsage(fmt.Sprintf( + "server archive service does not support %s; available formats: %s", + format, strings.Join(normalizeFormats(capability.Formats), ", "), + )) + } + result := ArchiveResult{ + Destination: request.Destination, Format: format, + Resources: make([]ArchiveResource, 0, len(paths)), + } + for _, remote := range paths { + if err := addArchiveResource(client, remote, capability, &result); err != nil { + return err + } + } + if request.DryRun { + result.DryRun = true + return writeArchiveResult(result, options) + } + + protocol, err := client.archiverClient(capability.URL) + if err != nil { + return fmt.Errorf("configure archive download: %w", err) + } + file, err := os.CreateTemp( + filepath.Dir(request.Destination), ".ocis-archive-*.part", + ) + if err != nil { + return fmt.Errorf("create archive temporary file: %w", err) + } + temporary := file.Name() + committed := false + defer func() { + _ = file.Close() + if !committed { + _ = os.Remove(temporary) + } + }() + progress, finishProgress := archiveProgressReporter( + options, request.Destination, + ) + downloaded, err := protocol.Download( + ctx, archiveclient.DownloadRequest{ + ResourceIDs: archiveResourceIDs(result.Resources), Format: format, + }, file, progress, + ) + finishProgress(downloaded.Bytes) + if err != nil { + return err + } + if err := file.Sync(); err != nil { + return fmt.Errorf("sync archive temporary file: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close archive temporary file: %w", err) + } + if err := archiveclient.ValidateFile( + temporary, format, archiveclient.ValidationLimits{ + MaxEntries: capability.MaxNumFiles, + MaxBytes: capability.MaxSize, + }, + ); err != nil { + return err + } + if err := transfer.CommitFile( + temporary, request.Destination, request.Overwrite, + ); err != nil { + if errors.Is(err, transfer.ErrDestinationExists) { + return apperror.Wrap(apperror.KindConflict, "archive download", err) + } + return err + } + committed = true + result.ArchiveBytes = downloaded.Bytes + return writeArchiveResult(result, options) +} + +func discoverArchiver( + ctx context.Context, client *client, +) (sharing.ArchiverCapabilities, error) { + capabilities, err := client.sharingClient().Capabilities(ctx) + if err != nil { + return sharing.ArchiverCapabilities{}, fmt.Errorf( + "discover archive service: %w", err, + ) + } + return selectArchiver(capabilities.Files.Archivers) +} + +func selectArchiver( + capabilities []sharing.ArchiverCapabilities, +) (sharing.ArchiverCapabilities, error) { + var selected *sharing.ArchiverCapabilities + for index := range capabilities { + value := &capabilities[index] + if !value.Enabled || strings.TrimSpace(value.Version) == "" || + strings.TrimSpace(value.URL) == "" || len(normalizeFormats(value.Formats)) == 0 { + continue + } + if selected == nil || compareArchiveVersions(value.Version, selected.Version) > 0 { + selected = value + } + } + if selected == nil { + return sharing.ArchiverCapabilities{}, errors.New( + "server does not advertise an enabled archive service", + ) + } + result := *selected + result.Formats = normalizeFormats(result.Formats) + return result, nil +} + +func addArchiveResource( + client *client, + remote string, + capability sharing.ArchiverCapabilities, + result *ArchiveResult, +) error { + root, err := client.stat(remote) + if err != nil { + return err + } + if root.ResourceID == "" { + return fmt.Errorf( + "server did not return a stable resource ID for %s", remote, + ) + } + for _, existing := range result.Resources { + if existing.ResourceID == root.ResourceID { + return archiveUsage(fmt.Sprintf( + "%s resolves to the same resource as %s", + remote, existing.Path, + )) + } + } + result.Resources = append(result.Resources, ArchiveResource{ + Path: remote, ResourceID: root.ResourceID, Type: root.Type, + }) + includeRoot := remote != "/" + return scanArchiveItem(client, root, includeRoot, capability, result) +} + +func scanArchiveItem( + client *client, + value item, + include bool, + capability sharing.ArchiverCapabilities, + result *ArchiveResult, +) error { + if include { + result.Entries++ + if value.Type == "directory" { + result.Directories++ + } else { + result.Files++ + if value.Size > 0 && result.LogicalBytes > math.MaxInt64-value.Size { + return archiveUsage("selected source size exceeds the supported integer range") + } + result.LogicalBytes += value.Size + } + if capability.MaxNumFiles > 0 && result.Entries > capability.MaxNumFiles { + return archiveUsage(fmt.Sprintf( + "selection contains more than the server limit of %d archive entries", + capability.MaxNumFiles, + )) + } + if capability.MaxSize > 0 && result.LogicalBytes > capability.MaxSize { + return archiveUsage(fmt.Sprintf( + "selection exceeds the server limit of %d source bytes", + capability.MaxSize, + )) + } + } + if value.Type != "directory" { + return nil + } + children, err := client.list(value.Path) + if err != nil { + return err + } + for _, child := range children { + if err := scanArchiveItem( + client, child, true, capability, result, + ); err != nil { + return err + } + } + return nil +} + +func normalizeArchivePaths(values []string) ([]string, error) { + result := make([]string, len(values)) + for index, value := range values { + if strings.TrimSpace(value) == "" { + return nil, errors.New("remote archive path cannot be empty") + } + result[index] = cleanRemote(value) + } + for left := range result { + for right := left + 1; right < len(result); right++ { + if result[left] == result[right] { + return nil, fmt.Errorf("remote path %s is selected more than once", result[left]) + } + if archivePathContains(result[left], result[right]) || + archivePathContains(result[right], result[left]) { + return nil, fmt.Errorf( + "nested selections %s and %s would duplicate archive entries", + result[left], result[right], + ) + } + } + } + return result, nil +} + +func archivePathContains(parent, child string) bool { + if parent == "/" { + return child != "/" + } + return strings.HasPrefix(child, parent+"/") +} + +func resolveArchiveFormat(requested, destination string) (string, error) { + format := strings.ToLower(strings.TrimSpace(requested)) + extension := strings.ToLower(filepath.Ext(destination)) + if format == "" { + if extension == ".tar" { + return "tar", nil + } + return "zip", nil + } + if format != "zip" && format != "tar" { + return "", fmt.Errorf("--format must be zip or tar, got %q", requested) + } + if (extension == ".zip" || extension == ".tar") && + extension != "."+format { + return "", fmt.Errorf( + "--format %s conflicts with destination extension %s", + format, extension, + ) + } + return format, nil +} + +func validateArchiveDestination(destination string, overwrite bool) error { + info, err := os.Lstat(destination) + switch { + case err == nil && info.IsDir(): + return apperror.Wrap( + apperror.KindConflict, "archive download", + fmt.Errorf("destination is a directory: %s", destination), + ) + case err == nil && !overwrite: + return apperror.Wrap( + apperror.KindConflict, "archive download", + fmt.Errorf("destination already exists: %s; pass --overwrite to replace it", destination), + ) + case err == nil: + return nil + case errors.Is(err, os.ErrNotExist): + return nil + default: + return fmt.Errorf("inspect archive destination: %w", err) + } +} + +func archiveFormats(capability sharing.ArchiverCapabilities) []ArchiveFormat { + formats := make([]ArchiveFormat, 0, len(capability.Formats)) + for _, format := range normalizeFormats(capability.Formats) { + formats = append(formats, ArchiveFormat{ + Format: format, Version: capability.Version, + MaxNumFiles: capability.MaxNumFiles, MaxSize: capability.MaxSize, + }) + } + return formats +} + +func normalizeFormats(values []string) []string { + seen := make(map[string]bool) + result := make([]string, 0, len(values)) + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if (value == "zip" || value == "tar") && !seen[value] { + seen[value] = true + result = append(result, value) + } + } + sort.Strings(result) + return result +} + +func containsArchiveFormat(values []string, selected string) bool { + for _, value := range normalizeFormats(values) { + if value == selected { + return true + } + } + return false +} + +func compareArchiveVersions(left, right string) int { + leftParts := strings.Split(strings.TrimPrefix(strings.ToLower(left), "v"), ".") + rightParts := strings.Split(strings.TrimPrefix(strings.ToLower(right), "v"), ".") + for index := 0; index < max(len(leftParts), len(rightParts)); index++ { + leftNumber, rightNumber := 0, 0 + if index < len(leftParts) { + leftNumber, _ = strconv.Atoi(strings.SplitN(leftParts[index], "-", 2)[0]) + } + if index < len(rightParts) { + rightNumber, _ = strconv.Atoi(strings.SplitN(rightParts[index], "-", 2)[0]) + } + if leftNumber < rightNumber { + return -1 + } + if leftNumber > rightNumber { + return 1 + } + } + return strings.Compare(left, right) +} + +func archiveResourceIDs(values []ArchiveResource) []string { + result := make([]string, len(values)) + for index, value := range values { + result[index] = value.ResourceID + } + return result +} + +func archiveProgressReporter( + options RunOptions, destination string, +) (func(int64), func(int64)) { + if options.Quiet || options.OutputMode != appoutput.Human { + return nil, func(int64) {} + } + terminalOutput := false + if descriptor, ok := options.Err.(interface{ Fd() uintptr }); ok { + terminalOutput = term.IsTerminal(int(descriptor.Fd())) + } + lastUpdate := time.Time{} + lastBytes := int64(-1) + update := func(written int64) { + now := time.Now() + if written == lastBytes || now.Sub(lastUpdate) < 200*time.Millisecond { + return + } + lastUpdate, lastBytes = now, written + if terminalOutput { + _, _ = fmt.Fprintf( + options.Err, "\rArchive download: %d bytes written to %s", + written, destination, + ) + } + } + finish := func(written int64) { + if terminalOutput { + _, _ = fmt.Fprintf( + options.Err, "\rArchive download: %d bytes written to %s\n", + written, destination, + ) + } + } + return update, finish +} + +func archiverCapabilityDetail(capabilities sharing.Capabilities) string { + selected, err := selectArchiver(capabilities.Files.Archivers) + if err != nil { + return "not advertised" + } + details := []string{ + "version " + selected.Version, + "formats " + strings.Join(selected.Formats, ", "), + } + if selected.MaxNumFiles > 0 { + details = append(details, fmt.Sprintf( + "maximum %d entries", selected.MaxNumFiles, + )) + } + if selected.MaxSize > 0 { + details = append(details, fmt.Sprintf( + "maximum %d source bytes", selected.MaxSize, + )) + } + return strings.Join(details, "; ") +} + +func writeArchiveResult(result ArchiveResult, options RunOptions) error { + if options.OutputMode != appoutput.Human { + return writeOutput(options, "archive", result) + } + if result.DryRun { + _, err := fmt.Fprintf( + options.Out, + "Would archive %d entries (%d source bytes) from %d selection(s) to %s as %s\n", + result.Entries, result.LogicalBytes, len(result.Resources), + result.Destination, result.Format, + ) + return err + } + _, err := fmt.Fprintf( + options.Out, + "Downloaded %d entries (%d source bytes) to %s as %s (%d archive bytes)\n", + result.Entries, result.LogicalBytes, result.Destination, + result.Format, result.ArchiveBytes, + ) + return err +} + +func archiveUsage(message string) error { + return apperror.Wrap( + apperror.KindUsage, "archive download", errors.New(message), + ) +} + +func limitText(value int64) string { + if value <= 0 { + return "not advertised" + } + return strconv.FormatInt(value, 10) +} diff --git a/internal/app/archive_service_test.go b/internal/app/archive_service_test.go new file mode 100644 index 0000000..d0cd437 --- /dev/null +++ b/internal/app/archive_service_test.go @@ -0,0 +1,240 @@ +package app + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/mzner/ocis-cli/internal/apperror" + appoutput "github.com/mzner/ocis-cli/internal/output" +) + +const archiveDirectoryStat = ` + + /remote.php/dav/files/alice/reports/ + HTTP/1.1 200 OK + reports + + storage$space!reports + + +` + +const archiveDirectoryList = ` + + /remote.php/dav/files/alice/reports/ + HTTP/1.1 200 OK + reports + + storage$space!reports + + + /remote.php/dav/files/alice/reports/report.txt + HTTP/1.1 200 OK + report.txt + 5 + storage$space!report + + +` + +func TestArchiveDownloadAndDryRun(t *testing.T) { + payload := appArchiveZIP(t, "reports/report.txt", "hello") + var archiveRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func( + writer http.ResponseWriter, request *http.Request, + ) { + switch { + case request.URL.Path == "/ocs/v2.php/cloud/capabilities": + writeAppOCS(writer, `{"capabilities":{"files":{"archivers":[{ + "enabled":true,"version":"2.0.0","formats":["zip","tar"], + "archiver_url":"/archiver","max_num_files":"10","max_size":"100" + }]}}}`) + case request.Method == "PROPFIND" && request.Header.Get("Depth") == "0": + writer.WriteHeader(http.StatusMultiStatus) + _, _ = io.WriteString(writer, archiveDirectoryStat) + case request.Method == "PROPFIND" && request.Header.Get("Depth") == "1": + writer.WriteHeader(http.StatusMultiStatus) + _, _ = io.WriteString(writer, archiveDirectoryList) + case request.URL.Path == "/archiver": + archiveRequests.Add(1) + if request.URL.Query().Get("id") != "storage$space!reports" || + request.URL.Query().Get("output-format") != "zip" { + t.Fatalf("archive query: %s", request.URL.RawQuery) + } + _, _ = writer.Write(payload) + default: + t.Fatalf("unexpected request: %s %s depth=%s", request.Method, request.URL.Path, request.Header.Get("Depth")) + } + })) + defer server.Close() + configureSpaceTestProfile(t, server.URL, "") + destination := filepath.Join(t.TempDir(), "reports.zip") + + var dryOutput bytes.Buffer + if err := RunArchiveDownloadWithOptions( + context.Background(), ArchiveDownloadRequest{ + Paths: []string{"/reports"}, Destination: destination, DryRun: true, + }, "", RunOptions{Out: &dryOutput, Err: io.Discard}, + ); err != nil { + t.Fatal(err) + } + if archiveRequests.Load() != 0 || + !strings.Contains(dryOutput.String(), "Would archive 2 entries (5 source bytes)") { + t.Fatalf("requests=%d output=%q", archiveRequests.Load(), dryOutput.String()) + } + if _, err := os.Stat(destination); !os.IsNotExist(err) { + t.Fatalf("dry-run created destination: %v", err) + } + + var output bytes.Buffer + if err := RunArchiveDownloadWithOptions( + context.Background(), ArchiveDownloadRequest{ + Paths: []string{"reports"}, Destination: destination, + }, "", RunOptions{ + Out: &output, Err: io.Discard, OutputMode: appoutput.JSON, + }, + ); err != nil { + t.Fatal(err) + } + if archiveRequests.Load() != 1 { + t.Fatalf("archive requests: %d", archiveRequests.Load()) + } + if data, err := os.ReadFile(destination); err != nil || !bytes.Equal(data, payload) { + t.Fatalf("archive bytes=%d error=%v", len(data), err) + } + var envelope appoutput.Envelope + if err := json.Unmarshal(output.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Type != "archive" || !strings.Contains(output.String(), `"entries": 2`) || + !strings.Contains(output.String(), `"archiveBytes":`) { + t.Fatalf("output: %s", output.String()) + } +} + +func TestArchiveFormatsAndPreferredVersion(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func( + writer http.ResponseWriter, _ *http.Request, + ) { + writeAppOCS(writer, `{"capabilities":{"files":{"archivers":[ + {"enabled":true,"version":"1.0.0","formats":["zip"], + "archiver_url":"/old","max_num_files":"5","max_size":"50"}, + {"enabled":true,"version":"2.0.0","formats":["tar","zip"], + "archiver_url":"/archiver","max_num_files":"10","max_size":"100"}, + {"enabled":false,"version":"3.0.0","formats":["zip"], + "archiver_url":"/disabled"} + ]}}}`) + })) + defer server.Close() + configureSpaceTestProfile(t, server.URL, "") + var output bytes.Buffer + if err := RunArchiveFormatsWithOptions( + context.Background(), "", RunOptions{Out: &output}, + ); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "FORMAT") || + !strings.Contains(output.String(), "tar") || + !strings.Contains(output.String(), "2.0.0") || + strings.Contains(output.String(), "1.0.0") { + t.Fatalf("output: %q", output.String()) + } +} + +func TestArchiveValidationFailsBeforeProfileLoad(t *testing.T) { + t.Setenv("OCIS_CONFIG", filepath.Join(t.TempDir(), "missing", "config.json")) + tests := []ArchiveDownloadRequest{ + {Destination: "archive.zip"}, + {Paths: []string{"/reports"}}, + {Paths: []string{"/reports"}, Destination: "-"}, + {Paths: []string{"/reports"}, Destination: "archive.zip", Format: "tar"}, + {Paths: []string{"/reports", "/reports/file"}, Destination: "archive.zip"}, + } + for _, request := range tests { + err := RunArchiveDownloadWithOptions( + context.Background(), request, "", RunOptions{Out: io.Discard}, + ) + if !apperror.IsKind(err, apperror.KindUsage) { + t.Fatalf("request=%#v error=%v", request, err) + } + } +} + +func TestArchiveDownloadRefusesExistingDestinationBeforeNetwork(t *testing.T) { + destination := filepath.Join(t.TempDir(), "archive.zip") + if err := os.WriteFile(destination, []byte("existing"), 0600); err != nil { + t.Fatal(err) + } + err := RunArchiveDownloadWithOptions( + context.Background(), ArchiveDownloadRequest{ + Paths: []string{"/reports"}, Destination: destination, + }, "", RunOptions{Out: io.Discard}, + ) + if !apperror.IsKind(err, apperror.KindConflict) || + !strings.Contains(err.Error(), "--overwrite") { + t.Fatalf("error: %v", err) + } + if data, readErr := os.ReadFile(destination); readErr != nil || string(data) != "existing" { + t.Fatalf("destination=%q error=%v", data, readErr) + } +} + +func TestArchiveCapabilityLimitsFailBeforeDownload(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func( + writer http.ResponseWriter, request *http.Request, + ) { + switch { + case request.URL.Path == "/ocs/v2.php/cloud/capabilities": + writeAppOCS(writer, `{"capabilities":{"files":{"archivers":[{ + "enabled":true,"version":"2.0.0","formats":["zip"], + "archiver_url":"/archiver","max_num_files":"1","max_size":"4" + }]}}}`) + case request.Method == "PROPFIND" && request.Header.Get("Depth") == "0": + writer.WriteHeader(http.StatusMultiStatus) + _, _ = io.WriteString(writer, archiveDirectoryStat) + case request.Method == "PROPFIND" && request.Header.Get("Depth") == "1": + writer.WriteHeader(http.StatusMultiStatus) + _, _ = io.WriteString(writer, archiveDirectoryList) + case request.URL.Path == "/archiver": + t.Fatal("archive request sent despite preflight limit") + } + })) + defer server.Close() + configureSpaceTestProfile(t, server.URL, "") + err := RunArchiveDownloadWithOptions( + context.Background(), ArchiveDownloadRequest{ + Paths: []string{"/reports"}, Destination: filepath.Join(t.TempDir(), "archive.zip"), + }, "", RunOptions{Out: io.Discard, Err: io.Discard}, + ) + if !apperror.IsKind(err, apperror.KindUsage) || + !strings.Contains(err.Error(), "server limit") { + t.Fatalf("error: %v", err) + } +} + +func appArchiveZIP(t *testing.T, name, content string) []byte { + t.Helper() + var output bytes.Buffer + archive := zip.NewWriter(&output) + entry, err := archive.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(entry, content); err != nil { + t.Fatal(err) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + return output.Bytes() +} diff --git a/internal/app/doctor.go b/internal/app/doctor.go index de4dcc1..33c12f7 100644 --- a/internal/app/doctor.go +++ b/internal/app/doctor.go @@ -97,6 +97,14 @@ func RunDoctorWithOptions( Name: "resumable uploads", Status: tusStatus, Detail: resumableUploadCapabilityDetail(features), }) + archiveStatus := "unsupported" + if _, err := selectArchiver(features.Files.Archivers); err == nil { + archiveStatus = "ok" + } + checks = append(checks, DoctorCheck{ + Name: "archive downloads", Status: archiveStatus, + Detail: archiverCapabilityDetail(features), + }) eventStatus := "unsupported" if features.Core.SupportSSE { eventStatus = "ok" diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 9e06b68..ef42bbc 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -11,6 +11,7 @@ import ( "github.com/mzner/ocis-cli/internal/activities" "github.com/mzner/ocis-cli/internal/apperror" + archiveclient "github.com/mzner/ocis-cli/internal/archiver" "github.com/mzner/ocis-cli/internal/auth" appconfig "github.com/mzner/ocis-cli/internal/config" "github.com/mzner/ocis-cli/internal/credentials" @@ -63,6 +64,12 @@ func (client *client) activitiesClient() *activities.Client { return client.activities } +func (client *client) archiverClient( + endpoint string, +) (*archiveclient.Client, error) { + return archiveclient.NewClient(client.apiConfig(), endpoint, client.http) +} + func (client *client) eventStreamClient() *eventstream.Client { if client.events == nil { config := client.apiConfig() diff --git a/internal/archiver/client.go b/internal/archiver/client.go new file mode 100644 index 0000000..9d89279 --- /dev/null +++ b/internal/archiver/client.go @@ -0,0 +1,158 @@ +// Package archiver implements authenticated downloads from the oCIS archive +// service. Capability selection and local-file policy belong to the +// application layer. +package archiver + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/mzner/ocis-cli/internal/httpapi" +) + +// DownloadRequest identifies resources and the requested archive container. +type DownloadRequest struct { + ResourceIDs []string + Format string +} + +// DownloadResult reports bytes received from the archive service. +type DownloadResult struct { + Bytes int64 +} + +// Client downloads archives from one same-origin advertised endpoint. +type Client struct { + api *httpapi.Client + resource string +} + +// NewClient validates endpoint before constructing an authenticated client. +// Credentials are never sent to a cross-origin capability URL. +func NewClient( + config httpapi.Config, endpoint string, httpClient *http.Client, +) (*Client, error) { + resource, err := sameOriginResource(config.Server, endpoint) + if err != nil { + return nil, err + } + return &Client{ + api: httpapi.NewClient(config, httpClient), resource: resource, + }, nil +} + +// Download streams one archive and reports cumulative bytes written. +func (client *Client) Download( + ctx context.Context, + request DownloadRequest, + destination io.Writer, + progress func(int64), +) (DownloadResult, error) { + if len(request.ResourceIDs) == 0 { + return DownloadResult{}, errors.New("archive resource list cannot be empty") + } + format := strings.ToLower(strings.TrimSpace(request.Format)) + if format != "zip" && format != "tar" { + return DownloadResult{}, fmt.Errorf("unsupported archive format %q", request.Format) + } + resourceURL, err := url.Parse(client.resource) + if err != nil { + return DownloadResult{}, fmt.Errorf("parse archive endpoint: %w", err) + } + query := resourceURL.Query() + for _, resourceID := range request.ResourceIDs { + if strings.TrimSpace(resourceID) == "" { + return DownloadResult{}, errors.New("archive resource ID cannot be empty") + } + query.Add("id", resourceID) + } + query.Set("output-format", format) + resourceURL.RawQuery = query.Encode() + headers := make(http.Header) + if format == "zip" { + headers.Set("Accept", "application/zip") + } else { + headers.Set("Accept", "application/x-tar") + } + response, err := client.api.Do( + ctx, http.MethodGet, resourceURL.String(), nil, headers, + ) + if err != nil { + return DownloadResult{}, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode < http.StatusOK || + response.StatusCode >= http.StatusMultipleChoices { + return DownloadResult{}, httpapi.ResponseError(response) + } + counter := &progressWriter{destination: destination, progress: progress} + if _, err := io.Copy(counter, response.Body); err != nil { + if ctx.Err() != nil { + return DownloadResult{Bytes: counter.written}, ctx.Err() + } + return DownloadResult{Bytes: counter.written}, fmt.Errorf("download archive: %w", err) + } + if progress != nil { + progress(counter.written) + } + return DownloadResult{Bytes: counter.written}, nil +} + +type progressWriter struct { + destination io.Writer + progress func(int64) + written int64 +} + +func (writer *progressWriter) Write(data []byte) (int, error) { + written, err := writer.destination.Write(data) + writer.written += int64(written) + if writer.progress != nil { + writer.progress(writer.written) + } + return written, err +} + +func sameOriginResource(server, endpoint string) (string, error) { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "", errors.New("server advertised an empty archive URL") + } + base, err := url.Parse(server) + if err != nil { + return "", fmt.Errorf("parse server URL: %w", err) + } + advertised, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("parse advertised archive URL: %w", err) + } + resolved := base.ResolveReference(advertised) + if resolved.Scheme != "http" && resolved.Scheme != "https" { + return "", fmt.Errorf( + "archive URL uses unsupported scheme %q", resolved.Scheme, + ) + } + if resolved.User != nil || resolved.Fragment != "" { + return "", errors.New("archive URL must not contain user information or a fragment") + } + if !strings.EqualFold(resolved.Scheme, base.Scheme) || + !strings.EqualFold(resolved.Host, base.Host) { + return "", fmt.Errorf( + "refusing cross-origin archive URL %s; authenticated archive endpoints must use %s://%s", + resolved.Redacted(), base.Scheme, base.Host, + ) + } + resource := resolved.EscapedPath() + if resource == "" { + resource = "/" + } + if resolved.RawQuery != "" { + resource += "?" + resolved.RawQuery + } + return resource, nil +} diff --git a/internal/archiver/client_test.go b/internal/archiver/client_test.go new file mode 100644 index 0000000..f72d579 --- /dev/null +++ b/internal/archiver/client_test.go @@ -0,0 +1,275 @@ +package archiver + +import ( + "archive/tar" + "archive/zip" + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/mzner/ocis-cli/internal/httpapi" +) + +func TestDownloadAuthenticatesAndStreamsSelectedIDs(t *testing.T) { + payload := testZIP(t, map[string]string{"reports/report.txt": "hello"}) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func( + writer http.ResponseWriter, request *http.Request, + ) { + if attempts.Add(1) == 1 { + writer.WriteHeader(http.StatusServiceUnavailable) + return + } + if request.URL.Path != "/archiver" || + request.URL.Query().Get("existing") != "value" || + request.URL.Query().Get("output-format") != "zip" || + request.Header.Get("Authorization") != "Bearer token" || + request.Header.Get("Accept") != "application/zip" { + t.Fatalf("request: %s headers=%v", request.URL.String(), request.Header) + } + ids := request.URL.Query()["id"] + if len(ids) != 2 || ids[0] != "storage$space!one" || + ids[1] != "storage$space!two" { + t.Fatalf("IDs: %v", ids) + } + _, _ = writer.Write(payload) + })) + defer server.Close() + client, err := NewClient(httpapi.Config{ + Server: server.URL, AuthType: "oidc", AccessToken: "token", Retries: 1, + RetryWait: 1, + }, "/archiver?existing=value", server.Client()) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + var progress int64 + result, err := client.Download( + context.Background(), DownloadRequest{ + ResourceIDs: []string{"storage$space!one", "storage$space!two"}, + Format: "zip", + }, &output, func(written int64) { progress = written }, + ) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(output.Bytes(), payload) || result.Bytes != int64(len(payload)) || + progress != int64(len(payload)) || attempts.Load() != 2 { + t.Fatalf( + "bytes=%d progress=%d attempts=%d", result.Bytes, progress, attempts.Load(), + ) + } +} + +func TestNewClientRejectsUnsafeAdvertisedURLs(t *testing.T) { + for _, endpoint := range []string{ + "https://attacker.example/archiver", + "file:///tmp/archive", + "https://user@example.test/archiver", + "https://example.test/archiver#fragment", + "", + " ", + } { + _, err := NewClient( + httpapi.Config{Server: "https://example.test"}, endpoint, nil, + ) + if err == nil { + t.Fatalf("unsafe endpoint accepted: %q", endpoint) + } + } + client, err := NewClient( + httpapi.Config{Server: "https://example.test"}, + "https://example.test/archiver", nil, + ) + if err != nil || client.resource != "/archiver" { + t.Fatalf("same-origin endpoint: %#v, %v", client, err) + } +} + +func TestDownloadValidationAndHTTPFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func( + writer http.ResponseWriter, _ *http.Request, + ) { + writer.WriteHeader(http.StatusRequestEntityTooLarge) + _, _ = io.WriteString(writer, "reached max total files size") + })) + defer server.Close() + client, err := NewClient( + httpapi.Config{Server: server.URL}, "/archiver", server.Client(), + ) + if err != nil { + t.Fatal(err) + } + for _, request := range []DownloadRequest{ + {Format: "zip"}, + {ResourceIDs: []string{"id"}, Format: "rar"}, + {ResourceIDs: []string{""}, Format: "zip"}, + } { + if _, err := client.Download( + context.Background(), request, io.Discard, nil, + ); err == nil { + t.Fatalf("invalid request accepted: %#v", request) + } + } + _, err = client.Download( + context.Background(), DownloadRequest{ + ResourceIDs: []string{"id"}, Format: "zip", + }, io.Discard, nil, + ) + if err == nil || !strings.Contains(err.Error(), "413") || + !strings.Contains(err.Error(), "max total") { + t.Fatalf("HTTP error: %v", err) + } +} + +func TestDownloadHonorsCancellation(t *testing.T) { + started := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func( + writer http.ResponseWriter, request *http.Request, + ) { + writer.WriteHeader(http.StatusOK) + writer.(http.Flusher).Flush() + close(started) + <-request.Context().Done() + })) + defer server.Close() + client, err := NewClient( + httpapi.Config{Server: server.URL}, "/archiver", server.Client(), + ) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := client.Download(ctx, DownloadRequest{ + ResourceIDs: []string{"id"}, Format: "zip", + }, io.Discard, nil) + done <- err + }() + <-started + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("error: %v", err) + } +} + +func TestValidateFileAcceptsCompleteZIPAndTAR(t *testing.T) { + root := t.TempDir() + zipName := filepath.Join(root, "archive.zip") + if err := os.WriteFile( + zipName, testZIP(t, map[string]string{"file.txt": "hello"}), 0600, + ); err != nil { + t.Fatal(err) + } + if err := ValidateFile(zipName, "zip", ValidationLimits{}); err != nil { + t.Fatal(err) + } + tarName := filepath.Join(root, "archive.tar") + if err := os.WriteFile( + tarName, testTAR(t, map[string]string{"file.txt": "hello"}), 0600, + ); err != nil { + t.Fatal(err) + } + if err := ValidateFile(tarName, "tar", ValidationLimits{}); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + format string + }{ + {zipName, "tar"}, {tarName, "zip"}, {zipName, "rar"}, + } { + if err := ValidateFile(test.name, test.format, ValidationLimits{}); err == nil { + t.Fatalf("invalid %s as %s accepted", test.name, test.format) + } + } +} + +func TestValidateFileEnforcesDecodedLimits(t *testing.T) { + root := t.TempDir() + zipName := filepath.Join(root, "large.zip") + if err := os.WriteFile( + zipName, testZIP(t, map[string]string{ + "one.txt": "12345", "two.txt": "67890", + }), 0600, + ); err != nil { + t.Fatal(err) + } + if err := ValidateFile(zipName, "zip", ValidationLimits{ + MaxEntries: 1, MaxBytes: 100, + }); err == nil || !strings.Contains(err.Error(), "more than 1 entries") { + t.Fatalf("entry limit error: %v", err) + } + if err := ValidateFile(zipName, "zip", ValidationLimits{ + MaxEntries: 10, MaxBytes: 9, + }); err == nil || !strings.Contains(err.Error(), "exceeds 9 bytes") { + t.Fatalf("byte limit error: %v", err) + } + + tarName := filepath.Join(root, "large.tar") + if err := os.WriteFile( + tarName, testTAR(t, map[string]string{ + "one.txt": "12345", "two.txt": "67890", + }), 0600, + ); err != nil { + t.Fatal(err) + } + if err := ValidateFile(tarName, "tar", ValidationLimits{ + MaxEntries: 1, MaxBytes: 100, + }); err == nil || !strings.Contains(err.Error(), "more than 1 entries") { + t.Fatalf("entry limit error: %v", err) + } + if err := ValidateFile(tarName, "tar", ValidationLimits{ + MaxEntries: 10, MaxBytes: 9, + }); err == nil || !strings.Contains(err.Error(), "exceeds 9 bytes") { + t.Fatalf("byte limit error: %v", err) + } +} + +func testZIP(t *testing.T, files map[string]string) []byte { + t.Helper() + var output bytes.Buffer + archive := zip.NewWriter(&output) + for name, content := range files { + entry, err := archive.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(entry, content); err != nil { + t.Fatal(err) + } + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + return output.Bytes() +} + +func testTAR(t *testing.T, files map[string]string) []byte { + t.Helper() + var output bytes.Buffer + archive := tar.NewWriter(&output) + for name, content := range files { + if err := archive.WriteHeader(&tar.Header{ + Name: name, Mode: 0600, Size: int64(len(content)), + }); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(archive, content); err != nil { + t.Fatal(err) + } + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + return output.Bytes() +} diff --git a/internal/archiver/validate.go b/internal/archiver/validate.go new file mode 100644 index 0000000..6e7dcd2 --- /dev/null +++ b/internal/archiver/validate.go @@ -0,0 +1,135 @@ +package archiver + +import ( + "archive/tar" + "archive/zip" + "errors" + "fmt" + "io" + "math" + "os" + "strings" +) + +const ( + defaultMaxEntries = int64(10_000) + defaultMaxBytes = int64(1 << 30) +) + +// ValidationLimits bound archive decoding. Non-positive values use +// conservative oCIS-compatible defaults instead of permitting unbounded +// decompression when a server omitted capability limits. +type ValidationLimits struct { + MaxEntries int64 + MaxBytes int64 +} + +// ValidateFile reads the complete archive before it is committed to the user +// selected destination. This detects truncated streams and ZIP checksum +// failures, including server errors written after response streaming began. +func ValidateFile(name, format string, limits ValidationLimits) error { + limits = normalizedLimits(limits) + switch strings.ToLower(strings.TrimSpace(format)) { + case "zip": + return validateZIP(name, limits) + case "tar": + return validateTAR(name, limits) + default: + return fmt.Errorf("unsupported archive format %q", format) + } +} + +func validateZIP(name string, limits ValidationLimits) error { + archive, err := zip.OpenReader(name) + if err != nil { + return fmt.Errorf("validate ZIP archive: %w", err) + } + defer func() { _ = archive.Close() }() + if int64(len(archive.File)) > limits.MaxEntries { + return fmt.Errorf( + "validate ZIP archive: archive contains more than %d entries", + limits.MaxEntries, + ) + } + var total int64 + for _, entry := range archive.File { + if entry.UncompressedSize64 > math.MaxInt64 || + int64(entry.UncompressedSize64) > limits.MaxBytes-total { + return fmt.Errorf( + "validate ZIP archive: uncompressed content exceeds %d bytes", + limits.MaxBytes, + ) + } + total += int64(entry.UncompressedSize64) + reader, err := entry.Open() + if err != nil { + return fmt.Errorf("validate ZIP entry %q: %w", entry.Name, err) + } + _, copyErr := io.CopyN( + io.Discard, reader, int64(entry.UncompressedSize64), + ) + if copyErr == nil { + var extra [1]byte + _, copyErr = reader.Read(extra[:]) + if errors.Is(copyErr, io.EOF) { + copyErr = nil + } else if copyErr == nil { + copyErr = errors.New("entry exceeds its declared size") + } + } + closeErr := reader.Close() + if copyErr != nil { + return fmt.Errorf("validate ZIP entry %q: %w", entry.Name, copyErr) + } + if closeErr != nil { + return fmt.Errorf("close ZIP entry %q: %w", entry.Name, closeErr) + } + } + return nil +} + +func validateTAR(name string, limits ValidationLimits) error { + file, err := os.Open(name) //nolint:gosec // user-selected archive temporary file + if err != nil { + return fmt.Errorf("open TAR archive: %w", err) + } + defer func() { _ = file.Close() }() + reader := tar.NewReader(file) + var entries, total int64 + for { + entry, err := reader.Next() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("validate TAR archive: %w", err) + } + entries++ + if entries > limits.MaxEntries { + return fmt.Errorf( + "validate TAR archive: archive contains more than %d entries", + limits.MaxEntries, + ) + } + if entry.Size < 0 || entry.Size > limits.MaxBytes-total { + return fmt.Errorf( + "validate TAR archive: content exceeds %d bytes", + limits.MaxBytes, + ) + } + total += entry.Size + if _, err := io.CopyN(io.Discard, reader, entry.Size); err != nil { + return fmt.Errorf("validate TAR entry %q: %w", entry.Name, err) + } + } +} + +func normalizedLimits(limits ValidationLimits) ValidationLimits { + if limits.MaxEntries <= 0 { + limits.MaxEntries = defaultMaxEntries + } + if limits.MaxBytes <= 0 { + limits.MaxBytes = defaultMaxBytes + } + return limits +} diff --git a/internal/command/archive.go b/internal/command/archive.go new file mode 100644 index 0000000..278eda9 --- /dev/null +++ b/internal/command/archive.go @@ -0,0 +1,73 @@ +package command + +import ( + "strings" + + "github.com/mzner/ocis-cli/internal/app" + "github.com/spf13/cobra" +) + +func newArchiveCommand(options *globalOptions) *cobra.Command { + command := &cobra.Command{ + Use: "archive", Aliases: []string{"archives"}, + Short: "Download server-created ZIP or TAR archives", + } + command.AddCommand( + newArchiveDownloadCommand(options), + &cobra.Command{ + Use: "formats", Short: "List server-supported archive formats", Args: noArgs, + RunE: func(command *cobra.Command, _ []string) error { + return app.RunArchiveFormatsWithOptions( + command.Context(), options.profile, options.runOptions(command), + ) + }, + }, + ) + return command +} + +func newArchiveDownloadCommand(options *globalOptions) *cobra.Command { + var destination, format string + var overwrite, dryRun bool + command := &cobra.Command{ + Use: "download REMOTE_PATH...", + Short: "Download selected resources as one archive", + Args: minimumArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if strings.TrimSpace(destination) == "" { + return usageError("archive download", "--output is required") + } + return app.RunArchiveDownloadWithOptions( + command.Context(), app.ArchiveDownloadRequest{ + Paths: append([]string(nil), args...), Destination: destination, + Format: format, Overwrite: overwrite, DryRun: dryRun, + }, options.profile, options.runOptions(command), + ) + }, + } + command.Flags().StringVarP( + &destination, "output", "o", "", "local archive destination (required)", + ) + command.Flags().StringVar( + &format, "format", "", + "archive format: zip or tar (inferred from --output; default zip)", + ) + command.Flags().BoolVar( + &overwrite, "overwrite", false, + "explicitly allow replacing the local destination", + ) + command.Flags().BoolVar( + &dryRun, "dry-run", false, + "resolve and measure resources without downloading an archive", + ) + _ = command.RegisterFlagCompletionFunc( + "format", func( + _ *cobra.Command, _ []string, _ string, + ) ([]string, cobra.ShellCompDirective) { + return []string{ + "zip\tZIP archive", "tar\tTAR archive", + }, cobra.ShellCompDirectiveNoFileComp + }, + ) + return command +} diff --git a/internal/command/root.go b/internal/command/root.go index 541ae88..3eb6a31 100644 --- a/internal/command/root.go +++ b/internal/command/root.go @@ -141,6 +141,7 @@ func NewRootCommand() *cobra.Command { newTreeCommand(options), newDUCommand(options), newBatchCommand(options), + newArchiveCommand(options), newUploadCommand(options), newDownloadCommand(options), newMkdirCommand(options), diff --git a/internal/command/root_test.go b/internal/command/root_test.go index 5e81958..1e70c51 100644 --- a/internal/command/root_test.go +++ b/internal/command/root_test.go @@ -74,6 +74,7 @@ func TestGeneratedHelpIncludesGlobalFlags(t *testing.T) { "notification, notifications", "activity, activities", "event, events", + "archive, archives", } { if !strings.Contains(help, expected) { t.Fatalf("help does not contain %q:\n%s", expected, help) @@ -81,6 +82,54 @@ func TestGeneratedHelpIncludesGlobalFlags(t *testing.T) { } } +func TestArchiveCommandsAreDiscoverable(t *testing.T) { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&output) + root.SetArgs([]string{"archive", "--help"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + for _, expected := range []string{"download", "formats"} { + if !strings.Contains(output.String(), expected) { + t.Fatalf("archive help missing %q:\n%s", expected, output.String()) + } + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&output) + root.SetArgs([]string{"archive", "download", "--help"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + for _, expected := range []string{ + "--output", "--format", "--overwrite", "--dry-run", + } { + if !strings.Contains(output.String(), expected) { + t.Fatalf("archive download help missing %q:\n%s", expected, output.String()) + } + } + command, _, err := root.Find([]string{"archive", "download"}) + if err != nil { + t.Fatal(err) + } + complete, found := command.GetFlagCompletionFunc("format") + if !found { + t.Fatal("--format completion is not registered") + } + values, directive := complete(command, nil, "") + if directive != cobra.ShellCompDirectiveNoFileComp || + !strings.Contains(strings.Join(values, "\n"), "zip\tZIP archive") { + t.Fatalf("completion values=%v directive=%v", values, directive) + } + if _, _, err := root.Find([]string{"archives", "formats"}); err != nil { + t.Fatal(err) + } +} + func TestEventCommandsAndAliasesAreDiscoverable(t *testing.T) { root := NewRootCommand() var output bytes.Buffer diff --git a/internal/sharing/client.go b/internal/sharing/client.go index 602c1e1..8f1e878 100644 --- a/internal/sharing/client.go +++ b/internal/sharing/client.go @@ -63,7 +63,8 @@ type Capabilities struct { Reports []string `json:"reports,omitempty"` } `json:"dav"` Files struct { - TUS TUSCapabilities `json:"tus"` + TUS TUSCapabilities `json:"tus"` + Archivers []ArchiverCapabilities `json:"archivers,omitempty"` } `json:"files"` Sharing struct { APIEnabled bool `json:"apiEnabled"` @@ -97,6 +98,16 @@ type Capabilities struct { } `json:"graph"` } +// ArchiverCapabilities describes one server-advertised archive service. +type ArchiverCapabilities struct { + Enabled bool `json:"enabled"` + Version string `json:"version,omitempty"` + Formats []string `json:"formats,omitempty"` + URL string `json:"url,omitempty"` + MaxNumFiles int64 `json:"maxNumFiles,omitempty"` + MaxSize int64 `json:"maxSize,omitempty"` +} + // TUSCapabilities contains resumable-upload policy advertised by oCIS. type TUSCapabilities struct { Version string `json:"version,omitempty"` @@ -265,6 +276,14 @@ func (client *Client) Capabilities(ctx context.Context) (Capabilities, error) { Reports []string `json:"reports"` } `json:"dav"` Files struct { + Archivers []struct { + Enabled bool `json:"enabled"` + Version string `json:"version"` + Formats []string `json:"formats"` + URL string `json:"archiver_url"` + MaxNumFiles int64Value `json:"max_num_files"` + MaxSize int64Value `json:"max_size"` + } `json:"archivers"` TUSSupport struct { Version string `json:"version"` Resumable string `json:"resumable"` @@ -315,6 +334,18 @@ func (client *Client) Capabilities(ctx context.Context) (Capabilities, error) { result.Auth.MFA.SessionDuration = raw.Capabilities.Auth.MFA.SessionDuration result.DAV.Reports = raw.Capabilities.DAV.Reports + result.Files.Archivers = make( + []ArchiverCapabilities, 0, len(raw.Capabilities.Files.Archivers), + ) + for _, value := range raw.Capabilities.Files.Archivers { + result.Files.Archivers = append( + result.Files.Archivers, ArchiverCapabilities{ + Enabled: value.Enabled, Version: value.Version, + Formats: append([]string(nil), value.Formats...), URL: value.URL, + MaxNumFiles: int64(value.MaxNumFiles), MaxSize: int64(value.MaxSize), + }, + ) + } result.Files.TUS.Version = raw.Capabilities.Files.TUSSupport.Version result.Files.TUS.Resumable = raw.Capabilities.Files.TUSSupport.Resumable result.Files.TUS.Extensions = splitCapabilityList( @@ -499,6 +530,26 @@ func (value *intValue) UnmarshalJSON(data []byte) error { return nil } +type int64Value int64 + +func (value *int64Value) UnmarshalJSON(data []byte) error { + var number int64 + if err := json.Unmarshal(data, &number); err == nil { + *value = int64Value(number) + return nil + } + var text string + if err := json.Unmarshal(data, &text); err != nil { + return err + } + number, err := strconv.ParseInt(text, 10, 64) + if err != nil { + return err + } + *value = int64Value(number) + return nil +} + func cleanPath(value string) string { cleaned := "/" + strings.Trim(strings.TrimSpace(value), "/") if cleaned == "/" { diff --git a/internal/sharing/client_test.go b/internal/sharing/client_test.go index b371074..81f74f8 100644 --- a/internal/sharing/client_test.go +++ b/internal/sharing/client_test.go @@ -85,7 +85,11 @@ func TestCapabilities(t *testing.T) { writeOCS(writer, `{"capabilities":{ "core":{"support-sse":true}, "dav":{"reports":["search-files"]}, - "files":{"tus_support":{ + "files":{"archivers":[{ + "enabled":true,"version":"2.0.0","formats":["zip","tar"], + "archiver_url":"/archiver","max_num_files":"42", + "max_size":"1073741824" + }],"tus_support":{ "version":"1.0.0","resumable":"1.0.0", "extension":"creation,creation-with-upload", "max_chunk_size":10000000,"http_method_override":"true" @@ -121,6 +125,10 @@ func TestCapabilities(t *testing.T) { capabilities.Files.TUS.MaxChunkSize != 10000000 || len(capabilities.Files.TUS.Extensions) != 2 || !capabilities.Files.TUS.HTTPMethodOverride || + len(capabilities.Files.Archivers) != 1 || + capabilities.Files.Archivers[0].URL != "/archiver" || + capabilities.Files.Archivers[0].MaxNumFiles != 42 || + capabilities.Files.Archivers[0].MaxSize != 1073741824 || !capabilities.Spaces.Projects || !capabilities.Auth.MFA.Enabled || len(capabilities.Auth.MFA.LevelNames) != 1 || diff --git a/internal/transfer/local.go b/internal/transfer/local.go index b2bdfa8..f7b83e8 100644 --- a/internal/transfer/local.go +++ b/internal/transfer/local.go @@ -1,7 +1,39 @@ package transfer +import ( + "errors" + "fmt" + "io/fs" + "os" +) + +// ErrDestinationExists reports a no-clobber commit collision. +var ErrDestinationExists = errors.New("destination already exists") + // ReplaceFile atomically replaces destination with temporary where supported, // preserving the previous destination if the final rename fails. func ReplaceFile(temporary, destination string) error { return replaceFile(temporary, destination) } + +// CommitFile installs a completed temporary file. The no-overwrite path uses +// an atomic hard-link creation, so a destination created after preflight is not +// silently replaced. temporary and destination must be on the same filesystem. +func CommitFile(temporary, destination string, overwrite bool) error { + if overwrite { + return ReplaceFile(temporary, destination) + } + if err := os.Link(temporary, destination); err != nil { + if errors.Is(err, fs.ErrExist) { + return fmt.Errorf("%w: %s", ErrDestinationExists, destination) + } + return fmt.Errorf("commit download without overwrite: %w", err) + } + if err := os.Remove(temporary); err != nil { + return fmt.Errorf( + "archive was installed at %s but temporary-file cleanup failed: %w", + destination, err, + ) + } + return nil +} diff --git a/internal/transfer/local_test.go b/internal/transfer/local_test.go index 7749ed1..c63968d 100644 --- a/internal/transfer/local_test.go +++ b/internal/transfer/local_test.go @@ -1,6 +1,7 @@ package transfer import ( + "errors" "os" "path/filepath" "testing" @@ -24,3 +25,53 @@ func TestReplaceFilePreservesNewContent(t *testing.T) { t.Fatalf("destination: %q, %v", data, err) } } + +func TestCommitFileNoClobber(t *testing.T) { + root := t.TempDir() + temporary := filepath.Join(root, "archive.part") + destination := filepath.Join(root, "archive.zip") + if err := os.WriteFile(temporary, []byte("archive"), 0600); err != nil { + t.Fatal(err) + } + if err := CommitFile(temporary, destination, false); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(temporary); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary still exists: %v", err) + } + if data, err := os.ReadFile(destination); err != nil || string(data) != "archive" { + t.Fatalf("destination=%q error=%v", data, err) + } + + second := filepath.Join(root, "second.part") + if err := os.WriteFile(second, []byte("replacement"), 0600); err != nil { + t.Fatal(err) + } + if err := CommitFile(second, destination, false); !errors.Is(err, ErrDestinationExists) { + t.Fatalf("error: %v", err) + } + if data, err := os.ReadFile(destination); err != nil || string(data) != "archive" { + t.Fatalf("destination changed=%q error=%v", data, err) + } + if _, err := os.Stat(second); err != nil { + t.Fatalf("failed commit must retain temporary: %v", err) + } +} + +func TestCommitFileOverwrite(t *testing.T) { + root := t.TempDir() + temporary := filepath.Join(root, "archive.part") + destination := filepath.Join(root, "archive.zip") + if err := os.WriteFile(temporary, []byte("new"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destination, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + if err := CommitFile(temporary, destination, true); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(destination); err != nil || string(data) != "new" { + t.Fatalf("destination=%q error=%v", data, err) + } +}