diff --git a/.env.example b/.env.example index 9a2ea87..674369f 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,13 @@ REPO_ROOT=data/repos # where the host key file is kept (for ssh) SSH_HOST_KEY_PATH=data/ssh/host_ed25519 +# background maintenance intervals (go duration syntax, e.g. "30m", "12h") +# a zero duration ("0") disables the loop +TOKEN_GC_INTERVAL=1h +REPO_GC_INTERVAL=5h +# goroutines delivering webhook events +WEBHOOK_WORKERS=3 + # enable the LFS endpoints (batch + verify + object transfer) LFS_ENABLED=false diff --git a/AGENTS.md b/AGENTS.md index 14662fb..167ad87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,10 @@ It provides: - Git over SSH and HTTP (clone / fetch / push). - Git LFS (object transfer over HTTP; disk or S3 storage). - A control API to manage repositories, users, SSH keys, tokens, permissions, and webhooks. +- Repo APIs on the control plane: contents listing, blob read/upload, streamed + zip/tar.gz archives (optional LFS smudging), and commit creation on bare repos + (blobs + commits, CAS ref updates, `.gitattributes`-driven LFS cleaning) — so + products never need local clones. - A `read` / `write` / `admin` permission model enforced before every Git operation. - Push webhooks, signed and delivered off the push path. @@ -52,7 +56,8 @@ Package map: | `internal/domain` | Core types (repository, account, role, token, ssh key, lfs). | | `internal/services/*` | Business logic per area (repositories, users, auth, permissions, lfs); each has a service + a registry over `db`. | | `internal/storage` | LFS object storage behind an interface (`disk`, `s3`). | -| `internal/gitbackend` | Git pack protocol behind a small interface (`Local` execs system `git`; future `RPC` to storage nodes). | +| `internal/archive` | Pure mechanism: streaming tar re-encode to zip/tar.gz with an injected LFS smudge callback. | +| `internal/gitbackend` | Git subprocesses behind a small interface: pack protocol, read ops (ls-tree, blobs, archive), commit creation. | | `internal/server` | Composition root: wires control + git servers, runs and shuts down listeners. | | `internal/server/control` | Control API (REST); sub-handlers in `repositories/`, `users/`, `permissions/`. | | `internal/server/git/gitssh` | Git-over-SSH transport (custom in-process SSH server). | @@ -113,6 +118,12 @@ Prefer simple Go: small interfaces at module boundaries, context-aware I/O, expl errors with context, table-driven tests, standard library first. Avoid large global state, framework-heavy abstractions, and dependencies added for small tasks. +**File layout.** Do not multiply files or packages. A service is `service.go` + +`errors.go` + `registry.go` — new service methods go into `service.go`, not new +files. One test file per package where practical (`service_test.go`, +`handlers_test.go`). Small parsing/vocabulary helpers (formats, pointers, modes) +belong in `domain` next to the types they produce, not in new packages. + **Dependencies & interfaces.** The package that _invokes_ a dependency defines a minimal consumer interface for it (e.g. `gitssh`'s `Authenticator`/`TokenMinter`, the handler packages' own interfaces). Composition/router layers (`server`, `git`, @@ -125,7 +136,9 @@ concretes once. **HTTP.** One `net/http` + `chi` stack (no `fasthttp`/Fiber: the smart-HTTP path streams bodies to/from the `git-upload-pack`/`git-receive-pack` subprocesses). The Git/LFS routes must stay streaming-safe — no body-size limits or body-buffering -middleware. Health (`/healthz`) is unauthenticated and sits outside the audit chain. +middleware; the same applies to the control API's archive route, which streams +`git archive` output. Health (`/healthz`) is unauthenticated and sits outside the +audit chain. **Logging.** `zap` structured logging. Never log secrets (tokens, webhook secrets, SSH key material, full auth headers). Each transport request emits one audit line: @@ -143,10 +156,11 @@ storage clients. Do not roll your own SSH or Git protocol implementation. ## Webhooks -- Emitted only **after** a successful push, off the push path: receive-pack runs, - `gitbackend` diffs the repo's refs before/after, and the transport hands the - changes to the webhooks service. Delivery is an in-process bounded queue with - worker goroutines and retries — no external job system. +- Emitted only **after** a ref actually moved, off the push path: receive-pack + runs and `gitbackend` diffs the repo's refs before/after (transports dispatch), + or an api commit lands its CAS ref update (repositories service dispatches). + Both produce identical push events. Delivery is an in-process bounded queue + with worker goroutines and retries — no external job system. - One delivery per changed ref. Payload is self-describing: `event`, `ref`, `before`/`after`, `created`/`deleted`, a `repository` object (`id`, `name`, `full_name`), a `pusher` object (`id`, `username`), and `timestamp`. Creates/ diff --git a/README.md b/README.md index 523396e..333ac1a 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ Basically, this is a Git layer of infrastructure you'd put _underneath_ a projec - Basic Git over **SSH** and **HTTP** for clone / fetch / push. - **Git LFS** for large files, with object storage on local disk or any S3-compatible bucket (AWS S3, Cloudflare R2, MinIO). - A small **control API**, RESTful api to manage repositories, users, SSH keys, tokens, and permissions. +- A **repo content API** — list trees, read files, download zip/tar.gz archives, and create commits over REST, so your backend never needs a local clone. - Simple **permission model** (`read` / `write` / `admin`) enforced before every Git operation. -- **Push webhooks** — signed deliveries on every successful push. +- **Push webhooks** — signed deliveries on every ref change, pushed or committed via the API. - Bare-repository storage on a filesystem, with SQLite for metadata. ## Example @@ -78,6 +79,9 @@ All configuration is via environment variables. | `REPO_ROOT` | `data/repos` | Where bare repositories are stored. | | `SSH_HOST_KEY_PATH` | `data/ssh/host_ed25519` | SSH host key file (generated on first boot if absent). | | `ADMIN_TOKEN` | _(empty)_ | Raw token for the seeded admin account. Only its hash is stored. Empty = no admin seeded. | +| `TOKEN_GC_INTERVAL` | `1h` | How often expired tokens are deleted. `0` disables the loop. | +| `REPO_GC_INTERVAL` | `5h` | How often `git gc` sweeps the repositories (repack + prune). `0` disables the loop. | +| `WEBHOOK_WORKERS` | `3` | Goroutines delivering webhook events. | See [`.env.example`](.env.example). @@ -138,13 +142,108 @@ Every request requires `Authorization: Bearer `. Responses are enve | `POST` | `/repositories/{id}/webhooks` | `{url}` | Register a push webhook; the signing secret is returned **once**. | | `DELETE` | `/repositories/{id}/webhooks/{hookId}` | — | Delete a webhook. | +**Repository contents & commits** + +| Method | Path | Body | Description | +| ------ | ---------------------------------------------- | ----------- | ---------------------------------------------------------------------- | +| `GET` | `/repositories/{id}/contents?ref=&path=` | — | List one directory level of the tree at a ref. | +| `GET` | `/repositories/{id}/blob?ref=&path=&lfs=` | — | Stream one file's raw content. | +| `GET` | `/repositories/{id}/archive?ref=&format=&lfs=` | — | Stream a `zip` (default) or `tar.gz` archive of the tree. | +| `POST` | `/repositories/{id}/blobs` | _raw bytes_ | Upload content into the repo's object database; returns `{sha, size}`. | +| `POST` | `/repositories/{id}/commits` | JSON | Create a commit on a branch from uploaded blobs. | + +### Reading a repository + +`ref` accepts anything git can resolve to a commit — a branch, tag, sha, or expression like `main~2` — and defaults to `HEAD`. Every response is pinned to the exact commit it was answered from, so consumers can page through a repository without seeing a torn view mid-push. + +`GET /contents` returns the entries of one directory level: + +```json +{ + "data": { + "ref": "main", + "sha": "9fb037999f264ba9a7fc6274d15fa3ae2ab98312", + "path": "src", + "entries": [ + { + "name": "main.go", + "path": "src/main.go", + "type": "file", + "mode": "100644", + "size": 1234, + "sha": "..." + }, + { + "name": "vendor", + "path": "src/vendor", + "type": "dir", + "mode": "040000", + "sha": "..." + } + ] + } +} +``` + +`type` is `file` | `dir` | `symlink` | `submodule`. Listings over 10k entries set `"truncated": true`. + +`GET /blob` streams the file bytes with `Content-Length`, a strong `ETag` (the blob sha — content-addressed, so `If-None-Match` caching works perfectly), and `X-HeadlessGit-Commit` carrying the resolved commit. With `lfs=true`, an LFS pointer file is replaced by the real object; a missing object is a `404` rather than silently serving the pointer. + +`GET /archive` streams the whole tree as an artifact, named `-.zip` with a matching top-level folder. With `lfs=true`, pointer files are swapped for the real objects **in-flight** — the archive is re-encoded entry by entry, nothing is buffered or written to disk: + +![archive](images/archive.png) + +A pointer whose object is missing stays a pointer. + +### Writing without a clone + +Commits follow two-step model: upload content first, then commit metadata referencing it. + +![commit](images/commit.png) + +```sh +# 1. upload each new/changed file's bytes (raw body, streamed) +curl -H "Authorization: Bearer $TOKEN" \ + --data-binary @config.yaml \ + http://localhost:4001/repositories/7/blobs +# -> {"data": {"sha": "44b4fc6d...", "size": 812}} + +# 2. create the commit (atomic, any number of operations) +curl -H "Authorization: Bearer $TOKEN" -X POST \ + http://localhost:4001/repositories/7/commits -d '{ + "branch": "main", + "message": "update config", + "author": { "name": "deploy-bot", "email": "bot@example.com" }, + "expectedHeadSha": "9fb03799...", + "operations": [ + { "op": "put", "path": "config.yaml", "blobSha": "44b4fc6d..." }, + { "op": "delete", "path": "config.old.yaml" } + ] + }' +# -> 201 {"data": {"branch": "main", "commitSha": "...", "before": "9fb03799..."}} +``` + +`expectedHeadSha` controls concurrency: + +| Value | Meaning | +| ---------------- | ---------------------------------------------------------------------------------------- | +| _(omitted)_ | Last write wins. | +| a commit sha | Compare-and-swap: `409 head_mismatch` if the branch moved. | +| the all-zero sha | The branch must not exist yet — creates it (or the first commit on an empty repository). | + +Content is deduplicated by sha, so retrying an upload is free and a lost `409` race can be retried without re-uploading anything. Blobs that never get committed are garbage-collected after a grace period (see `REPO_GC_INTERVAL`). + +**LFS is automatic**, the same way it is for a git client: if the repo's `.gitattributes` marks a path as `filter=lfs` (including attributes added in the very same commit), the server stores the content as an LFS object and commits a pointer instead. Content that already _is_ a valid pointer passes through untouched, so pre-uploading big files via the LFS API (presigned, straight to the bucket) and committing the pointer yourself remains the efficient path for large objects. + +API commits dispatch the same signed [webhooks](#webhooks) as a `git push` — consumers can't tell them apart. + ### Health The control port also serves an unauthenticated `GET /healthz` readiness probe. It returns `200 {"status":"ok"}` when the database is reachable and `503 {"status":"unavailable"}` otherwise, and backs the container `HEALTHCHECK`. ## Webhooks -Register a webhook on a repository and `headlessgit` will `POST` to it after every successful push. +Register a webhook on a repository and `headlessgit` will `POST` to it after every ref change — a `git push` or a commit created through the [content API](#writing-without-a-clone) produce identical events. One delivery is sent **per changed ref** (a branch/tag create, update, or delete — not per file or commit). The JSON body: diff --git a/cmd/app/main.go b/cmd/app/main.go index 3ba8f65..752185f 100644 --- a/cmd/app/main.go +++ b/cmd/app/main.go @@ -55,26 +55,6 @@ func main() { log.Fatal(err) } - repoService := repositories.NewService( - root.With(zap.String("service", "repositories")), - repositories.NewRegistry(db), - gitBackend, - ) - - authService := auth.NewService( - root.With(zap.String("service", "auth")), - auth.NewRegistry(db), - ) - - if config.AdminToken != "" { - if err := authService.SeedAdmin(context.Background(), config.AdminToken); err != nil { - log.Fatal(err) - } - } - - permsService := permissions.NewService(permissions.NewRegistry(db)) - usersService := users.NewService(users.NewRegistry(db)) - // nil when LFS is disabled var lfsService *lfs.Service if config.LFS.Enabled { @@ -93,11 +73,39 @@ func main() { ) } + // assign through a typed variable so a disabled LFS stays a real nil interface + var repoLFS repositories.LFSObjects + if lfsService != nil { + repoLFS = lfsService + } + webhooksService := webhooks.NewService( root.With(zap.String("service", "webhooks")), webhooks.NewRegistry(db), ) + repoService := repositories.NewService( + root.With(zap.String("service", "repositories")), + repositories.NewRegistry(db), + gitBackend, + repoLFS, + webhooksService, + ) + + authService := auth.NewService( + root.With(zap.String("service", "auth")), + auth.NewRegistry(db), + ) + + if config.AdminToken != "" { + if err := authService.SeedAdmin(context.Background(), config.AdminToken); err != nil { + log.Fatal(err) + } + } + + permsService := permissions.NewService(permissions.NewRegistry(db)) + usersService := users.NewService(users.NewRegistry(db)) + ctx, stop := signal.NotifyContext( context.Background(), syscall.SIGINT, diff --git a/images/archive.png b/images/archive.png new file mode 100644 index 0000000..12cccf9 Binary files /dev/null and b/images/archive.png differ diff --git a/images/commit.png b/images/commit.png new file mode 100644 index 0000000..fed2533 Binary files /dev/null and b/images/commit.png differ diff --git a/internal/archive/encoder.go b/internal/archive/encoder.go new file mode 100644 index 0000000..816a851 --- /dev/null +++ b/internal/archive/encoder.go @@ -0,0 +1,83 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "io" + "strings" +) + +type Encoder interface { + Write(hdr *tar.Header, body io.Reader) error + Close() error +} + +type zipEncoder struct { + zw *zip.Writer +} + +func NewZipEncoder(out io.Writer) Encoder { + return &zipEncoder{zw: zip.NewWriter(out)} +} + +func (e *zipEncoder) Write(hdr *tar.Header, body io.Reader) error { + fi := hdr.FileInfo() + fh, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + + fh.Name = hdr.Name + if fi.IsDir() { + fh.Name = strings.TrimSuffix(fh.Name, "/") + "/" + fh.Method = zip.Store + } else { + fh.Method = zip.Deflate + } + + w, err := e.zw.CreateHeader(fh) + if err != nil { + return err + } + + switch hdr.Typeflag { + case tar.TypeReg: + _, err = io.Copy(w, body) + case tar.TypeSymlink: + // zip stores the link target as the entry body + _, err = io.Copy(w, strings.NewReader(hdr.Linkname)) + } + return err +} + +func (e *zipEncoder) Close() error { return e.zw.Close() } + +type tarGzEncoder struct { + gz *gzip.Writer + tw *tar.Writer +} + +func NewTarGzEncoder(out io.Writer) Encoder { + gz := gzip.NewWriter(out) + return &tarGzEncoder{gz: gz, tw: tar.NewWriter(gz)} +} + +func (e *tarGzEncoder) Write(hdr *tar.Header, body io.Reader) error { + if err := e.tw.WriteHeader(hdr); err != nil { + return err + } + if hdr.Typeflag == tar.TypeReg { + if _, err := io.Copy(e.tw, body); err != nil { + return err + } + } + return nil +} + +func (e *tarGzEncoder) Close() error { + if err := e.tw.Close(); err != nil { + return err + } + return e.gz.Close() +} diff --git a/internal/archive/transform.go b/internal/archive/transform.go new file mode 100644 index 0000000..0b3421e --- /dev/null +++ b/internal/archive/transform.go @@ -0,0 +1,60 @@ +package archive + +import ( + "archive/tar" + "bytes" + "io" + + "github.com/Axenos-dev/HeadlessGit/internal/domain" +) + +type SmudgeFunc func(oid string) (io.ReadCloser, int64, error) + +func Transform(src io.Reader, prefix string, smudge SmudgeFunc, enc Encoder) error { + tr := tar.NewReader(src) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + // drop the pax global header (the commit sha comment git emits) + if hdr.Typeflag == tar.TypeXGlobalHeader { + continue + } + hdr.Name = prefix + hdr.Name + + if err := writeEntry(enc, hdr, tr, smudge); err != nil { + return err + } + } + return enc.Close() +} + +func writeEntry(enc Encoder, hdr *tar.Header, body io.Reader, smudge SmudgeFunc) error { + // only small regular files can be LFS pointers, everything else passes through + if smudge == nil || hdr.Typeflag != tar.TypeReg || hdr.Size > domain.LFSPointerMaxSize { + return enc.Write(hdr, body) + } + + data, err := io.ReadAll(body) + if err != nil { + return err + } + ptr, ok := domain.ParseLFSPointer(data) + if !ok { + return enc.Write(hdr, bytes.NewReader(data)) + } + + rc, size, err := smudge(ptr.OID) + if err != nil { + // missing or foreign object: keep the pointer bytes, never fail the archive + return enc.Write(hdr, bytes.NewReader(data)) + } + defer rc.Close() + + hdr.Size = size + return enc.Write(hdr, rc) +} diff --git a/internal/archive/transform_test.go b/internal/archive/transform_test.go new file mode 100644 index 0000000..4d567f6 --- /dev/null +++ b/internal/archive/transform_test.go @@ -0,0 +1,180 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "strings" + "testing" +) + +// builds a tar the way git archive would emit it: pax global header first, +// then dirs, files (including an LFS pointer), and a symlink +func buildTestTar(t *testing.T, pointer string) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + write := func(hdr *tar.Header, body string) { + t.Helper() + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if body != "" { + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + } + + write(&tar.Header{ + Typeflag: tar.TypeXGlobalHeader, + Name: "pax_global_header", + PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}, + Format: tar.FormatPAX, + }, "") + write(&tar.Header{Typeflag: tar.TypeDir, Name: "src/", Mode: 0o755}, "") + write(&tar.Header{Typeflag: tar.TypeReg, Name: "README.md", Mode: 0o644, Size: 6}, "hello\n") + write(&tar.Header{Typeflag: tar.TypeReg, Name: "src/big.bin", Mode: 0o644, Size: int64(len(pointer))}, pointer) + write(&tar.Header{Typeflag: tar.TypeSymlink, Name: "link", Linkname: "README.md", Mode: 0o777}, "") + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func readZip(t *testing.T, data []byte) map[string]string { + t.Helper() + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatal(err) + } + got[f.Name] = string(body) + } + return got +} + +func smudgeFrom(objects map[string]string) SmudgeFunc { + return func(oid string) (io.ReadCloser, int64, error) { + content, ok := objects[oid] + if !ok { + return nil, 0, errors.New("object not found") + } + return io.NopCloser(strings.NewReader(content)), int64(len(content)), nil + } +} + +func TestTransformZipSmudgesLFS(t *testing.T) { + oid := strings.Repeat("ab", 32) + content := "REAL LFS CONTENT, MUCH BIGGER THAN A POINTER" + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(content)) + + var out bytes.Buffer + src := bytes.NewReader(buildTestTar(t, pointer)) + if err := Transform(src, "repo-abc/", smudgeFrom(map[string]string{oid: content}), NewZipEncoder(&out)); err != nil { + t.Fatal(err) + } + + got := readZip(t, out.Bytes()) + if _, ok := got["repo-abc/src/"]; !ok { + t.Errorf("missing dir entry, got %d entries", len(got)) + } + if got["repo-abc/README.md"] != "hello\n" { + t.Errorf("README.md = %q", got["repo-abc/README.md"]) + } + if got["repo-abc/src/big.bin"] != content { + t.Errorf("big.bin not smudged: %q", got["repo-abc/src/big.bin"]) + } + if got["repo-abc/link"] != "README.md" { + t.Errorf("symlink target = %q", got["repo-abc/link"]) + } + if _, ok := got["repo-abc/pax_global_header"]; ok { + t.Error("pax global header leaked into the archive") + } +} + +func TestTransformKeepsPointerWhenObjectMissing(t *testing.T) { + oid := strings.Repeat("ab", 32) + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize 44\n", oid) + + var out bytes.Buffer + src := bytes.NewReader(buildTestTar(t, pointer)) + if err := Transform(src, "repo-abc/", smudgeFrom(nil), NewZipEncoder(&out)); err != nil { + t.Fatal(err) + } + + if got := readZip(t, out.Bytes())["repo-abc/src/big.bin"]; got != pointer { + t.Errorf("missing object should keep pointer bytes, got %q", got) + } +} + +func TestTransformWithoutSmudge(t *testing.T) { + oid := strings.Repeat("ab", 32) + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize 44\n", oid) + + var out bytes.Buffer + src := bytes.NewReader(buildTestTar(t, pointer)) + if err := Transform(src, "repo-abc/", nil, NewZipEncoder(&out)); err != nil { + t.Fatal(err) + } + + if got := readZip(t, out.Bytes())["repo-abc/src/big.bin"]; got != pointer { + t.Errorf("nil smudge should keep pointer bytes, got %q", got) + } +} + +func TestTransformTarGz(t *testing.T) { + oid := strings.Repeat("ab", 32) + content := "REAL LFS CONTENT" + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(content)) + + var out bytes.Buffer + src := bytes.NewReader(buildTestTar(t, pointer)) + if err := Transform(src, "repo-abc/", smudgeFrom(map[string]string{oid: content}), NewTarGzEncoder(&out)); err != nil { + t.Fatal(err) + } + + gz, err := gzip.NewReader(bytes.NewReader(out.Bytes())) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + got[hdr.Name] = string(body) + } + + if got["repo-abc/src/big.bin"] != content { + t.Errorf("big.bin not smudged: %q", got["repo-abc/src/big.bin"]) + } + if got["repo-abc/README.md"] != "hello\n" { + t.Errorf("README.md = %q", got["repo-abc/README.md"]) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 202bca7..6333ab8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "strings" + "time" "github.com/caarlos0/env/v11" ) @@ -18,6 +19,12 @@ type ServerConfig struct { GitSSHPort int `env:"GIT_SSH_PORT" envDefault:"2222"` RepoRoot string `env:"REPO_ROOT" envDefault:"data/repos"` HostKeyPath string `env:"SSH_HOST_KEY_PATH" envDefault:"data/ssh/host_ed25519"` + + // background maintenance, + // a zero duration disables the loop + TokenGCInterval time.Duration `env:"TOKEN_GC_INTERVAL" envDefault:"1h"` + RepoGCInterval time.Duration `env:"REPO_GC_INTERVAL" envDefault:"5h"` + WebhookWorkers int `env:"WEBHOOK_WORKERS" envDefault:"3"` } type LFSConfig struct { diff --git a/internal/db/gen/repositories.sql.go b/internal/db/gen/repositories.sql.go index 5915b50..5fe79d7 100644 --- a/internal/db/gen/repositories.sql.go +++ b/internal/db/gen/repositories.sql.go @@ -101,6 +101,41 @@ func (q *Queries) GetRepositoryByPath(ctx context.Context, arg GetRepositoryByPa return i, err } +const listRepositories = `-- name: ListRepositories :many +select id, owner_id, repository_name, storage_path, visibility, created_at_unix_ms, updated_at_unix_ms from repositories +` + +func (q *Queries) ListRepositories(ctx context.Context) ([]Repository, error) { + rows, err := q.db.QueryContext(ctx, listRepositories) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Repository + for rows.Next() { + var i Repository + if err := rows.Scan( + &i.ID, + &i.OwnerID, + &i.RepositoryName, + &i.StoragePath, + &i.Visibility, + &i.CreatedAtUnixMs, + &i.UpdatedAtUnixMs, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listUserRepositories = `-- name: ListUserRepositories :many select id, owner_id, repository_name, storage_path, visibility, created_at_unix_ms, updated_at_unix_ms from repositories where owner_id=? diff --git a/internal/db/queries/repositories.sql b/internal/db/queries/repositories.sql index c617127..e53e3b0 100644 --- a/internal/db/queries/repositories.sql +++ b/internal/db/queries/repositories.sql @@ -6,6 +6,9 @@ where id=? limit 1; select * from repositories where owner_id=?; +-- name: ListRepositories :many +select * from repositories; + -- name: CreateRepository :one insert into repositories ( owner_id, repository_name, storage_path, visibility diff --git a/internal/domain/archive.go b/internal/domain/archive.go new file mode 100644 index 0000000..3fbbd94 --- /dev/null +++ b/internal/domain/archive.go @@ -0,0 +1,46 @@ +package domain + +import "fmt" + +type ArchiveFormat string + +const ( + ArchiveFormatZip ArchiveFormat = "zip" + ArchiveFormatTarGz ArchiveFormat = "tar.gz" +) + +func ParseArchiveFormat(s string) (ArchiveFormat, bool) { + switch s { + case "", "zip": + return ArchiveFormatZip, true + case "tar.gz", "tgz": + return ArchiveFormatTarGz, true + } + return "", false +} + +func (f ArchiveFormat) Extension() string { + if f == ArchiveFormatTarGz { + return "tar.gz" + } + return "zip" +} + +type ArchiveRequest struct { + Repository Repository + CommitSHA string + Format ArchiveFormat + IncludeLFS bool +} + +// Filename is the suggested artifact name: -. +func (r ArchiveRequest) Filename() string { + return fmt.Sprintf("%s-%s.%s", r.Repository.RepositoryName, ShortSHA(r.CommitSHA), r.Format.Extension()) +} + +func ShortSHA(sha string) string { + if len(sha) > 12 { + return sha[:12] + } + return sha +} diff --git a/internal/domain/blob.go b/internal/domain/blob.go new file mode 100644 index 0000000..d5e4542 --- /dev/null +++ b/internal/domain/blob.go @@ -0,0 +1,10 @@ +package domain + +type BlobRequest struct { + Repository Repository + CommitSHA string + BlobSHA string + Path string + Size int64 // exact byte count of what will be streamed + LFSOID string // non-empty when the blob is an LFS pointer being smudged +} diff --git a/internal/domain/commit.go b/internal/domain/commit.go new file mode 100644 index 0000000..868358e --- /dev/null +++ b/internal/domain/commit.go @@ -0,0 +1,28 @@ +package domain + +type CommitIdentity struct { + Name string + Email string +} + +type CommitFileOp struct { + Delete bool + Path string + BlobSHA string + Executable bool +} + +type CommitRequest struct { + Branch string + Message string + Author CommitIdentity + ExpectedHeadSHA string // pins the commit to an exact branch state + PusherID int64 // optionally attributes the push event to an account + Operations []CommitFileOp +} + +type CommitResult struct { + Branch string + CommitSHA string + Before string // the branch head the commit was built on; all-zero for a new branch +} diff --git a/internal/domain/contents.go b/internal/domain/contents.go new file mode 100644 index 0000000..cf37056 --- /dev/null +++ b/internal/domain/contents.go @@ -0,0 +1,40 @@ +package domain + +type TreeEntryType string + +const ( + TreeEntryFile TreeEntryType = "file" + TreeEntryDir TreeEntryType = "dir" + TreeEntrySymlink TreeEntryType = "symlink" + TreeEntrySubmodule TreeEntryType = "submodule" +) + +func TreeEntryTypeFromMode(mode string) TreeEntryType { + switch mode { + case "040000": + return TreeEntryDir + case "120000": + return TreeEntrySymlink + case "160000": + return TreeEntrySubmodule + default: // 100644, 100755 + return TreeEntryFile + } +} + +type TreeEntry struct { + Name string + Path string // full path from the repo root + Type TreeEntryType + Mode string + SHA string + Size int64 // object size in bytes, -1 for non-blobs (dirs, submodules) +} + +type RepositoryContents struct { + Ref string + CommitSHA string + Path string + Entries []TreeEntry + Truncated bool +} diff --git a/internal/domain/lfs.go b/internal/domain/lfs.go index fed516b..ccf8f49 100644 --- a/internal/domain/lfs.go +++ b/internal/domain/lfs.go @@ -1,6 +1,10 @@ package domain -import "time" +import ( + "strconv" + "strings" + "time" +) type LFSOperation string @@ -9,6 +13,9 @@ const ( LFSOperationDownload LFSOperation = "download" ) +const LFSPointerMaxSize = 1024 +const lfsVersionLine = "version https://git-lfs.github.com/spec/v1" + type LFSPointer struct { OID string Size int64 @@ -31,3 +38,45 @@ type LFSObjectResponse struct { Actions map[string]LFSAction Error *LFSObjectError } + +func ParseLFSPointer(data []byte) (LFSPointer, bool) { + if len(data) == 0 || len(data) > LFSPointerMaxSize { + return LFSPointer{}, false + } + + lines := strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") + if len(lines) < 3 || lines[0] != lfsVersionLine { + return LFSPointer{}, false + } + + ptr := LFSPointer{Size: -1} + for _, line := range lines[1:] { + if oid, ok := strings.CutPrefix(line, "oid sha256:"); ok { + ptr.OID = oid + } else if raw, ok := strings.CutPrefix(line, "size "); ok { + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return LFSPointer{}, false + } + ptr.Size = n + } + } + + if !validLFSOID(ptr.OID) || ptr.Size < 0 { + return LFSPointer{}, false + } + return ptr, true +} + +// enforces a 64 char lowercase hex sha256 +func validLFSOID(oid string) bool { + if len(oid) != 64 { + return false + } + for _, c := range oid { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/domain/lfs_test.go b/internal/domain/lfs_test.go new file mode 100644 index 0000000..ae0d8b6 --- /dev/null +++ b/internal/domain/lfs_test.go @@ -0,0 +1,39 @@ +package domain + +import ( + "strings" + "testing" +) + +func TestParseLFSPointer(t *testing.T) { + oid := strings.Repeat("ab", 32) + valid := "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\nsize 12345\n" + + cases := []struct { + name string + data string + ok bool + }{ + {"valid", valid, true}, + {"empty", "", false}, + {"plain text", "hello world\nthis mentions oid sha256: things\nsize 5\n", false}, + {"wrong version", "version https://example.com/spec/v1\noid sha256:" + oid + "\nsize 5\n", false}, + {"short oid", "version https://git-lfs.github.com/spec/v1\noid sha256:abcd\nsize 5\n", false}, + {"uppercase oid", "version https://git-lfs.github.com/spec/v1\noid sha256:" + strings.ToUpper(oid) + "\nsize 5\n", false}, + {"missing size", "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\n\n", false}, + {"bad size", "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + "\nsize lots\n", false}, + {"oversized", valid + strings.Repeat("x", LFSPointerMaxSize), false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ptr, ok := ParseLFSPointer([]byte(tc.data)) + if ok != tc.ok { + t.Fatalf("ParseLFSPointer ok = %v, want %v", ok, tc.ok) + } + if ok && (ptr.OID != oid || ptr.Size != 12345) { + t.Errorf("ParseLFSPointer = %+v", ptr) + } + }) + } +} diff --git a/internal/gitbackend/backend.go b/internal/gitbackend/backend.go index e7b6e7e..56e49cf 100644 --- a/internal/gitbackend/backend.go +++ b/internal/gitbackend/backend.go @@ -12,4 +12,12 @@ type Backend interface { AdvertiseRefs(ctx context.Context, storagePath string, svc Service, stdout io.Writer) error UploadPack(ctx context.Context, storagePath string, stateless bool, stdin io.Reader, stdout, stderr io.Writer) error ReceivePack(ctx context.Context, storagePath string, stateless bool, stdin io.Reader, stdout, stderr io.Writer) ([]RefChange, error) + ListTree(ctx context.Context, storagePath, rev, treePath string) (TreeListing, error) + ResolveCommit(ctx context.Context, storagePath, rev string) (string, error) + ArchiveTar(ctx context.Context, storagePath, rev string, out io.Writer) (string, error) + StatBlob(ctx context.Context, storagePath, rev, treePath string) (BlobInfo, error) + ReadBlob(ctx context.Context, storagePath, blobSHA string, out io.Writer) error + WriteBlob(ctx context.Context, storagePath string, r io.Reader) (string, int64, error) + ApplyCommit(ctx context.Context, storagePath string, spec CommitSpec, ops []CommitOp, clean CleanFunc) (RefChange, error) + GC(ctx context.Context, storagePath string) error } diff --git a/internal/gitbackend/errors.go b/internal/gitbackend/errors.go new file mode 100644 index 0000000..43351e8 --- /dev/null +++ b/internal/gitbackend/errors.go @@ -0,0 +1,19 @@ +package gitbackend + +import "errors" + +var ( + ErrInvalidRev = errors.New("invalid revision") + ErrInvalidPath = errors.New("invalid tree path") + ErrRevNotFound = errors.New("revision not found") + ErrPathNotFound = errors.New("path not found in tree") + ErrNotABlob = errors.New("path is not a blob") + + // commit creation + ErrInvalidBranch = errors.New("invalid branch name") + ErrInvalidOps = errors.New("invalid commit operations") + ErrUnknownBlob = errors.New("blob not found in repository") + ErrHeadMismatch = errors.New("branch head mismatch") + ErrNothingToCommit = errors.New("nothing to commit") + ErrLFSRequired = errors.New("path is lfs-tracked but no clean filter is available") +) diff --git a/internal/gitbackend/local.go b/internal/gitbackend/local.go index d46dbb8..e1cf9fd 100644 --- a/internal/gitbackend/local.go +++ b/internal/gitbackend/local.go @@ -1,14 +1,15 @@ package gitbackend import ( - "bufio" "bytes" "context" "fmt" "io" "os" "os/exec" + "path" "path/filepath" + "strconv" "strings" "sync" "time" @@ -24,6 +25,9 @@ const ( ReceivePack // push ) +// repacking a large repo can take a while +const gcTimeout = 30 * time.Minute + // returns the command name of the service func (s Service) Name() string { if s == ReceivePack { @@ -174,28 +178,622 @@ func (l *Local) listRefs(ctx context.Context, storagePath string) (map[string]st return nil, err } + out, err := l.runGit(ctx, dir, nil, nil, "for-each-ref", "--format=%(objectname) %(refname)") + if err != nil { + return nil, err + } + + refs := make(map[string]string) + for line := range strings.SplitSeq(out, "\n") { + // cut the string by " ", to separate sha from ref + sha, ref, ok := strings.Cut(line, " ") + if !ok { + continue + } + refs[ref] = sha + } + return refs, nil +} + +func (l *Local) ListTree(ctx context.Context, storagePath, rev, treePath string) (TreeListing, error) { + dir, err := l.resolve(storagePath) + if err != nil { + return TreeListing{}, err + } + + treePath, err = normalizeTreePath(treePath) + if err != nil { + return TreeListing{}, err + } + + commitSHA, err := l.ResolveCommit(ctx, storagePath, rev) + if err != nil { + return TreeListing{}, err + } + + treeish := commitSHA + if treePath != "" { + treeish += ":" + treePath + } + + out, err := l.runGit(ctx, dir, nil, nil, "ls-tree", "--long", "-z", "--end-of-options", treeish) + if err != nil { + // the rev already resolved, so this is a missing path or a non-directory + return TreeListing{}, fmt.Errorf("%w: %q", ErrPathNotFound, treePath) + } + + entries, truncated, err := parseLsTree([]byte(out), treePath) + if err != nil { + return TreeListing{}, err + } + return TreeListing{CommitSHA: commitSHA, Entries: entries, Truncated: truncated}, nil +} + +// streams an uncompressed tar archive of the repo tree, +// the tar entries carries LFS pointers files as-is, smudging is a service concern! +func (l *Local) ArchiveTar(ctx context.Context, storagePath, rev string, out io.Writer) (string, error) { + dir, err := l.resolve(storagePath) + if err != nil { + return "", err + } + + // only the resolution step gets the short timeout + commitSHA, err := l.ResolveCommit(ctx, storagePath, rev) + if err != nil { + return "", err + } + + cmd := exec.CommandContext(ctx, l.gitPath, "-C", dir, "archive", "--format=tar", "--end-of-options", commitSHA) + cmd.Stdout = out + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git archive: %w: %s", err, strings.TrimSpace(stderr.String())) + } + return commitSHA, nil +} + +func (l *Local) StatBlob(ctx context.Context, storagePath, rev, treePath string) (BlobInfo, error) { + dir, err := l.resolve(storagePath) + if err != nil { + return BlobInfo{}, err + } + + treePath, err = normalizeTreePath(treePath) + if err != nil { + return BlobInfo{}, err + } + if treePath == "" { + // the root is a tree by definition (and its not a blob) + return BlobInfo{}, fmt.Errorf("%w: %q", ErrNotABlob, treePath) + } + + commitSHA, err := l.ResolveCommit(ctx, storagePath, rev) + if err != nil { + return BlobInfo{}, err + } + + blobSHA, err := l.revParse(ctx, dir, commitSHA+":"+treePath) + if err != nil { + return BlobInfo{}, fmt.Errorf("%w: %q", ErrPathNotFound, treePath) + } + + out, err := l.runGit(ctx, dir, nil, strings.NewReader(blobSHA+"\n"), "cat-file", "--batch-check") + if err != nil { + return BlobInfo{}, err + } + + // output shape: " " + fields := strings.Fields(out) + if len(fields) != 3 { + return BlobInfo{}, fmt.Errorf("malformed batch-check output: %q", out) + } + if fields[1] != "blob" { + return BlobInfo{}, fmt.Errorf("%w: %q is a %s", ErrNotABlob, treePath, fields[1]) + } + size, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return BlobInfo{}, fmt.Errorf("malformed blob size %q: %w", fields[2], err) + } + + return BlobInfo{CommitSHA: commitSHA, BlobSHA: blobSHA, Size: size}, nil +} + +func (l *Local) ReadBlob(ctx context.Context, storagePath, blobSHA string, out io.Writer) error { + dir, err := l.resolve(storagePath) + if err != nil { + return err + } + + if !isHexSHA(blobSHA) { + return fmt.Errorf("%w: %q", ErrInvalidRev, blobSHA) + } + + cmd := exec.CommandContext(ctx, l.gitPath, "-C", dir, "cat-file", "blob", blobSHA) + cmd.Stdout = out + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("cat-file blob: %w: %s", err, strings.TrimSpace(stderr.String())) + } + return nil +} + +func (l *Local) ResolveCommit(ctx context.Context, storagePath, rev string) (string, error) { + dir, err := l.resolve(storagePath) + if err != nil { + return "", err + } + + rev, err = normalizeRev(rev) + if err != nil { + return "", err + } + + // ^{commit} forces the object to exist and peel to a commit + commitSHA, err := l.revParse(ctx, dir, rev+"^{commit}") + if err != nil { + return "", fmt.Errorf("%w: %s", ErrRevNotFound, rev) + } + return commitSHA, nil +} + +func (l *Local) WriteBlob(ctx context.Context, storagePath string, r io.Reader) (string, int64, error) { + dir, err := l.resolve(storagePath) + if err != nil { + return "", 0, err + } + + counter := &countingReader{r: r} + cmd := exec.CommandContext(ctx, l.gitPath, "-C", dir, "hash-object", "-w", "--stdin") + cmd.Stdin = counter + + var out, stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", 0, fmt.Errorf("hash-object: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + return strings.TrimSpace(out.String()), counter.n, nil +} + +func (l *Local) GC(ctx context.Context, storagePath string) error { + dir, err := l.resolve(storagePath) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(ctx, gcTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, l.gitPath, "-C", dir, "gc", "--quiet") + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git gc: %w: %s", err, strings.TrimSpace(stderr.String())) + } + return nil +} + +// creates a commit on a branch from already-uploaded blobs +func (l *Local) ApplyCommit(ctx context.Context, storagePath string, spec CommitSpec, ops []CommitOp, clean CleanFunc) (RefChange, error) { + dir, err := l.resolve(storagePath) + if err != nil { + return RefChange{}, err + } + + spec, err = l.validateCommitSpec(ctx, dir, spec) + if err != nil { + return RefChange{}, err + } + ops, err = validateCommitOps(ops) + if err != nil { + return RefChange{}, err + } + + // resolve the current branch head + ref := "refs/heads/" + spec.Branch + oldSHA, err := l.revParse(ctx, dir, ref) + unborn := err != nil + switch { + case unborn && spec.ExpectedOld != zeroSHA: + return RefChange{}, fmt.Errorf("%w: branch %s", ErrRevNotFound, spec.Branch) + case !unborn && spec.ExpectedOld == zeroSHA: + return RefChange{}, fmt.Errorf("%w: branch %s already exists", ErrHeadMismatch, spec.Branch) + case !unborn && spec.ExpectedOld != "" && spec.ExpectedOld != oldSHA: + return RefChange{}, fmt.Errorf("%w: expected %s, head is %s", ErrHeadMismatch, spec.ExpectedOld, oldSHA) + } + if unborn { + oldSHA = zeroSHA + } + + // one batch-check verifies every referenced blob (and captures sizes for + // the clean filter) plus the existence of every delete target + sizes, err := l.verifyCommitInputs(ctx, dir, oldSHA, unborn, ops) + if err != nil { + return RefChange{}, err + } + + // private index file: commits never touch the repo's real index (bare + // repos have none) and concurrent commits cannot see each other + idx, err := os.CreateTemp(dir, "headlessgit-index-*") + if err != nil { + return RefChange{}, fmt.Errorf("create temp index: %w", err) + } + idx.Close() + defer os.Remove(idx.Name()) + env := []string{"GIT_INDEX_FILE=" + idx.Name()} + + if unborn { + if _, err := l.runGit(ctx, dir, env, nil, "read-tree", "--empty"); err != nil { + return RefChange{}, err + } + } else { + if _, err := l.runGit(ctx, dir, env, nil, "read-tree", oldSHA); err != nil { + return RefChange{}, err + } + } + + // .gitattributes changes land first, so lfs tracking added in this very + // commit already applies to the files committed alongside it + attrOps, fileOps := splitAttrOps(ops) + if err := l.updateIndex(ctx, dir, env, attrOps); err != nil { + return RefChange{}, err + } + + fileOps, err = l.cleanLFSTracked(ctx, dir, env, fileOps, sizes, clean) + if err != nil { + return RefChange{}, err + } + if err := l.updateIndex(ctx, dir, env, fileOps); err != nil { + return RefChange{}, err + } + + treeSHA, err := l.runGit(ctx, dir, env, nil, "write-tree") + if err != nil { + return RefChange{}, fmt.Errorf("%w: %s", ErrInvalidOps, err) + } + if !unborn { + oldTree, err := l.revParse(ctx, dir, oldSHA+"^{tree}") + if err != nil { + return RefChange{}, err + } + if treeSHA == oldTree { + return RefChange{}, ErrNothingToCommit + } + } + + commitEnv := []string{ + "GIT_AUTHOR_NAME=" + spec.Author.Name, + "GIT_AUTHOR_EMAIL=" + spec.Author.Email, + "GIT_COMMITTER_NAME=" + spec.Committer.Name, + "GIT_COMMITTER_EMAIL=" + spec.Committer.Email, + } + args := []string{"commit-tree", treeSHA} + if !unborn { + args = append(args, "-p", oldSHA) + } + args = append(args, "-m", spec.Message) + newSHA, err := l.runGit(ctx, dir, commitEnv, nil, args...) + if err != nil { + return RefChange{}, err + } + + // update-ref only moves the branch if it still points at oldSHA + if _, err := l.runGit(ctx, dir, nil, nil, "update-ref", ref, newSHA, oldSHA); err != nil { + return RefChange{}, fmt.Errorf("%w: %s", ErrHeadMismatch, err) + } + + return RefChange{Ref: ref, OldSHA: oldSHA, NewSHA: newSHA}, nil +} + +// validateCommitSpec checks the branch name and identities +func (l *Local) validateCommitSpec(ctx context.Context, dir string, spec CommitSpec) (CommitSpec, error) { + if spec.Branch == "" || strings.HasPrefix(spec.Branch, "-") { + return spec, fmt.Errorf("%w: %q", ErrInvalidBranch, spec.Branch) + } + for _, r := range spec.Branch { + if r < 0x20 || r == 0x7f { + return spec, fmt.Errorf("%w: control character", ErrInvalidBranch) + } + } + // git itself is the authority on ref name rules + if _, err := l.runGit(ctx, dir, nil, nil, "check-ref-format", "--branch", spec.Branch); err != nil { + return spec, fmt.Errorf("%w: %q", ErrInvalidBranch, spec.Branch) + } + + if spec.ExpectedOld != "" && !isHexSHA(spec.ExpectedOld) { + return spec, fmt.Errorf("%w: expected old %q", ErrInvalidRev, spec.ExpectedOld) + } + if spec.Author.Name == "" || spec.Author.Email == "" { + return spec, fmt.Errorf("%w: author name and email are required", ErrInvalidOps) + } + if spec.Committer.Name == "" { + spec.Committer = spec.Author + } + if spec.Message == "" { + return spec, fmt.Errorf("%w: message is required", ErrInvalidOps) + } + return spec, nil +} + +// normalizes paths and enforces the op rules +func validateCommitOps(ops []CommitOp) ([]CommitOp, error) { + if len(ops) == 0 { + return nil, fmt.Errorf("%w: no operations", ErrInvalidOps) + } + if len(ops) > maxCommitOps { + return nil, fmt.Errorf("%w: more than %d operations", ErrInvalidOps, maxCommitOps) + } + + out := make([]CommitOp, len(ops)) + seen := make(map[string]bool, len(ops)) + for i, op := range ops { + p, err := normalizeTreePath(op.Path) + if err != nil || p == "" { + return nil, fmt.Errorf("%w: path %q", ErrInvalidOps, op.Path) + } + // the batch-check and check-attr line protocols cannot carry these + for _, r := range p { + if r < 0x20 || r == 0x7f { + return nil, fmt.Errorf("%w: control character in path", ErrInvalidOps) + } + } + if seen[p] { + return nil, fmt.Errorf("%w: duplicate path %q", ErrInvalidOps, p) + } + seen[p] = true + + op.Path = p + if !op.Delete { + if !isHexSHA(op.BlobSHA) { + return nil, fmt.Errorf("%w: blob sha %q", ErrInvalidOps, op.BlobSHA) + } + switch op.Mode { + case "": + op.Mode = "100644" + case "100644", "100755": + default: + return nil, fmt.Errorf("%w: mode %q", ErrInvalidOps, op.Mode) + } + } + out[i] = op + } + return out, nil +} + +// runs one cat-file --batch-check over every put blob sha (returning their sizes) +// and every delete target +// so unknown blobs and missing delete paths fail +func (l *Local) verifyCommitInputs(ctx context.Context, dir, oldSHA string, unborn bool, ops []CommitOp) (map[string]int64, error) { + var in strings.Builder + type query struct { + op CommitOp + isPut bool + } + var queries []query + for _, op := range ops { + if op.Delete { + if unborn { + return nil, fmt.Errorf("%w: %q", ErrPathNotFound, op.Path) + } + in.WriteString(oldSHA + ":" + op.Path + "\n") + } else { + in.WriteString(op.BlobSHA + "\n") + } + queries = append(queries, query{op: op, isPut: !op.Delete}) + } + + out, err := l.runGit(ctx, dir, nil, strings.NewReader(in.String()), "cat-file", "--batch-check") + if err != nil { + return nil, err + } + + lines := strings.Split(out, "\n") + if len(lines) != len(queries) { + return nil, fmt.Errorf("unexpected batch-check output: %d lines for %d queries", len(lines), len(queries)) + } + + sizes := make(map[string]int64) + for i, line := range lines { + q := queries[i] + fields := strings.Fields(line) + switch { + case len(fields) >= 2 && fields[len(fields)-1] == "missing": + if q.isPut { + return nil, fmt.Errorf("%w: %s", ErrUnknownBlob, q.op.BlobSHA) + } + return nil, fmt.Errorf("%w: %q", ErrPathNotFound, q.op.Path) + case len(fields) == 3 && fields[1] == "blob": + size, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("malformed batch-check size %q: %w", fields[2], err) + } + if q.isPut { + sizes[q.op.BlobSHA] = size + } + case len(fields) == 3: + // a delete target resolving to a tree or a put sha naming a non-blob + if q.isPut { + return nil, fmt.Errorf("%w: %s is a %s", ErrUnknownBlob, q.op.BlobSHA, fields[1]) + } + return nil, fmt.Errorf("%w: %q is a %s", ErrNotABlob, q.op.Path, fields[1]) + default: + return nil, fmt.Errorf("malformed batch-check line: %q", line) + } + } + return sizes, nil +} + +// separates .gitattributes changes from regular file changes +func splitAttrOps(ops []CommitOp) (attr, files []CommitOp) { + for _, op := range ops { + if op.Path == ".gitattributes" || strings.HasSuffix(op.Path, "/.gitattributes") { + attr = append(attr, op) + } else { + files = append(files, op) + } + } + return attr, files +} + +// asks git which put paths are lfs-tracked, +// and swaps their blobs for the pointer blobs produced by the clean filter +func (l *Local) cleanLFSTracked(ctx context.Context, dir string, env []string, ops []CommitOp, sizes map[string]int64, clean CleanFunc) ([]CommitOp, error) { + var paths []string + byPath := make(map[string]int) + for i, op := range ops { + if !op.Delete { + paths = append(paths, op.Path) + byPath[op.Path] = i + } + } + if len(paths) == 0 { + return ops, nil + } + + args := append([]string{"check-attr", "-z", "--cached", "filter", "--"}, paths...) + out, err := l.runGit(ctx, dir, env, nil, args...) + if err != nil { + return nil, err + } + + // -z output is NUL-separated (path, attr, value) triples + fields := strings.Split(out, "\x00") + for i := 0; i+2 < len(fields); i += 3 { + path, value := fields[i], fields[i+2] + if value != "lfs" { + continue + } + idx, ok := byPath[path] + if !ok { + continue + } + if clean == nil { + return nil, fmt.Errorf("%w: %q", ErrLFSRequired, path) + } + + pointerSHA, err := clean(path, ops[idx].BlobSHA, sizes[ops[idx].BlobSHA]) + if err != nil { + return nil, fmt.Errorf("lfs clean %q: %w", path, err) + } + if !isHexSHA(pointerSHA) { + return nil, fmt.Errorf("lfs clean %q returned invalid sha %q", path, pointerSHA) + } + ops[idx].BlobSHA = pointerSHA + } + return ops, nil +} + +// applies puts and deletes in one subprocess +func (l *Local) updateIndex(ctx context.Context, dir string, env []string, ops []CommitOp) error { + if len(ops) == 0 { + return nil + } + var in strings.Builder + for _, op := range ops { + if op.Delete { + in.WriteString("0 " + zeroSHA + "\t" + op.Path + "\x00") + } else { + in.WriteString(op.Mode + " " + op.BlobSHA + "\t" + op.Path + "\x00") + } + } + if _, err := l.runGit(ctx, dir, env, strings.NewReader(in.String()), "update-index", "-z", "--index-info"); err != nil { + return fmt.Errorf("%w: %s", ErrInvalidOps, err) + } + return nil +} + +// runGit executes one short-lived git command with the repo as context, +// applying the standard timeout, and returns its trimmed stdout +func (l *Local) runGit(ctx context.Context, dir string, env []string, stdin io.Reader, args ...string) (string, error) { ctx, cancel := context.WithTimeout(ctx, l.timeout) defer cancel() - cmd := exec.CommandContext(ctx, l.gitPath, "-C", dir, "for-each-ref", "--format=%(objectname) %(refname)") + cmd := exec.CommandContext(ctx, l.gitPath, append([]string{"-C", dir}, args...)...) + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + cmd.Stdin = stdin var out, stderr bytes.Buffer cmd.Stdout = &out cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("for-each-ref: %w: %s", err, strings.TrimSpace(stderr.String())) + return "", fmt.Errorf("git %s: %w: %s", args[0], err, strings.TrimSpace(stderr.String())) } + return strings.TrimSpace(out.String()), nil +} - refs := make(map[string]string) - sc := bufio.NewScanner(&out) - for sc.Scan() { - // cut the string by " ", to separate sha from ref - sha, ref, ok := strings.Cut(sc.Text(), " ") - if !ok { +// revParse resolves a rev expression to an object id, failing when the object does not exist +func (l *Local) revParse(ctx context.Context, dir, spec string) (string, error) { + return l.runGit(ctx, dir, nil, nil, "rev-parse", "--verify", "--end-of-options", spec) +} + +func parseLsTree(out []byte, treePath string) ([]TreeEntry, bool, error) { + var entries []TreeEntry + // each record has a shape like " \t" + for record := range bytes.SplitSeq(out, []byte{0}) { + if len(record) == 0 { continue } - refs[ref] = sha + if len(entries) == maxTreeEntries { + return entries, true, nil + } + + // header never contains a tab, but a filename can, so cut at the first one + header, name, ok := bytes.Cut(record, []byte{'\t'}) + if !ok { + return nil, false, fmt.Errorf("malformed ls-tree record: %q", record) + } + fields := strings.Fields(string(header)) + if len(fields) != 4 { + return nil, false, fmt.Errorf("malformed ls-tree header: %q", header) + } + + size := int64(-1) + // if size is "-" -> its non-blob item -> size = -1 + if fields[3] != "-" { + parsed, err := strconv.ParseInt(fields[3], 10, 64) + if err != nil { + return nil, false, fmt.Errorf("malformed ls-tree size %q: %w", fields[3], err) + } + size = parsed + } + + entries = append(entries, TreeEntry{ + Mode: fields[0], + Type: fields[1], + SHA: fields[2], + Size: size, + Path: path.Join(treePath, string(name)), + }) } - return refs, sc.Err() + return entries, false, nil +} + +// normalizeRev validates an untrusted revision expression; empty means HEAD +func normalizeRev(rev string) (string, error) { + if rev == "" { + return "HEAD", nil + } + if strings.HasPrefix(rev, "-") { + return "", fmt.Errorf("%w: %q", ErrInvalidRev, rev) + } + for _, r := range rev { + if r < 0x20 || r == 0x7f { + return "", fmt.Errorf("%w: control character", ErrInvalidRev) + } + } + return rev, nil +} + +// normalizeTreePath validates an untrusted tree path and normalizes it +func normalizeTreePath(p string) (string, error) { + if strings.ContainsRune(p, 0) { + return "", fmt.Errorf("%w: contains NUL", ErrInvalidPath) + } + return path.Clean("/" + p)[1:], nil } func (l *Local) lockRepo(storagePath string) func() { @@ -246,3 +844,27 @@ func (l *Local) resolve(storagePath string) (string, error) { } return full, nil } + +func isHexSHA(s string) bool { + if len(s) != 40 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +// just to keep track how much bytes were streamed +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} diff --git a/internal/gitbackend/local_test.go b/internal/gitbackend/local_test.go index 2461b76..7dc6b77 100644 --- a/internal/gitbackend/local_test.go +++ b/internal/gitbackend/local_test.go @@ -1,12 +1,45 @@ package gitbackend import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" "path/filepath" "sort" "strings" "testing" + "time" ) +// runs a git command in dir with a deterministic identity, failing the test on error +func gitRun(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } +} + +// like gitRun but returns the trimmed stdout +func gitOut(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + out, err := cmd.Output() + if err != nil { + t.Fatalf("git %v: %v", args, err) + } + return strings.TrimSpace(string(out)) +} + func TestResolveContainment(t *testing.T) { root := "/srv/repos" l := &Local{root: root} @@ -40,6 +73,640 @@ func TestResolveContainment(t *testing.T) { } } +func TestNormalizeRev(t *testing.T) { + cases := []struct { + name, rev, want string + wantErr error + }{ + {"empty defaults to HEAD", "", "HEAD", nil}, + {"branch", "main", "main", nil}, + {"expression", "main~2", "main~2", nil}, + {"option injection", "--help", "", ErrInvalidRev}, + {"leading dash", "-x", "", ErrInvalidRev}, + {"newline", "main\nx", "", ErrInvalidRev}, + {"nul", "main\x00", "", ErrInvalidRev}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeRev(tc.rev) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("normalizeRev(%q) error = %v, want %v", tc.rev, err, tc.wantErr) + } + if got != tc.want { + t.Errorf("normalizeRev(%q) = %q, want %q", tc.rev, got, tc.want) + } + }) + } +} + +func TestNormalizeTreePath(t *testing.T) { + cases := []struct { + name, path, want string + wantErr error + }{ + {"empty is root", "", "", nil}, + {"dot is root", ".", "", nil}, + {"slash is root", "/", "", nil}, + {"plain", "src", "src", nil}, + {"trailing slash", "src/", "src", nil}, + {"traversal contained", "../../etc", "etc", nil}, + {"dotdot in middle", "a/../b", "b", nil}, + {"nul", "src\x00", "", ErrInvalidPath}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeTreePath(tc.path) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("normalizeTreePath(%q) error = %v, want %v", tc.path, err, tc.wantErr) + } + if got != tc.want { + t.Errorf("normalizeTreePath(%q) = %q, want %q", tc.path, got, tc.want) + } + }) + } +} + +func TestParseLsTree(t *testing.T) { + // real `ls-tree --long -z` framing: size right-padded to 7, "-" for non-blobs + out := []byte("100644 blob aaaa 30\tmain.go\x00" + + "040000 tree bbbb -\tsub dir\x00" + + "100755 blob cccc 123\twith\ttab\x00") + + entries, truncated, err := parseLsTree(out, "src") + if err != nil { + t.Fatal(err) + } + if truncated { + t.Error("unexpected truncation") + } + + want := []TreeEntry{ + {Mode: "100644", Type: "blob", SHA: "aaaa", Size: 30, Path: "src/main.go"}, + {Mode: "040000", Type: "tree", SHA: "bbbb", Size: -1, Path: "src/sub dir"}, + {Mode: "100755", Type: "blob", SHA: "cccc", Size: 123, Path: "src/with\ttab"}, + } + if len(entries) != len(want) { + t.Fatalf("got %d entries, want %d: %+v", len(entries), len(want), entries) + } + for i := range want { + if entries[i] != want[i] { + t.Errorf("entry %d = %+v, want %+v", i, entries[i], want[i]) + } + } +} + +func TestParseLsTreeTruncates(t *testing.T) { + var sb strings.Builder + for i := 0; i < maxTreeEntries+5; i++ { + fmt.Fprintf(&sb, "100644 blob aaaa 1\tf%d\x00", i) + } + + entries, truncated, err := parseLsTree([]byte(sb.String()), "") + if err != nil { + t.Fatal(err) + } + if !truncated { + t.Error("want truncated listing") + } + if len(entries) != maxTreeEntries { + t.Errorf("got %d entries, want cap %d", len(entries), maxTreeEntries) + } +} + +// end-to-end against a real git binary, following the repo convention of +// skipping when git is not on PATH +func TestListTree(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + + root := t.TempDir() + l, err := NewLocal(root) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + if err := l.InitBare(ctx, "1/test.git"); err != nil { + t.Fatal(err) + } + + // empty repo: HEAD is unborn + if _, err := l.ListTree(ctx, "1/test.git", "", ""); !errors.Is(err, ErrRevNotFound) { + t.Fatalf("empty repo: want ErrRevNotFound, got %v", err) + } + + // build a working tree and push it to a known branch name + wt := filepath.Join(t.TempDir(), "wt") + gitRun(t, ".", "clone", filepath.Join(root, "1/test.git"), wt) + writeFile := func(rel, content string, mode os.FileMode) { + t.Helper() + full := filepath.Join(wt, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), mode); err != nil { + t.Fatal(err) + } + } + writeFile("README.md", "hello\n", 0o644) + writeFile("src/main.go", "package main\n", 0o644) + writeFile("src/run.sh", "#!/bin/sh\n", 0o755) + gitRun(t, wt, "add", "-A") + gitRun(t, wt, "commit", "-m", "init") + gitRun(t, wt, "push", "origin", "HEAD:refs/heads/main") + + t.Run("root listing", func(t *testing.T) { + listing, err := l.ListTree(ctx, "1/test.git", "main", "") + if err != nil { + t.Fatal(err) + } + if listing.CommitSHA == "" { + t.Error("want resolved commit sha") + } + if listing.Truncated { + t.Error("unexpected truncation") + } + if len(listing.Entries) != 2 { + t.Fatalf("want 2 entries, got %+v", listing.Entries) + } + if listing.Entries[0].Path != "README.md" || listing.Entries[0].Type != "blob" || listing.Entries[0].Size != 6 { + t.Errorf("README.md entry = %+v", listing.Entries[0]) + } + if listing.Entries[1].Path != "src" || listing.Entries[1].Type != "tree" || listing.Entries[1].Size != -1 { + t.Errorf("src entry = %+v", listing.Entries[1]) + } + }) + + t.Run("subdir listing", func(t *testing.T) { + listing, err := l.ListTree(ctx, "1/test.git", "main", "src") + if err != nil { + t.Fatal(err) + } + if len(listing.Entries) != 2 { + t.Fatalf("want 2 entries, got %+v", listing.Entries) + } + if listing.Entries[0].Path != "src/main.go" { + t.Errorf("want path src/main.go, got %q", listing.Entries[0].Path) + } + if listing.Entries[1].Mode != "100755" { + t.Errorf("run.sh: want mode 100755, got %q", listing.Entries[1].Mode) + } + }) + + t.Run("errors", func(t *testing.T) { + cases := []struct { + name, rev, path string + want error + }{ + {"unknown rev", "nope", "", ErrRevNotFound}, + {"unknown path", "main", "nope", ErrPathNotFound}, + {"path is a blob", "main", "README.md", ErrPathNotFound}, + {"hostile rev", "--help", "", ErrInvalidRev}, + {"traversal path", "main", "../../etc", ErrPathNotFound}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := l.ListTree(ctx, "1/test.git", tc.rev, tc.path); !errors.Is(err, tc.want) { + t.Errorf("ListTree(%q, %q) = %v, want %v", tc.rev, tc.path, err, tc.want) + } + }) + } + }) +} + +func TestArchiveTar(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + + root := t.TempDir() + l, err := NewLocal(root) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + if err := l.InitBare(ctx, "1/test.git"); err != nil { + t.Fatal(err) + } + + // empty repo: HEAD is unborn + if _, err := l.ArchiveTar(ctx, "1/test.git", "", io.Discard); !errors.Is(err, ErrRevNotFound) { + t.Fatalf("empty repo: want ErrRevNotFound, got %v", err) + } + + // an LFS-pointer-shaped blob: the archive must carry it byte-for-byte, + // smudging is explicitly not this layer's job + pointer := "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:" + strings.Repeat("a", 64) + "\n" + + "size 12345\n" + + wt := filepath.Join(t.TempDir(), "wt") + gitRun(t, ".", "clone", filepath.Join(root, "1/test.git"), wt) + files := map[string]string{ + "README.md": "hello\n", + "src/main.go": "package main\n", + "big.bin": pointer, + } + for rel, content := range files { + full := filepath.Join(wt, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + gitRun(t, wt, "add", "-A") + gitRun(t, wt, "commit", "-m", "init") + gitRun(t, wt, "push", "origin", "HEAD:refs/heads/main") + + var buf bytes.Buffer + sha, err := l.ArchiveTar(ctx, "1/test.git", "main", &buf) + if err != nil { + t.Fatal(err) + } + if len(sha) != 40 { + t.Errorf("want 40-hex commit sha, got %q", sha) + } + + // the stream must be a valid tar containing exactly the committed files + got := map[string]string{} + tr := tar.NewReader(&buf) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if hdr.Typeflag != tar.TypeReg { // skip dirs and the pax global header + continue + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + got[hdr.Name] = string(body) + } + if len(got) != len(files) { + t.Fatalf("got %d regular files, want %d: %v", len(got), len(files), got) + } + for rel, content := range files { + if got[rel] != content { + t.Errorf("%s = %q, want %q", rel, got[rel], content) + } + } + + if _, err := l.ArchiveTar(ctx, "1/test.git", "nope", io.Discard); !errors.Is(err, ErrRevNotFound) { + t.Errorf("unknown rev: want ErrRevNotFound, got %v", err) + } + if _, err := l.ArchiveTar(ctx, "1/test.git", "--help", io.Discard); !errors.Is(err, ErrInvalidRev) { + t.Errorf("hostile rev: want ErrInvalidRev, got %v", err) + } +} + +func TestBlob(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + + root := t.TempDir() + l, err := NewLocal(root) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + if err := l.InitBare(ctx, "1/test.git"); err != nil { + t.Fatal(err) + } + + wt := filepath.Join(t.TempDir(), "wt") + gitRun(t, ".", "clone", filepath.Join(root, "1/test.git"), wt) + if err := os.MkdirAll(filepath.Join(wt, "src"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(wt, "src", "main.go"), []byte("package main\n"), 0o644); err != nil { + t.Fatal(err) + } + gitRun(t, wt, "add", "-A") + gitRun(t, wt, "commit", "-m", "init") + gitRun(t, wt, "push", "origin", "HEAD:refs/heads/main") + + info, err := l.StatBlob(ctx, "1/test.git", "main", "src/main.go") + if err != nil { + t.Fatal(err) + } + if info.Size != int64(len("package main\n")) { + t.Errorf("size = %d", info.Size) + } + if !isHexSHA(info.CommitSHA) || !isHexSHA(info.BlobSHA) { + t.Errorf("shas not resolved: %+v", info) + } + + var buf bytes.Buffer + if err := l.ReadBlob(ctx, "1/test.git", info.BlobSHA, &buf); err != nil { + t.Fatal(err) + } + if buf.String() != "package main\n" { + t.Errorf("content = %q", buf.String()) + } + + t.Run("write blob", func(t *testing.T) { + // git blob shas are deterministic: "hello\n" is famously 0xce0136... + sha, size, err := l.WriteBlob(ctx, "1/test.git", strings.NewReader("hello\n")) + if err != nil { + t.Fatal(err) + } + if sha != "ce013625030ba8dba906f756967f9e9ca394464a" { + t.Errorf("sha = %q", sha) + } + if size != 6 { + t.Errorf("size = %d", size) + } + + // the object must be readable back before any commit references it + var buf bytes.Buffer + if err := l.ReadBlob(ctx, "1/test.git", sha, &buf); err != nil { + t.Fatal(err) + } + if buf.String() != "hello\n" { + t.Errorf("content = %q", buf.String()) + } + + // uploading the same content again dedupes to the same sha + again, _, err := l.WriteBlob(ctx, "1/test.git", strings.NewReader("hello\n")) + if err != nil { + t.Fatal(err) + } + if again != sha { + t.Errorf("re-upload sha = %q, want %q", again, sha) + } + + // empty content is the canonical empty blob + empty, size, err := l.WriteBlob(ctx, "1/test.git", strings.NewReader("")) + if err != nil { + t.Fatal(err) + } + if empty != "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" || size != 0 { + t.Errorf("empty blob = %q size %d", empty, size) + } + }) + + t.Run("errors", func(t *testing.T) { + cases := []struct { + name, rev, path string + want error + }{ + {"root is a tree", "main", "", ErrNotABlob}, + {"dir is a tree", "main", "src", ErrNotABlob}, + {"missing path", "main", "nope.txt", ErrPathNotFound}, + {"unknown rev", "nope", "src/main.go", ErrRevNotFound}, + {"hostile rev", "--help", "src/main.go", ErrInvalidRev}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := l.StatBlob(ctx, "1/test.git", tc.rev, tc.path); !errors.Is(err, tc.want) { + t.Errorf("StatBlob(%q, %q) = %v, want %v", tc.rev, tc.path, err, tc.want) + } + }) + } + + // ReadBlob refuses anything that is not a plain object id + for _, sha := range []string{"main", "--help", "HEAD", strings.Repeat("a", 39), strings.Repeat("A", 40)} { + if err := l.ReadBlob(ctx, "1/test.git", sha, io.Discard); !errors.Is(err, ErrInvalidRev) { + t.Errorf("ReadBlob(%q) = %v, want ErrInvalidRev", sha, err) + } + } + }) +} + +func TestApplyCommit(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + + root := t.TempDir() + l, err := NewLocal(root) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + const repo = "1/test.git" + + if err := l.InitBare(ctx, repo); err != nil { + t.Fatal(err) + } + + author := Identity{Name: "api-user", Email: "api@test"} + spec := func(expectedOld, msg string) CommitSpec { + return CommitSpec{Branch: "main", ExpectedOld: expectedOld, Author: author, Message: msg} + } + blob := func(content string) string { + t.Helper() + sha, _, err := l.WriteBlob(ctx, repo, strings.NewReader(content)) + if err != nil { + t.Fatal(err) + } + return sha + } + + hello := blob("hello\n") + script := blob("#!/bin/sh\n") + + // creating a branch requires explicitly expecting non-existence + if _, err := l.ApplyCommit(ctx, repo, spec("", "init"), []CommitOp{{Path: "README.md", BlobSHA: hello}}, nil); !errors.Is(err, ErrRevNotFound) { + t.Fatalf("missing branch without zero expected-old: want ErrRevNotFound, got %v", err) + } + + first, err := l.ApplyCommit(ctx, repo, spec(zeroSHA, "init"), []CommitOp{ + {Path: "README.md", BlobSHA: hello}, + {Path: "src/run.sh", BlobSHA: script, Mode: "100755"}, + }, nil) + if err != nil { + t.Fatal(err) + } + if first.OldSHA != zeroSHA || !isHexSHA(first.NewSHA) || first.Ref != "refs/heads/main" { + t.Fatalf("first change = %+v", first) + } + + // a real git client must see exactly what we committed + wt := filepath.Join(t.TempDir(), "wt") + gitRun(t, ".", "clone", "-b", "main", filepath.Join(root, repo), wt) + + readme, err := os.ReadFile(filepath.Join(wt, "README.md")) + if err != nil || string(readme) != "hello\n" { + t.Errorf("README.md = %q, %v", readme, err) + } + info, err := os.Stat(filepath.Join(wt, "src", "run.sh")) + if err != nil { + t.Fatal(err) + } + if info.Mode()&0o100 == 0 { + t.Errorf("run.sh not executable: %v", info.Mode()) + } + + logOut := gitOut(t, wt, "log", "-1", "--format=%H|%an|%ae|%s") + if logOut != first.NewSHA+"|api-user|api@test|init" { + t.Errorf("log = %q", logOut) + } + + // second commit: CAS on the known head, update one file, delete another + v2 := blob("hello v2\n") + second, err := l.ApplyCommit(ctx, repo, spec(first.NewSHA, "update"), []CommitOp{ + {Path: "README.md", BlobSHA: v2}, + {Path: "src/run.sh", Delete: true}, + }, nil) + if err != nil { + t.Fatal(err) + } + if second.OldSHA != first.NewSHA { + t.Errorf("second change = %+v", second) + } + + gitRun(t, wt, "pull") + if readme, _ := os.ReadFile(filepath.Join(wt, "README.md")); string(readme) != "hello v2\n" { + t.Errorf("README.md after pull = %q", readme) + } + if _, err := os.Stat(filepath.Join(wt, "src", "run.sh")); !os.IsNotExist(err) { + t.Errorf("run.sh should be deleted, stat err = %v", err) + } + + t.Run("errors", func(t *testing.T) { + cases := []struct { + name string + spec CommitSpec + ops []CommitOp + want error + }{ + {"stale cas", spec(first.NewSHA, "x"), []CommitOp{{Path: "a", BlobSHA: hello}}, ErrHeadMismatch}, + {"create existing branch", spec(zeroSHA, "x"), []CommitOp{{Path: "a", BlobSHA: hello}}, ErrHeadMismatch}, + {"unknown blob", spec("", "x"), []CommitOp{{Path: "a", BlobSHA: strings.Repeat("d", 40)}}, ErrUnknownBlob}, + {"nothing to commit", spec("", "x"), []CommitOp{{Path: "README.md", BlobSHA: v2}}, ErrNothingToCommit}, + {"delete missing path", spec("", "x"), []CommitOp{{Path: "nope.txt", Delete: true}}, ErrPathNotFound}, + {"bad branch", CommitSpec{Branch: "a..b", ExpectedOld: "", Author: author, Message: "x"}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidBranch}, + {"hostile branch", CommitSpec{Branch: "--help", Author: author, Message: "x"}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidBranch}, + {"no ops", spec("", "x"), nil, ErrInvalidOps}, + {"duplicate path", spec("", "x"), []CommitOp{{Path: "a", BlobSHA: hello}, {Path: "a", Delete: true}}, ErrInvalidOps}, + {"bad mode", spec("", "x"), []CommitOp{{Path: "a", BlobSHA: hello, Mode: "120000"}}, ErrInvalidOps}, + {"missing author", CommitSpec{Branch: "main", Author: Identity{}, Message: "x"}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidOps}, + {"missing message", CommitSpec{Branch: "main", Author: author}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidOps}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := l.ApplyCommit(ctx, repo, tc.spec, tc.ops, nil); !errors.Is(err, tc.want) { + t.Errorf("ApplyCommit = %v, want %v", err, tc.want) + } + }) + } + }) + + t.Run("lfs clean", func(t *testing.T) { + attrs := blob("*.bin filter=lfs diff=lfs merge=lfs -text\n") + payload := blob("REAL BINARY CONTENT") + pointerText := "version https://git-lfs.github.com/spec/v1\noid sha256:" + strings.Repeat("ab", 32) + "\nsize 19\n" + pointer := blob(pointerText) + + // tracked path without a clean filter fails loudly + if _, err := l.ApplyCommit(ctx, repo, spec("", "track"), []CommitOp{ + {Path: ".gitattributes", BlobSHA: attrs}, + {Path: "big.bin", BlobSHA: payload}, + }, nil); !errors.Is(err, ErrLFSRequired) { + t.Fatalf("want ErrLFSRequired, got %v", err) + } + + // tracking added in the SAME commit must clean the file next to it + var gotPath, gotSHA string + var gotSize int64 + clean := func(path, blobSHA string, size int64) (string, error) { + gotPath, gotSHA, gotSize = path, blobSHA, size + return pointer, nil + } + change, err := l.ApplyCommit(ctx, repo, spec("", "track + add"), []CommitOp{ + {Path: ".gitattributes", BlobSHA: attrs}, + {Path: "big.bin", BlobSHA: payload}, + {Path: "notes.txt", BlobSHA: hello}, // untracked, must NOT be cleaned + }, clean) + if err != nil { + t.Fatal(err) + } + if gotPath != "big.bin" || gotSHA != payload || gotSize != 19 { + t.Errorf("clean called with (%q, %q, %d)", gotPath, gotSHA, gotSize) + } + + // the committed tree holds the pointer, not the payload + committed, err := l.StatBlob(ctx, repo, change.NewSHA, "big.bin") + if err != nil { + t.Fatal(err) + } + if committed.BlobSHA != pointer { + t.Errorf("big.bin blob = %s, want pointer %s", committed.BlobSHA, pointer) + } + notes, err := l.StatBlob(ctx, repo, change.NewSHA, "notes.txt") + if err != nil { + t.Fatal(err) + } + if notes.BlobSHA != hello { + t.Errorf("notes.txt was cleaned but is not lfs-tracked") + } + }) +} + +func TestGC(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + + root := t.TempDir() + l, err := NewLocal(root) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + const repo = "1/test.git" + + if err := l.InitBare(ctx, repo); err != nil { + t.Fatal(err) + } + + // a fresh orphan must survive gc: the prune grace period protects + // uploads whose commit has not happened yet + fresh, _, err := l.WriteBlob(ctx, repo, strings.NewReader("pending upload\n")) + if err != nil { + t.Fatal(err) + } + if err := l.GC(ctx, repo); err != nil { + t.Fatal(err) + } + if err := l.ReadBlob(ctx, repo, fresh, io.Discard); err != nil { + t.Fatalf("fresh orphan pruned: %v", err) + } + + // an orphan backdated past gc.pruneExpire (2 weeks) must be pruned; + // backdate before gc runs, since gc moves loose objects into cruft packs + orphan, _, err := l.WriteBlob(ctx, repo, strings.NewReader("abandoned upload\n")) + if err != nil { + t.Fatal(err) + } + loose := filepath.Join(root, repo, "objects", orphan[:2], orphan[2:]) + old := time.Now().Add(-21 * 24 * time.Hour) + if err := os.Chtimes(loose, old, old); err != nil { + t.Fatal(err) + } + + if err := l.GC(ctx, repo); err != nil { + t.Fatal(err) + } + if err := l.ReadBlob(ctx, repo, orphan, io.Discard); err == nil { + t.Error("expired orphan survived gc") + } +} + func TestDiffRefs(t *testing.T) { cases := []struct { name string diff --git a/internal/gitbackend/types.go b/internal/gitbackend/types.go index 8479546..cfb1622 100644 --- a/internal/gitbackend/types.go +++ b/internal/gitbackend/types.go @@ -5,8 +5,58 @@ package gitbackend // or its zero after, if it was deleted before const zeroSHA = "0000000000000000000000000000000000000000" +// hard cap on entries returned per directory level +const maxTreeEntries = 10_000 + +// hard cap on operations per commit +const maxCommitOps = 1000 + type RefChange struct { Ref string OldSHA string NewSHA string } + +type TreeEntry struct { + Mode string // "100644", "100755", "040000", "120000", "160000" + Type string // "blob" | "tree" | "commit" + SHA string + Size int64 // object size in bytes, -1 where git reports none (trees, submodules) + Path string // full path from the repo root +} + +type TreeListing struct { + CommitSHA string // the exact commit the listing is a snapshot of + Entries []TreeEntry + Truncated bool +} + +type BlobInfo struct { + CommitSHA string + BlobSHA string + Size int64 +} + +type CommitOp struct { + Delete bool + Path string + BlobSHA string // puts only; must exist as a blob in this repo's odb + Mode string // puts only: "100644" (default) or "100755" +} + +type Identity struct { + Name string + Email string +} + +type CommitSpec struct { + Branch string + // ExpectedOld pins the commit to an exact branch state + ExpectedOld string + Author Identity + Committer Identity // defaults to Author when empty + Message string +} + +// write-side mirror of archive.SmudgeFunc +type CleanFunc func(path, blobSHA string, size int64) (string, error) diff --git a/internal/server/control/repositories/archive.go b/internal/server/control/repositories/archive.go new file mode 100644 index 0000000..ede4e74 --- /dev/null +++ b/internal/server/control/repositories/archive.go @@ -0,0 +1,107 @@ +package repositories + +import ( + "errors" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/Axenos-dev/HeadlessGit/internal/domain" + "github.com/Axenos-dev/HeadlessGit/internal/server/response" + reposervice "github.com/Axenos-dev/HeadlessGit/internal/services/repositories" + "github.com/go-chi/chi/v5" + "go.uber.org/zap" +) + +func (h *handlers) getArchive(w http.ResponseWriter, r *http.Request) error { + id, err := strconv.ParseInt(chi.URLParam(r, "repositoryID"), 10, 64) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid repository id") + } + + q := r.URL.Query() + includeLFS := false + if raw := q.Get("lfs"); raw != "" { + includeLFS, err = strconv.ParseBool(raw) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "lfs must be a boolean") + } + } + + req, err := h.service.PrepareArchive(r.Context(), id, q.Get("ref"), q.Get("format"), includeLFS) + switch { + case errors.Is(err, reposervice.ErrRepositoryNotFound): + return response.NewError(http.StatusNotFound, response.CodeRepositoryNotFound, "repository not found") + case errors.Is(err, reposervice.ErrRefNotFound): + return response.NewError(http.StatusNotFound, response.CodeRefNotFound, "ref not found") + case errors.Is(err, reposervice.ErrInvalidRef): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid ref") + case errors.Is(err, reposervice.ErrUnsupportedFormat): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "unsupported archive format") + case errors.Is(err, reposervice.ErrLFSNotEnabled): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "lfs is not enabled") + case err != nil: + h.logger.Error("failed to prepare archive", zap.Error(err)) + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to prepare archive") + } + + etag := archiveETag(req) + if r.Header.Get("If-None-Match") == etag { + w.Header().Set("ETag", etag) + w.WriteHeader(http.StatusNotModified) + return nil + } + + contentType := "application/zip" + if req.Format == domain.ArchiveFormatTarGz { + contentType = "application/gzip" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", req.Filename())) + w.Header().Set("ETag", etag) + w.Header().Set("X-HeadlessGit-Commit", req.CommitSHA) + + cw := &countingWriter{w: w} + if err := h.service.StreamArchive(r.Context(), req, cw); err != nil { + // nothing sent yet, + if cw.n == 0 { + // undo the archive headers + w.Header().Del("Content-Type") + w.Header().Del("Content-Disposition") + w.Header().Del("ETag") + w.Header().Del("X-HeadlessGit-Commit") + h.logger.Error("failed to stream archive", zap.Int64("repository_id", id), zap.Error(err)) + // and return normal error + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to stream archive") + } + + // otherwise just log, the bytes were already streamed + h.logger.Error("archive stream aborted mid-flight", + zap.Int64("repository_id", id), + zap.Int64("bytes_written", cw.n), + zap.Error(err), + ) + } + return nil +} + +func archiveETag(req domain.ArchiveRequest) string { + variant := string(req.Format) + if req.IncludeLFS { + variant += "-lfs" + } + return fmt.Sprintf(`W/"%s-%s"`, req.CommitSHA, variant) +} + +// just to keep track how much bytes were streamed +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} diff --git a/internal/server/control/repositories/blob.go b/internal/server/control/repositories/blob.go new file mode 100644 index 0000000..fc57edd --- /dev/null +++ b/internal/server/control/repositories/blob.go @@ -0,0 +1,119 @@ +package repositories + +import ( + "errors" + "fmt" + "net/http" + "path" + "strconv" + + "github.com/Axenos-dev/HeadlessGit/internal/domain" + "github.com/Axenos-dev/HeadlessGit/internal/server/response" + reposervice "github.com/Axenos-dev/HeadlessGit/internal/services/repositories" + "github.com/go-chi/chi/v5" + "go.uber.org/zap" +) + +func (h *handlers) uploadBlob(w http.ResponseWriter, r *http.Request) error { + id, err := strconv.ParseInt(chi.URLParam(r, "repositoryID"), 10, 64) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid repository id") + } + + sha, byteCount, err := h.service.WriteBlob(r.Context(), id, r.Body) + switch { + case errors.Is(err, reposervice.ErrRepositoryNotFound): + return response.NewError(http.StatusNotFound, response.CodeRepositoryNotFound, "repository not found") + case err != nil: + h.logger.Error("failed to write blob", zap.Error(err)) + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to write blob") + } + + return response.Data(w, http.StatusCreated, UploadBlobResponse{ + SHA: sha, + Size: byteCount, + }) +} + +func (h *handlers) getBlob(w http.ResponseWriter, r *http.Request) error { + id, err := strconv.ParseInt(chi.URLParam(r, "repositoryID"), 10, 64) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid repository id") + } + + q := r.URL.Query() + includeLFS := false + if raw := q.Get("lfs"); raw != "" { + includeLFS, err = strconv.ParseBool(raw) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "lfs must be a boolean") + } + } + + req, err := h.service.PrepareBlob(r.Context(), id, q.Get("ref"), q.Get("path"), includeLFS) + switch { + case errors.Is(err, reposervice.ErrRepositoryNotFound): + return response.NewError(http.StatusNotFound, response.CodeRepositoryNotFound, "repository not found") + case errors.Is(err, reposervice.ErrRefNotFound): + return response.NewError(http.StatusNotFound, response.CodeRefNotFound, "ref not found") + case errors.Is(err, reposervice.ErrPathNotFound): + return response.NewError(http.StatusNotFound, response.CodePathNotFound, "path not found") + case errors.Is(err, reposervice.ErrLFSObjectNotFound): + return response.NewError(http.StatusNotFound, response.CodeLFSObjectNotFound, "lfs object not found") + case errors.Is(err, reposervice.ErrNotAFile): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "path is not a file, use the contents endpoint") + case errors.Is(err, reposervice.ErrInvalidRef): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid ref") + case errors.Is(err, reposervice.ErrInvalidPath): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid path") + case errors.Is(err, reposervice.ErrLFSNotEnabled): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "lfs is not enabled") + case err != nil: + h.logger.Error("failed to prepare blob", zap.Error(err)) + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to prepare blob") + } + + etag := blobETag(req) + if r.Header.Get("If-None-Match") == etag { + w.Header().Set("ETag", etag) + w.WriteHeader(http.StatusNotModified) + return nil + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(req.Size, 10)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", path.Base(req.Path))) + w.Header().Set("ETag", etag) + w.Header().Set("X-HeadlessGit-Commit", req.CommitSHA) + + cw := &countingWriter{w: w} + if err := h.service.StreamBlob(r.Context(), req, cw); err != nil { + // nothing sent yet + if cw.n == 0 { + // undo the blob headers + w.Header().Del("Content-Type") + w.Header().Del("Content-Length") + w.Header().Del("Content-Disposition") + w.Header().Del("ETag") + w.Header().Del("X-HeadlessGit-Commit") + h.logger.Error("failed to stream blob", zap.Int64("repository_id", id), zap.Error(err)) + // and return normal error + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to stream blob") + } + + // otherwise just log, the bytes were already streamed + h.logger.Error("blob stream aborted mid-flight", + zap.Int64("repository_id", id), + zap.Int64("bytes_written", cw.n), + zap.Error(err), + ) + } + return nil +} + +func blobETag(req domain.BlobRequest) string { + if req.LFSOID != "" { + return fmt.Sprintf(`"%s-lfs"`, req.BlobSHA) + } + return fmt.Sprintf(`"%s"`, req.BlobSHA) +} diff --git a/internal/server/control/repositories/commits.go b/internal/server/control/repositories/commits.go new file mode 100644 index 0000000..908309c --- /dev/null +++ b/internal/server/control/repositories/commits.go @@ -0,0 +1,75 @@ +package repositories + +import ( + "encoding/json" + "errors" + "net/http" + "strconv" + + "github.com/Axenos-dev/HeadlessGit/internal/domain" + "github.com/Axenos-dev/HeadlessGit/internal/server/response" + reposervice "github.com/Axenos-dev/HeadlessGit/internal/services/repositories" + "github.com/go-chi/chi/v5" + "go.uber.org/zap" +) + +func (h *handlers) createCommit(w http.ResponseWriter, r *http.Request) error { + id, err := strconv.ParseInt(chi.URLParam(r, "repositoryID"), 10, 64) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid repository id") + } + + var req CreateCommitRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid request body") + } + if err := req.Validate(); err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, err.Error()) + } + + ops := make([]domain.CommitFileOp, len(req.Operations)) + for i, op := range req.Operations { + ops[i] = domain.CommitFileOp{ + Delete: op.Op == "delete", + Path: op.Path, + BlobSHA: op.BlobSHA, + Executable: op.Executable, + } + } + + result, err := h.service.Commit(r.Context(), id, domain.CommitRequest{ + Branch: req.Branch, + Message: req.Message, + Author: domain.CommitIdentity{Name: req.Author.Name, Email: req.Author.Email}, + ExpectedHeadSHA: req.ExpectedHeadSHA, + PusherID: req.PusherID, + Operations: ops, + }) + switch { + case errors.Is(err, reposervice.ErrRepositoryNotFound): + return response.NewError(http.StatusNotFound, response.CodeRepositoryNotFound, "repository not found") + case errors.Is(err, reposervice.ErrRefNotFound): + return response.NewError(http.StatusNotFound, response.CodeRefNotFound, "branch not found; pass the all-zero expectedHeadSha to create it") + case errors.Is(err, reposervice.ErrPathNotFound): + return response.NewError(http.StatusNotFound, response.CodePathNotFound, "delete target not found") + case errors.Is(err, reposervice.ErrHeadMismatch): + return response.NewError(http.StatusConflict, response.CodeHeadMismatch, "branch head does not match expectedHeadSha") + case errors.Is(err, reposervice.ErrUnknownBlob): + return response.NewError(http.StatusUnprocessableEntity, response.CodeUnknownBlob, "referenced blob not found, upload it first") + case errors.Is(err, reposervice.ErrNothingToCommit): + return response.NewError(http.StatusUnprocessableEntity, response.CodeNothingToCommit, "operations produce no change") + case errors.Is(err, reposervice.ErrNotAFile): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "delete target is not a file") + case errors.Is(err, reposervice.ErrInvalidBranch): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid branch name") + case errors.Is(err, reposervice.ErrInvalidCommitOps): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, err.Error()) + case errors.Is(err, reposervice.ErrLFSNotEnabled): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "path is lfs-tracked but lfs is not enabled") + case err != nil: + h.logger.Error("failed to create commit", zap.Error(err)) + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to create commit") + } + + return response.Data(w, http.StatusCreated, newCommit(result)) +} diff --git a/internal/server/control/repositories/contents.go b/internal/server/control/repositories/contents.go new file mode 100644 index 0000000..73ea467 --- /dev/null +++ b/internal/server/control/repositories/contents.go @@ -0,0 +1,43 @@ +package repositories + +import ( + "errors" + "net/http" + "strconv" + + "github.com/Axenos-dev/HeadlessGit/internal/server/response" + reposervice "github.com/Axenos-dev/HeadlessGit/internal/services/repositories" + "github.com/go-chi/chi/v5" + "go.uber.org/zap" +) + +func (h *handlers) getContents(w http.ResponseWriter, r *http.Request) error { + id, err := strconv.ParseInt(chi.URLParam(r, "repositoryID"), 10, 64) + if err != nil { + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid repository id") + } + + // ref defaults to HEAD + ref := r.URL.Query().Get("ref") + // path defaults to the repo root + treePath := r.URL.Query().Get("path") + + contents, err := h.service.Contents(r.Context(), id, ref, treePath) + switch { + case errors.Is(err, reposervice.ErrRepositoryNotFound): + return response.NewError(http.StatusNotFound, response.CodeRepositoryNotFound, "repository not found") + case errors.Is(err, reposervice.ErrRefNotFound): + return response.NewError(http.StatusNotFound, response.CodeRefNotFound, "ref not found") + case errors.Is(err, reposervice.ErrPathNotFound): + return response.NewError(http.StatusNotFound, response.CodePathNotFound, "path not found") + case errors.Is(err, reposervice.ErrInvalidRef): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid ref") + case errors.Is(err, reposervice.ErrInvalidPath): + return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid path") + case err != nil: + h.logger.Error("failed to list repository contents", zap.Error(err)) + return response.NewError(http.StatusInternalServerError, response.CodeInternalError, "failed to list repository contents") + } + + return response.Data(w, http.StatusOK, newContents(contents)) +} diff --git a/internal/server/control/repositories/handlers.go b/internal/server/control/repositories/handlers.go index 51affd4..4158525 100644 --- a/internal/server/control/repositories/handlers.go +++ b/internal/server/control/repositories/handlers.go @@ -2,6 +2,7 @@ package repositories import ( "context" + "io" "github.com/Axenos-dev/HeadlessGit/internal/domain" "github.com/Axenos-dev/HeadlessGit/internal/server/response" @@ -15,6 +16,13 @@ type RepositoryManager interface { Delete(ctx context.Context, repositoryID int64) error SetVisibility(ctx context.Context, repositoryID int64, visibility domain.RepoVisibility) (domain.Repository, error) ListByOwner(ctx context.Context, ownerID int64) ([]domain.Repository, error) + Contents(ctx context.Context, repositoryID int64, ref, treePath string) (domain.RepositoryContents, error) + PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool) (domain.ArchiveRequest, error) + StreamArchive(ctx context.Context, req domain.ArchiveRequest, out io.Writer) error + PrepareBlob(ctx context.Context, repositoryID int64, ref, treePath string, includeLFS bool) (domain.BlobRequest, error) + StreamBlob(ctx context.Context, req domain.BlobRequest, out io.Writer) error + WriteBlob(ctx context.Context, repositoryID int64, in io.Reader) (string, int64, error) + Commit(ctx context.Context, repositoryID int64, req domain.CommitRequest) (domain.CommitResult, error) } type handlers struct { @@ -32,7 +40,12 @@ func NewHandlers(logger *zap.Logger, service RepositoryManager) *handlers { func (h *handlers) RegisterRoutes(parent chi.Router) { parent.Route("/repositories", func(r chi.Router) { r.Post("/", response.Handler(h.logger, h.createRepository)) + r.Post("/{repositoryID}/blobs", response.Handler(h.logger, h.uploadBlob)) + r.Post("/{repositoryID}/commits", response.Handler(h.logger, h.createCommit)) r.Get("/{repositoryID}", response.Handler(h.logger, h.getRepository)) + r.Get("/{repositoryID}/contents", response.Handler(h.logger, h.getContents)) + r.Get("/{repositoryID}/archive", response.Handler(h.logger, h.getArchive)) + r.Get("/{repositoryID}/blob", response.Handler(h.logger, h.getBlob)) r.Put("/{repositoryID}/visibility", response.Handler(h.logger, h.setVisibility)) r.Delete("/{repositoryID}", response.Handler(h.logger, h.deleteRepository)) }) diff --git a/internal/server/control/repositories/handlers_test.go b/internal/server/control/repositories/handlers_test.go new file mode 100644 index 0000000..3099943 --- /dev/null +++ b/internal/server/control/repositories/handlers_test.go @@ -0,0 +1,505 @@ +package repositories + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Axenos-dev/HeadlessGit/internal/domain" + reposervice "github.com/Axenos-dev/HeadlessGit/internal/services/repositories" + "github.com/go-chi/chi/v5" + "go.uber.org/zap" +) + +const testSHA = "aaaabbbbccccddddeeeeffff0000111122223333" + +// fakeManager stubs RepositoryManager for handler tests: embed the interface +// and override only what the endpoint under test touches +type fakeManager struct { + RepositoryManager + prepareReq domain.ArchiveRequest + prepareErr error + streamBody string + streamErr error + + blobReq domain.BlobRequest + blobErr error + + writeSHA string + writeErr error + + commitResult domain.CommitResult + commitErr error + commitReq domain.CommitRequest + commitCalled bool +} + +func (f *fakeManager) Commit(ctx context.Context, repositoryID int64, req domain.CommitRequest) (domain.CommitResult, error) { + f.commitCalled = true + f.commitReq = req + return f.commitResult, f.commitErr +} + +func (f fakeManager) PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool) (domain.ArchiveRequest, error) { + return f.prepareReq, f.prepareErr +} + +func (f fakeManager) StreamArchive(ctx context.Context, req domain.ArchiveRequest, out io.Writer) error { + if f.streamBody != "" { + zw := zip.NewWriter(out) + w, err := zw.Create("file.txt") + if err != nil { + return err + } + if _, err := w.Write([]byte(f.streamBody)); err != nil { + return err + } + if err := zw.Close(); err != nil { + return err + } + } + return f.streamErr +} + +func (f fakeManager) PrepareBlob(ctx context.Context, repositoryID int64, ref, treePath string, includeLFS bool) (domain.BlobRequest, error) { + return f.blobReq, f.blobErr +} + +func (f fakeManager) StreamBlob(ctx context.Context, req domain.BlobRequest, out io.Writer) error { + if f.streamBody != "" { + if _, err := io.WriteString(out, f.streamBody); err != nil { + return err + } + } + return f.streamErr +} + +func (f fakeManager) WriteBlob(ctx context.Context, repositoryID int64, in io.Reader) (string, int64, error) { + n, err := io.Copy(io.Discard, in) // consume the stream like the real thing + if err != nil { + return "", 0, err + } + if f.writeErr != nil { + return "", 0, f.writeErr + } + return f.writeSHA, n, nil +} + +// newTestRouter mounts the handlers the same way the control server does +func newTestRouter(svc RepositoryManager) http.Handler { + r := chi.NewRouter() + NewHandlers(zap.NewNop(), svc).RegisterRoutes(r) + return r +} + +func testArchiveRequest() domain.ArchiveRequest { + return domain.ArchiveRequest{ + Repository: domain.Repository{ID: 7, RepositoryName: "myrepo"}, + CommitSHA: testSHA, + Format: domain.ArchiveFormatZip, + } +} + +func TestGetArchive(t *testing.T) { + router := newTestRouter(&fakeManager{prepareReq: testArchiveRequest(), streamBody: "hello"}) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/repositories/7/archive?ref=main", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body.String()) + } + if got := rec.Header().Get("Content-Type"); got != "application/zip" { + t.Errorf("Content-Type = %q", got) + } + if got := rec.Header().Get("Content-Disposition"); got != `attachment; filename="myrepo-aaaabbbbcccc.zip"` { + t.Errorf("Content-Disposition = %q", got) + } + if got := rec.Header().Get("ETag"); got != `W/"`+testSHA+`-zip"` { + t.Errorf("ETag = %q", got) + } + if got := rec.Header().Get("X-HeadlessGit-Commit"); got != testSHA { + t.Errorf("X-HeadlessGit-Commit = %q", got) + } + + // the body must be a readable zip + zr, err := zip.NewReader(bytes.NewReader(rec.Body.Bytes()), int64(rec.Body.Len())) + if err != nil { + t.Fatal(err) + } + if len(zr.File) != 1 || zr.File[0].Name != "file.txt" { + t.Errorf("zip entries = %v", zr.File) + } +} + +func TestGetArchiveNotModified(t *testing.T) { + router := newTestRouter(&fakeManager{prepareReq: testArchiveRequest(), streamBody: "hello"}) + + req := httptest.NewRequest(http.MethodGet, "/repositories/7/archive?ref=main", nil) + req.Header.Set("If-None-Match", `W/"`+testSHA+`-zip"`) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotModified { + t.Fatalf("status = %d", rec.Code) + } + if rec.Body.Len() != 0 { + t.Errorf("304 must have no body, got %d bytes", rec.Body.Len()) + } +} + +func TestGetArchiveErrors(t *testing.T) { + cases := []struct { + name string + target string + prepareErr error + streamErr error + wantStatus int + wantCode string + }{ + {"bad id", "/repositories/abc/archive", nil, nil, http.StatusBadRequest, "invalid_request"}, + {"bad lfs param", "/repositories/7/archive?lfs=maybe", nil, nil, http.StatusBadRequest, "invalid_request"}, + {"repo not found", "/repositories/7/archive", reposervice.ErrRepositoryNotFound, nil, http.StatusNotFound, "repository_not_found"}, + {"ref not found", "/repositories/7/archive?ref=nope", reposervice.ErrRefNotFound, nil, http.StatusNotFound, "ref_not_found"}, + {"invalid ref", "/repositories/7/archive?ref=--x", reposervice.ErrInvalidRef, nil, http.StatusBadRequest, "invalid_request"}, + {"bad format", "/repositories/7/archive?format=rar", reposervice.ErrUnsupportedFormat, nil, http.StatusBadRequest, "invalid_request"}, + {"lfs disabled", "/repositories/7/archive?lfs=true", reposervice.ErrLFSNotEnabled, nil, http.StatusBadRequest, "invalid_request"}, + {"stream fails before first byte", "/repositories/7/archive", nil, io.ErrUnexpectedEOF, http.StatusInternalServerError, "internal_error"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + router := newTestRouter(&fakeManager{prepareReq: testArchiveRequest(), prepareErr: tc.prepareErr, streamErr: tc.streamErr}) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tc.target, nil)) + + if rec.Code != tc.wantStatus { + t.Fatalf("status = %d, want %d: %s", rec.Code, tc.wantStatus, rec.Body.String()) + } + var body struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Error.Code != tc.wantCode { + t.Errorf("code = %q, want %q", body.Error.Code, tc.wantCode) + } + if got := rec.Header().Get("Content-Disposition"); got != "" { + t.Errorf("error response leaked Content-Disposition %q", got) + } + }) + } +} + +func testBlobRequest(lfsOID string) domain.BlobRequest { + return domain.BlobRequest{ + Repository: domain.Repository{ID: 7, RepositoryName: "myrepo"}, + CommitSHA: testSHA, + BlobSHA: "1111222233334444555566667777888899990000", + Path: "src/main.go", + Size: 6, + LFSOID: lfsOID, + } +} + +func TestGetBlob(t *testing.T) { + router := newTestRouter(&fakeManager{blobReq: testBlobRequest(""), streamBody: "hello\n"}) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/repositories/7/blob?ref=main&path=src/main.go", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body.String()) + } + if rec.Body.String() != "hello\n" { + t.Errorf("body = %q", rec.Body.String()) + } + if got := rec.Header().Get("ETag"); got != `"1111222233334444555566667777888899990000"` { + t.Errorf("ETag = %q", got) + } + if got := rec.Header().Get("Content-Length"); got != "6" { + t.Errorf("Content-Length = %q", got) + } + if got := rec.Header().Get("Content-Disposition"); got != `attachment; filename="main.go"` { + t.Errorf("Content-Disposition = %q", got) + } + if got := rec.Header().Get("X-HeadlessGit-Commit"); got != testSHA { + t.Errorf("X-HeadlessGit-Commit = %q", got) + } +} + +func TestGetBlobLFSVariantETag(t *testing.T) { + router := newTestRouter(&fakeManager{blobReq: testBlobRequest("deadbeef"), streamBody: "hello\n"}) + + // the raw etag must not satisfy a smudged request + req := httptest.NewRequest(http.MethodGet, "/repositories/7/blob?ref=main&path=src/main.go&lfs=true", nil) + req.Header.Set("If-None-Match", `"1111222233334444555566667777888899990000"`) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("raw etag must not match lfs variant, status = %d", rec.Code) + } + if got := rec.Header().Get("ETag"); got != `"1111222233334444555566667777888899990000-lfs"` { + t.Errorf("ETag = %q", got) + } +} + +func TestGetBlobNotModified(t *testing.T) { + router := newTestRouter(&fakeManager{blobReq: testBlobRequest(""), streamBody: "hello\n"}) + + req := httptest.NewRequest(http.MethodGet, "/repositories/7/blob?ref=main&path=src/main.go", nil) + req.Header.Set("If-None-Match", `"1111222233334444555566667777888899990000"`) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotModified { + t.Fatalf("status = %d", rec.Code) + } + if rec.Body.Len() != 0 { + t.Errorf("304 must have no body, got %d bytes", rec.Body.Len()) + } +} + +func TestUploadBlob(t *testing.T) { + router := newTestRouter(&fakeManager{writeSHA: "ce013625030ba8dba906f756967f9e9ca394464a"}) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/repositories/7/blobs", strings.NewReader("hello\n"))) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body.String()) + } + var body struct { + Data struct { + SHA string `json:"sha"` + Size int64 `json:"size"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Data.SHA != "ce013625030ba8dba906f756967f9e9ca394464a" { + t.Errorf("sha = %q", body.Data.SHA) + } + if body.Data.Size != 6 { + t.Errorf("size = %d", body.Data.Size) + } +} + +func TestUploadBlobErrors(t *testing.T) { + cases := []struct { + name string + target string + writeErr error + wantStatus int + wantCode string + }{ + {"bad id", "/repositories/abc/blobs", nil, http.StatusBadRequest, "invalid_request"}, + {"repo not found", "/repositories/7/blobs", reposervice.ErrRepositoryNotFound, http.StatusNotFound, "repository_not_found"}, + {"write fails", "/repositories/7/blobs", io.ErrUnexpectedEOF, http.StatusInternalServerError, "internal_error"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + router := newTestRouter(&fakeManager{writeErr: tc.writeErr}) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, tc.target, strings.NewReader("x"))) + + if rec.Code != tc.wantStatus { + t.Fatalf("status = %d, want %d: %s", rec.Code, tc.wantStatus, rec.Body.String()) + } + var body struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Error.Code != tc.wantCode { + t.Errorf("code = %q, want %q", body.Error.Code, tc.wantCode) + } + }) + } +} + +func TestGetBlobErrors(t *testing.T) { + cases := []struct { + name string + target string + blobErr error + streamErr error + wantStatus int + wantCode string + }{ + {"bad id", "/repositories/abc/blob", nil, nil, http.StatusBadRequest, "invalid_request"}, + {"bad lfs param", "/repositories/7/blob?lfs=maybe", nil, nil, http.StatusBadRequest, "invalid_request"}, + {"repo not found", "/repositories/7/blob", reposervice.ErrRepositoryNotFound, nil, http.StatusNotFound, "repository_not_found"}, + {"ref not found", "/repositories/7/blob?ref=nope", reposervice.ErrRefNotFound, nil, http.StatusNotFound, "ref_not_found"}, + {"path not found", "/repositories/7/blob?path=nope", reposervice.ErrPathNotFound, nil, http.StatusNotFound, "path_not_found"}, + {"lfs object missing", "/repositories/7/blob?lfs=true", reposervice.ErrLFSObjectNotFound, nil, http.StatusNotFound, "lfs_object_not_found"}, + {"path is a directory", "/repositories/7/blob?path=src", reposervice.ErrNotAFile, nil, http.StatusBadRequest, "invalid_request"}, + {"invalid ref", "/repositories/7/blob?ref=--x", reposervice.ErrInvalidRef, nil, http.StatusBadRequest, "invalid_request"}, + {"lfs disabled", "/repositories/7/blob?lfs=true", reposervice.ErrLFSNotEnabled, nil, http.StatusBadRequest, "invalid_request"}, + {"stream fails before first byte", "/repositories/7/blob", nil, io.ErrUnexpectedEOF, http.StatusInternalServerError, "internal_error"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + router := newTestRouter(&fakeManager{blobReq: testBlobRequest(""), blobErr: tc.blobErr, streamErr: tc.streamErr}) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tc.target, nil)) + + if rec.Code != tc.wantStatus { + t.Fatalf("status = %d, want %d: %s", rec.Code, tc.wantStatus, rec.Body.String()) + } + var body struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Error.Code != tc.wantCode { + t.Errorf("code = %q, want %q", body.Error.Code, tc.wantCode) + } + if got := rec.Header().Get("Content-Disposition"); got != "" { + t.Errorf("error response leaked Content-Disposition %q", got) + } + }) + } +} + +func validCommitBody() string { + return `{ + "branch": "main", + "message": "update", + "author": {"name": "api-user", "email": "api@test"}, + "expectedHeadSha": "` + strings.Repeat("a", 40) + `", + "pusherId": 42, + "operations": [ + {"op": "put", "path": "run.sh", "blobSha": "` + strings.Repeat("b", 40) + `", "executable": true}, + {"op": "delete", "path": "old.txt"} + ] + }` +} + +func TestCreateCommit(t *testing.T) { + result := domain.CommitResult{Branch: "main", CommitSHA: testSHA, Before: strings.Repeat("a", 40)} + fake := &fakeManager{commitResult: result} + + rec := httptest.NewRecorder() + newTestRouter(fake).ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/repositories/7/commits", strings.NewReader(validCommitBody()))) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body.String()) + } + + var body struct { + Data Commit `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Data.CommitSHA != testSHA || body.Data.Before != result.Before || body.Data.Branch != "main" { + t.Errorf("body = %+v", body.Data) + } + + // the service must receive the fully mapped domain request + req := fake.commitReq + if req.Branch != "main" || req.Message != "update" || req.PusherID != 42 || + req.Author.Name != "api-user" || req.ExpectedHeadSHA != strings.Repeat("a", 40) { + t.Errorf("service request = %+v", req) + } + if len(req.Operations) != 2 || + req.Operations[0].Delete || !req.Operations[0].Executable || req.Operations[0].BlobSHA != strings.Repeat("b", 40) || + !req.Operations[1].Delete || req.Operations[1].Path != "old.txt" { + t.Errorf("service operations = %+v", req.Operations) + } +} + +func TestCreateCommitValidation(t *testing.T) { + cases := []struct { + name string + body string + }{ + {"not json", "nope"}, + {"missing branch", `{"message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"delete","path":"a"}]}`}, + {"missing message", `{"branch":"main","author":{"name":"a","email":"e"},"operations":[{"op":"delete","path":"a"}]}`}, + {"missing author", `{"branch":"main","message":"x","operations":[{"op":"delete","path":"a"}]}`}, + {"no operations", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[]}`}, + {"bad op kind", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"move","path":"a"}]}`}, + {"put without blobSha", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"put","path":"a"}]}`}, + {"delete with blobSha", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"delete","path":"a","blobSha":"abc"}]}`}, + {"missing path", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"delete"}]}`}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeManager{} + rec := httptest.NewRecorder() + newTestRouter(fake).ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/repositories/7/commits", strings.NewReader(tc.body))) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body.String()) + } + if fake.commitCalled { + t.Error("service must not be called on validation failure") + } + }) + } +} + +func TestCreateCommitErrors(t *testing.T) { + cases := []struct { + name string + commitErr error + wantStatus int + wantCode string + }{ + {"repo not found", reposervice.ErrRepositoryNotFound, http.StatusNotFound, "repository_not_found"}, + {"branch not found", reposervice.ErrRefNotFound, http.StatusNotFound, "ref_not_found"}, + {"delete target missing", reposervice.ErrPathNotFound, http.StatusNotFound, "path_not_found"}, + {"head mismatch", reposervice.ErrHeadMismatch, http.StatusConflict, "head_mismatch"}, + {"unknown blob", reposervice.ErrUnknownBlob, http.StatusUnprocessableEntity, "unknown_blob"}, + {"nothing to commit", reposervice.ErrNothingToCommit, http.StatusUnprocessableEntity, "nothing_to_commit"}, + {"delete target is a dir", reposervice.ErrNotAFile, http.StatusBadRequest, "invalid_request"}, + {"invalid branch", reposervice.ErrInvalidBranch, http.StatusBadRequest, "invalid_request"}, + {"invalid ops", reposervice.ErrInvalidCommitOps, http.StatusBadRequest, "invalid_request"}, + {"lfs not enabled", reposervice.ErrLFSNotEnabled, http.StatusBadRequest, "invalid_request"}, + {"internal", io.ErrUnexpectedEOF, http.StatusInternalServerError, "internal_error"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + newTestRouter(&fakeManager{commitErr: tc.commitErr}).ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/repositories/7/commits", strings.NewReader(validCommitBody()))) + + if rec.Code != tc.wantStatus { + t.Fatalf("status = %d, want %d: %s", rec.Code, tc.wantStatus, rec.Body.String()) + } + var body struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Error.Code != tc.wantCode { + t.Errorf("code = %q, want %q", body.Error.Code, tc.wantCode) + } + }) + } +} diff --git a/internal/server/control/repositories/types.go b/internal/server/control/repositories/types.go index 175e626..b34d9b9 100644 --- a/internal/server/control/repositories/types.go +++ b/internal/server/control/repositories/types.go @@ -2,6 +2,7 @@ package repositories import ( "errors" + "fmt" "time" "github.com/Axenos-dev/HeadlessGit/internal/domain" @@ -54,6 +55,52 @@ func newRepositories(repos []domain.Repository) []Repository { return out } +type Contents struct { + Ref string `json:"ref"` + SHA string `json:"sha"` + Path string `json:"path"` + Entries []ContentEntry `json:"entries"` + Truncated bool `json:"truncated,omitempty"` +} + +type ContentEntry struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` // file | dir | symlink | submodule + Mode string `json:"mode"` + Size *int64 `json:"size,omitempty"` // blobs only; note: LFS pointers report pointer size + SHA string `json:"sha"` +} + +func newContents(c domain.RepositoryContents) Contents { + entries := make([]ContentEntry, len(c.Entries)) + for i, e := range c.Entries { + entries[i] = newContentEntry(e) + } + return Contents{ + Ref: c.Ref, + SHA: c.CommitSHA, + Path: c.Path, + Entries: entries, + Truncated: c.Truncated, + } +} + +func newContentEntry(e domain.TreeEntry) ContentEntry { + entry := ContentEntry{ + Name: e.Name, + Path: e.Path, + Type: string(e.Type), + Mode: e.Mode, + SHA: e.SHA, + } + if e.Size >= 0 { + size := e.Size + entry.Size = &size + } + return entry +} + type UpdateVisibilityRequest struct { Visibility string `json:"visibility"` } @@ -64,3 +111,76 @@ func (r UpdateVisibilityRequest) Validate() error { } return nil } + +type UploadBlobResponse struct { + SHA string `json:"sha"` + Size int64 `json:"size"` +} + +type CommitAuthor struct { + Name string `json:"name"` + Email string `json:"email"` +} + +type CommitOperation struct { + Op string `json:"op"` // "put" | "delete" + Path string `json:"path"` + BlobSHA string `json:"blobSha,omitempty"` // puts only, from POST /blobs + Executable bool `json:"executable,omitempty"` // puts only +} + +type CreateCommitRequest struct { + Branch string `json:"branch"` + Message string `json:"message"` + Author CommitAuthor `json:"author"` + ExpectedHeadSHA string `json:"expectedHeadSha,omitempty"` + PusherID int64 `json:"pusherId,omitempty"` + Operations []CommitOperation `json:"operations"` +} + +func (r CreateCommitRequest) Validate() error { + if r.Branch == "" { + return errors.New("branch is required") + } + if r.Message == "" { + return errors.New("message is required") + } + if r.Author.Name == "" || r.Author.Email == "" { + return errors.New("author name and email are required") + } + if len(r.Operations) == 0 { + return errors.New("operations must not be empty") + } + for i, op := range r.Operations { + if op.Path == "" { + return fmt.Errorf("operations[%d]: path is required", i) + } + switch op.Op { + case "put": + if op.BlobSHA == "" { + return fmt.Errorf("operations[%d]: blobSha is required for put", i) + } + case "delete": + if op.BlobSHA != "" || op.Executable { + return fmt.Errorf("operations[%d]: delete takes no blobSha or executable", i) + } + default: + return fmt.Errorf("operations[%d]: op must be 'put' or 'delete'", i) + } + } + return nil +} + +type Commit struct { + Branch string `json:"branch"` + CommitSHA string `json:"commitSha"` + Before string `json:"before"` // the head the commit was built on +} + +func newCommit(res domain.CommitResult) Commit { + return Commit{ + Branch: res.Branch, + CommitSHA: res.CommitSHA, + Before: res.Before, + } +} diff --git a/internal/server/git/githttp/e2e_test.go b/internal/server/git/githttp/e2e_test.go index f535af6..b44da75 100644 --- a/internal/server/git/githttp/e2e_test.go +++ b/internal/server/git/githttp/e2e_test.go @@ -45,7 +45,7 @@ func TestGitHTTPEndToEnd(t *testing.T) { t.Fatal(err) } - repoSvc := repositories.NewService(log, repositories.NewRegistry(database), backend) + repoSvc := repositories.NewService(log, repositories.NewRegistry(database), backend, nil, nil) authSvc := auth.NewService(log, auth.NewRegistry(database)) permsSvc := permissions.NewService(permissions.NewRegistry(database)) usersSvc := users.NewService(users.NewRegistry(database)) diff --git a/internal/server/git/githttp/lfs_e2e_test.go b/internal/server/git/githttp/lfs_e2e_test.go index 7b0fc27..874e9f2 100644 --- a/internal/server/git/githttp/lfs_e2e_test.go +++ b/internal/server/git/githttp/lfs_e2e_test.go @@ -63,7 +63,7 @@ func TestGitLFSEndToEnd(t *testing.T) { t.Fatal(err) } - repoSvc := repositories.NewService(log, repositories.NewRegistry(database), backend) + repoSvc := repositories.NewService(log, repositories.NewRegistry(database), backend, nil, nil) authSvc := auth.NewService(log, auth.NewRegistry(database)) permsSvc := permissions.NewService(permissions.NewRegistry(database)) usersSvc := users.NewService(users.NewRegistry(database)) diff --git a/internal/server/response/codes.go b/internal/server/response/codes.go index c907ffd..705d852 100644 --- a/internal/server/response/codes.go +++ b/internal/server/response/codes.go @@ -10,4 +10,11 @@ const ( CodeUserNotFound = "user_not_found" CodeSSHKeyNotFound = "ssh_key_not_found" CodeTokenNotFound = "token_not_found" + CodeRefNotFound = "ref_not_found" + CodePathNotFound = "path_not_found" + CodeLFSObjectNotFound = "lfs_object_not_found" + + CodeHeadMismatch = "head_mismatch" + CodeUnknownBlob = "unknown_blob" + CodeNothingToCommit = "nothing_to_commit" ) diff --git a/internal/server/server.go b/internal/server/server.go index d1d0279..8fdbe6a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,7 +3,6 @@ package server import ( "context" "fmt" - "time" "github.com/Axenos-dev/HeadlessGit/internal/config" "github.com/Axenos-dev/HeadlessGit/internal/db" @@ -31,18 +30,13 @@ type Services struct { DB *db.DB } -// clean up expired tokens every hour -const tokenGCInterval = time.Hour - -// number of workers which going to handle webhooks -const webhookWorkers = 3 - type server struct { cfg config.ServerConfig logger *zap.Logger auth *authservice.Service webhooks *webhooksservice.Service + repos *reposervice.Service control *control.Server git *git.Server @@ -58,6 +52,7 @@ func NewServer( logger: logger, auth: svc.Authentication, webhooks: svc.Webhooks, + repos: svc.Repositories, control: control.NewServer(logger.With(zap.String("component", "control")), control.Services{ Repositories: svc.Repositories, Authentication: svc.Authentication, @@ -81,11 +76,18 @@ func (s *server) Run(ctx context.Context) error { errCh := make(chan error, 3) // clean up expired tokens - go s.auth.RunExpiredTokenGC(ctx, tokenGCInterval) + if s.cfg.TokenGCInterval > 0 { + go s.auth.RunExpiredTokenGC(ctx, s.cfg.TokenGCInterval) + } + + // run git GC, to remove orphaned blobs in OBD + if s.cfg.RepoGCInterval > 0 { + go s.repos.StartGC(ctx, s.cfg.RepoGCInterval) + } // handle webhooks // it runs N goroutines for us, so dont Start in goroutine - s.webhooks.Start(ctx, webhookWorkers) + s.webhooks.Start(ctx, s.cfg.WebhookWorkers) go func() { errCh <- s.control.Run(ctx, fmt.Sprintf(":%d", s.cfg.ControlPort)) diff --git a/internal/services/lfs/service.go b/internal/services/lfs/service.go index acce6b7..8925f2f 100644 --- a/internal/services/lfs/service.go +++ b/internal/services/lfs/service.go @@ -242,6 +242,28 @@ func (s *Service) PutObject(ctx context.Context, repo domain.Repository, oid str return err } +// StoreObject registers and uploads a server-generated LFS object: used when +// api commits clean lfs-tracked files. Already verified objects are skipped +// without reading r, so callers must not rely on the reader being drained. +func (s *Service) StoreObject(ctx context.Context, repo domain.Repository, uploaderID int64, oid string, size int64, r io.Reader) error { + if err := validateOID(oid); err != nil { + return err + } + + // content addressing: an already stored and confirmed object is a no-op + if row, err := s.registry.GetLFSObject(ctx, repo.ID, oid); err == nil && row.Verified { + return nil + } + + // record the pending object; a conflicting existing row is fine + if _, err := s.registry.CreateLFSObject(ctx, uploaderID, repo.ID, oid, size); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + + // PutObject streams to storage, verifies hash and size, and flips verified + return s.PutObject(ctx, repo, oid, size, r) +} + // {repo_id}/ab/cd/ func objectKey(repoID int64, oid string) string { return fmt.Sprintf("%d/%s/%s/%s", repoID, oid[0:2], oid[2:4], oid) diff --git a/internal/services/repositories/errors.go b/internal/services/repositories/errors.go index 68f5765..c8f9fe4 100644 --- a/internal/services/repositories/errors.go +++ b/internal/services/repositories/errors.go @@ -6,4 +6,22 @@ var ( ErrRepositoryNotFound = errors.New("repository not found") ErrInvalidRepositoryName = errors.New("invalid repository name") ErrInvalidVisibility = errors.New("invalid visibility") + + ErrRefNotFound = errors.New("ref not found") + ErrPathNotFound = errors.New("path not found") + ErrInvalidRef = errors.New("invalid ref") + ErrInvalidPath = errors.New("invalid path") + + ErrUnsupportedFormat = errors.New("unsupported archive format") + ErrLFSNotEnabled = errors.New("lfs is not enabled") + + ErrNotAFile = errors.New("path is not a file") + ErrLFSObjectNotFound = errors.New("lfs object not found") + + // api commits + ErrInvalidBranch = errors.New("invalid branch name") + ErrInvalidCommitOps = errors.New("invalid commit operations") + ErrHeadMismatch = errors.New("branch head mismatch") + ErrUnknownBlob = errors.New("blob not found in repository") + ErrNothingToCommit = errors.New("nothing to commit") ) diff --git a/internal/services/repositories/registry.go b/internal/services/repositories/registry.go index 01192a2..a92448b 100644 --- a/internal/services/repositories/registry.go +++ b/internal/services/repositories/registry.go @@ -36,6 +36,10 @@ func (r *RepositoryRegistry) ListUserRepositories(ctx context.Context, ownerID i return r.db.ListUserRepositories(ctx, ownerID) } +func (r *RepositoryRegistry) ListRepositories(ctx context.Context) ([]gen.Repository, error) { + return r.db.ListRepositories(ctx) +} + func (r *RepositoryRegistry) CreateRepository(ctx context.Context, ownerID int64, name, storagePath, visibility string) (gen.Repository, error) { return r.db.CreateRepository(ctx, gen.CreateRepositoryParams{ OwnerID: ownerID, diff --git a/internal/services/repositories/service.go b/internal/services/repositories/service.go index e9194c1..4ac759c 100644 --- a/internal/services/repositories/service.go +++ b/internal/services/repositories/service.go @@ -1,15 +1,22 @@ package repositories import ( + "bytes" "context" + "crypto/sha256" "database/sql" + "encoding/hex" "errors" "fmt" + "io" + "path" "strings" "time" + "github.com/Axenos-dev/HeadlessGit/internal/archive" "github.com/Axenos-dev/HeadlessGit/internal/db/gen" "github.com/Axenos-dev/HeadlessGit/internal/domain" + "github.com/Axenos-dev/HeadlessGit/internal/gitbackend" "go.uber.org/zap" ) @@ -20,24 +27,78 @@ type Registry interface { GetRepositoryByPath(ctx context.Context, namespace, name string) (gen.Repository, error) UpdateRepositoryVisibility(ctx context.Context, repositoryID int64, visibility string) (gen.Repository, error) ListUserRepositories(ctx context.Context, ownerID int64) ([]gen.Repository, error) + ListRepositories(ctx context.Context) ([]gen.Repository, error) } type RepositoryStorage interface { InitBare(ctx context.Context, storagePath string) error Remove(ctx context.Context, storagePath string) error + ListTree(ctx context.Context, storagePath, rev, treePath string) (gitbackend.TreeListing, error) + ResolveCommit(ctx context.Context, storagePath, rev string) (string, error) + ArchiveTar(ctx context.Context, storagePath, rev string, out io.Writer) (string, error) + StatBlob(ctx context.Context, storagePath, rev, treePath string) (gitbackend.BlobInfo, error) + ReadBlob(ctx context.Context, storagePath, blobSHA string, out io.Writer) error + WriteBlob(ctx context.Context, storagePath string, r io.Reader) (string, int64, error) + ApplyCommit(ctx context.Context, storagePath string, spec gitbackend.CommitSpec, ops []gitbackend.CommitOp, clean gitbackend.CleanFunc) (gitbackend.RefChange, error) + GC(ctx context.Context, storagePath string) error +} + +type LFSObjects interface { + GetObject(ctx context.Context, repo domain.Repository, oid string) (io.ReadCloser, int64, error) + StoreObject(ctx context.Context, repo domain.Repository, uploaderID int64, oid string, size int64, r io.Reader) error +} + +// implemented by the webhooks service; nil disables push events for api commits +type PushDispatcher interface { + DispatchEvent(ctx context.Context, event domain.RepositoryEvent) error } type Service struct { logger *zap.Logger registry Registry storage RepositoryStorage + lfs LFSObjects + events PushDispatcher } -func NewService(logger *zap.Logger, registry Registry, storage RepositoryStorage) *Service { +func NewService(logger *zap.Logger, registry Registry, storage RepositoryStorage, lfs LFSObjects, events PushDispatcher) *Service { return &Service{ logger: logger, registry: registry, storage: storage, + lfs: lfs, + events: events, + } +} + +func (s *Service) StartGC(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + repos, err := s.registry.ListRepositories(ctx) + if err != nil { + s.logger.Error("gc: failed to list repositories", zap.Error(err)) + continue + } + + for _, repo := range repos { + if err := s.storage.GC(ctx, repo.StoragePath); err != nil { + s.logger.Warn("gc failed", zap.Int64("repository_id", repo.ID), zap.Error(err)) + } + + // pace the sweep so gc never hammers the whole disk at once + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } } } @@ -139,6 +200,35 @@ func (s *Service) ListByOwner(ctx context.Context, ownerID int64) ([]domain.Repo return out, nil } +func (s *Service) Contents(ctx context.Context, repositoryID int64, ref, treePath string) (domain.RepositoryContents, error) { + repo, err := s.registry.GetRepository(ctx, repositoryID) + if errors.Is(err, sql.ErrNoRows) { + return domain.RepositoryContents{}, ErrRepositoryNotFound + } + if err != nil { + return domain.RepositoryContents{}, err + } + + listing, err := s.storage.ListTree(ctx, repo.StoragePath, ref, treePath) + switch { + case errors.Is(err, gitbackend.ErrInvalidRev): + return domain.RepositoryContents{}, ErrInvalidRef + case errors.Is(err, gitbackend.ErrInvalidPath): + return domain.RepositoryContents{}, ErrInvalidPath + case errors.Is(err, gitbackend.ErrRevNotFound): + return domain.RepositoryContents{}, ErrRefNotFound + case errors.Is(err, gitbackend.ErrPathNotFound): + return domain.RepositoryContents{}, ErrPathNotFound + case err != nil: + return domain.RepositoryContents{}, err + } + + if ref == "" { + ref = "HEAD" + } + return toContents(ref, treePath, listing), nil +} + func (s *Service) GetRepositoryByPath(ctx context.Context, namespace, name string) (domain.Repository, error) { repo, err := s.registry.GetRepositoryByPath(ctx, namespace, name) if errors.Is(err, sql.ErrNoRows) { @@ -150,6 +240,296 @@ func (s *Service) GetRepositoryByPath(ctx context.Context, namespace, name strin return toDomain(repo), nil } +func (s *Service) PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool) (domain.ArchiveRequest, error) { + f, ok := domain.ParseArchiveFormat(format) + if !ok { + return domain.ArchiveRequest{}, ErrUnsupportedFormat + } + if includeLFS && s.lfs == nil { + return domain.ArchiveRequest{}, ErrLFSNotEnabled + } + + repo, err := s.Get(ctx, repositoryID) + if err != nil { + return domain.ArchiveRequest{}, err + } + + sha, err := s.storage.ResolveCommit(ctx, repo.StoragePath, ref) + switch { + case errors.Is(err, gitbackend.ErrInvalidRev): + return domain.ArchiveRequest{}, ErrInvalidRef + case errors.Is(err, gitbackend.ErrRevNotFound): + return domain.ArchiveRequest{}, ErrRefNotFound + case err != nil: + return domain.ArchiveRequest{}, err + } + + return domain.ArchiveRequest{ + Repository: repo, + CommitSHA: sha, + Format: f, + IncludeLFS: includeLFS, + }, nil +} + +func (s *Service) StreamArchive(ctx context.Context, req domain.ArchiveRequest, out io.Writer) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // create pipe and stream git archive to writer + pr, pw := io.Pipe() + go func() { + _, err := s.storage.ArchiveTar(ctx, req.Repository.StoragePath, req.CommitSHA, pw) + pw.CloseWithError(err) + }() + + // create archive endocer depending on the archive type, connect it to output writer + var enc archive.Encoder + if req.Format == domain.ArchiveFormatTarGz { + enc = archive.NewTarGzEncoder(out) + } else { + enc = archive.NewZipEncoder(out) + } + + // define a spefic smudge function for LFS objects + // to translate LFS pointers into a real blobs + var smudge archive.SmudgeFunc + if req.IncludeLFS { + smudge = func(oid string) (io.ReadCloser, int64, error) { + rc, size, err := s.lfs.GetObject(ctx, req.Repository, oid) + if err != nil { + s.logger.Warn("lfs object unavailable for archive, keeping pointer", + zap.Int64("repository_id", req.Repository.ID), + zap.String("oid", oid), + zap.Error(err), + ) + } + return rc, size, err + } + } + + prefix := fmt.Sprintf("%s-%s/", req.Repository.RepositoryName, domain.ShortSHA(req.CommitSHA)) + return archive.Transform(pr, prefix, smudge, enc) +} + +func (s *Service) PrepareBlob(ctx context.Context, repositoryID int64, ref, treePath string, includeLFS bool) (domain.BlobRequest, error) { + if includeLFS && s.lfs == nil { + return domain.BlobRequest{}, ErrLFSNotEnabled + } + + repo, err := s.Get(ctx, repositoryID) + if err != nil { + return domain.BlobRequest{}, err + } + + info, err := s.storage.StatBlob(ctx, repo.StoragePath, ref, treePath) + switch { + case errors.Is(err, gitbackend.ErrInvalidRev): + return domain.BlobRequest{}, ErrInvalidRef + case errors.Is(err, gitbackend.ErrInvalidPath): + return domain.BlobRequest{}, ErrInvalidPath + case errors.Is(err, gitbackend.ErrRevNotFound): + return domain.BlobRequest{}, ErrRefNotFound + case errors.Is(err, gitbackend.ErrPathNotFound): + return domain.BlobRequest{}, ErrPathNotFound + case errors.Is(err, gitbackend.ErrNotABlob): + return domain.BlobRequest{}, ErrNotAFile + case err != nil: + return domain.BlobRequest{}, err + } + + req := domain.BlobRequest{ + Repository: repo, + CommitSHA: info.CommitSHA, + BlobSHA: info.BlobSHA, + Path: treePath, + Size: info.Size, + } + + // check pointer-sized blobs + if includeLFS && info.Size <= domain.LFSPointerMaxSize { + // read it + var buf bytes.Buffer + if err := s.storage.ReadBlob(ctx, repo.StoragePath, info.BlobSHA, &buf); err != nil { + return domain.BlobRequest{}, err + } + // then parse to see if its a pointer + if ptr, ok := domain.ParseLFSPointer(buf.Bytes()); ok { + // but the oid is repo content and untrusted + // so to be safe, we would pull it from dedicated lfs service with respect to repoID + rc, size, err := s.lfs.GetObject(ctx, repo, ptr.OID) + if err != nil { + return domain.BlobRequest{}, ErrLFSObjectNotFound + } + rc.Close() + + req.LFSOID = ptr.OID + req.Size = size + } + } + + return req, nil +} + +func (s *Service) StreamBlob(ctx context.Context, req domain.BlobRequest, out io.Writer) error { + if req.LFSOID != "" { + rc, _, err := s.lfs.GetObject(ctx, req.Repository, req.LFSOID) + if err != nil { + return err + } + defer rc.Close() + + _, err = io.Copy(out, rc) + return err + } + return s.storage.ReadBlob(ctx, req.Repository.StoragePath, req.BlobSHA, out) +} + +func (s *Service) WriteBlob(ctx context.Context, repositoryID int64, in io.Reader) (string, int64, error) { + repo, err := s.Get(ctx, repositoryID) + if err != nil { + return "", 0, err + } + + return s.storage.WriteBlob(ctx, repo.StoragePath, in) +} + +func (s *Service) Commit(ctx context.Context, repositoryID int64, req domain.CommitRequest) (domain.CommitResult, error) { + repo, err := s.Get(ctx, repositoryID) + if err != nil { + return domain.CommitResult{}, err + } + + ops := make([]gitbackend.CommitOp, len(req.Operations)) + for i, op := range req.Operations { + mode := "" + if op.Executable { + mode = "100755" + } + ops[i] = gitbackend.CommitOp{ + Delete: op.Delete, + Path: op.Path, + BlobSHA: op.BlobSHA, + Mode: mode, + } + } + + spec := gitbackend.CommitSpec{ + Branch: req.Branch, + ExpectedOld: req.ExpectedHeadSHA, + Author: gitbackend.Identity{ + Name: req.Author.Name, + Email: req.Author.Email, + }, + Message: req.Message, + } + + // nil when LFS is disabled + // lfs-tracked paths then fail with ErrLFSRequired + var clean gitbackend.CleanFunc + if s.lfs != nil { + clean = s.lfsCleanFunc(ctx, repo, req.PusherID) + } + + change, err := s.storage.ApplyCommit(ctx, repo.StoragePath, spec, ops, clean) + switch { + case errors.Is(err, gitbackend.ErrInvalidBranch): + return domain.CommitResult{}, ErrInvalidBranch + case errors.Is(err, gitbackend.ErrInvalidOps), errors.Is(err, gitbackend.ErrInvalidPath), errors.Is(err, gitbackend.ErrInvalidRev): + return domain.CommitResult{}, fmt.Errorf("%w: %s", ErrInvalidCommitOps, err) + case errors.Is(err, gitbackend.ErrRevNotFound): + return domain.CommitResult{}, ErrRefNotFound + case errors.Is(err, gitbackend.ErrPathNotFound): + return domain.CommitResult{}, ErrPathNotFound + case errors.Is(err, gitbackend.ErrNotABlob): + return domain.CommitResult{}, ErrNotAFile + case errors.Is(err, gitbackend.ErrHeadMismatch): + return domain.CommitResult{}, ErrHeadMismatch + case errors.Is(err, gitbackend.ErrUnknownBlob): + return domain.CommitResult{}, ErrUnknownBlob + case errors.Is(err, gitbackend.ErrNothingToCommit): + return domain.CommitResult{}, ErrNothingToCommit + case errors.Is(err, gitbackend.ErrLFSRequired): + return domain.CommitResult{}, ErrLFSNotEnabled + case err != nil: + return domain.CommitResult{}, err + } + + s.dispatchPush(ctx, repo, req, change) + + return domain.CommitResult{ + Branch: req.Branch, + CommitSHA: change.NewSHA, + Before: change.OldSHA, + }, nil +} + +func (s *Service) lfsCleanFunc(ctx context.Context, repo domain.Repository, pusherID int64) gitbackend.CleanFunc { + uploaderID := pusherID + if uploaderID == 0 { + uploaderID = repo.OwnerID + } + + return func(path, blobSHA string, size int64) (string, error) { + // check if blob is already LFS pointer + if size <= domain.LFSPointerMaxSize { + var buf bytes.Buffer + if err := s.storage.ReadBlob(ctx, repo.StoragePath, blobSHA, &buf); err != nil { + return "", err + } + if _, ok := domain.ParseLFSPointer(buf.Bytes()); ok { + return blobSHA, nil + } + } + + // hash the local blob to get the lfs oid + hasher := sha256.New() + if err := s.storage.ReadBlob(ctx, repo.StoragePath, blobSHA, hasher); err != nil { + return "", err + } + oid := hex.EncodeToString(hasher.Sum(nil)) + + // stream the same blob into lfs storage + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(s.storage.ReadBlob(ctx, repo.StoragePath, blobSHA, pw)) + }() + err := s.lfs.StoreObject(ctx, repo, uploaderID, oid, size, pr) + pr.Close() // unblocks the writer when StoreObject returned without draining + if err != nil { + return "", err + } + + // construct pointer by "hands" + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, size) + pointerSHA, _, err := s.storage.WriteBlob(ctx, repo.StoragePath, strings.NewReader(pointer)) + return pointerSHA, err + } +} + +// dispatch webhook event +func (s *Service) dispatchPush(ctx context.Context, repo domain.Repository, req domain.CommitRequest, change gitbackend.RefChange) { + if s.events == nil { + return + } + + err := s.events.DispatchEvent(ctx, domain.RepositoryEvent{ + Event: "push", + RepositoryID: repo.ID, + RepositoryName: repo.RepositoryName, + RepositoryFullName: fmt.Sprintf("%d/%s", repo.OwnerID, repo.RepositoryName), + PusherID: req.PusherID, + PusherUsername: req.Author.Name, + Ref: change.Ref, + OldSHA: change.OldSHA, + NewSHA: change.NewSHA, + Timestamp: time.Now().UTC(), + }) + if err != nil { + s.logger.Warn("failed to enqueue webhook event", zap.String("ref", change.Ref), zap.Error(err)) + } +} + func toDomain(r gen.Repository) domain.Repository { repo := domain.Repository{ ID: r.ID, @@ -172,3 +552,24 @@ func validRepositoryName(name string) bool { } return !strings.ContainsAny(name, "/\\") } + +func toContents(ref, treePath string, listing gitbackend.TreeListing) domain.RepositoryContents { + entries := make([]domain.TreeEntry, len(listing.Entries)) + for i, e := range listing.Entries { + entries[i] = domain.TreeEntry{ + Name: path.Base(e.Path), + Path: e.Path, + Type: domain.TreeEntryTypeFromMode(e.Mode), + Mode: e.Mode, + SHA: e.SHA, + Size: e.Size, + } + } + return domain.RepositoryContents{ + Ref: ref, + CommitSHA: listing.CommitSHA, + Path: treePath, + Entries: entries, + Truncated: listing.Truncated, + } +} diff --git a/internal/services/repositories/service_test.go b/internal/services/repositories/service_test.go new file mode 100644 index 0000000..ac9239f --- /dev/null +++ b/internal/services/repositories/service_test.go @@ -0,0 +1,525 @@ +package repositories + +import ( + "archive/tar" + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "io" + "strings" + "testing" + + "github.com/Axenos-dev/HeadlessGit/internal/db/gen" + "github.com/Axenos-dev/HeadlessGit/internal/domain" + "github.com/Axenos-dev/HeadlessGit/internal/gitbackend" + "go.uber.org/zap" +) + +const testSHA = "aaaabbbbccccddddeeeeffff0000111122223333" + +type fakeRegistry struct { + Registry + repo gen.Repository + err error +} + +func (f fakeRegistry) GetRepository(ctx context.Context, repositoryID int64) (gen.Repository, error) { + return f.repo, f.err +} + +type fakeStorage struct { + RepositoryStorage + sha string + resolveErr error + tarBytes []byte + + blobInfo gitbackend.BlobInfo + blobStatErr error + blobContent string + + writeBlobSHA string + applyChange gitbackend.RefChange + applyErr error + // optional hook to inspect (and exercise) what ApplyCommit received + applyFn func(spec gitbackend.CommitSpec, ops []gitbackend.CommitOp, clean gitbackend.CleanFunc) error +} + +func (f fakeStorage) ResolveCommit(ctx context.Context, storagePath, rev string) (string, error) { + if f.resolveErr != nil { + return "", f.resolveErr + } + return f.sha, nil +} + +func (f fakeStorage) ArchiveTar(ctx context.Context, storagePath, rev string, out io.Writer) (string, error) { + if _, err := out.Write(f.tarBytes); err != nil { + return "", err + } + return f.sha, nil +} + +func (f fakeStorage) StatBlob(ctx context.Context, storagePath, rev, treePath string) (gitbackend.BlobInfo, error) { + if f.blobStatErr != nil { + return gitbackend.BlobInfo{}, f.blobStatErr + } + return f.blobInfo, nil +} + +func (f fakeStorage) ReadBlob(ctx context.Context, storagePath, blobSHA string, out io.Writer) error { + _, err := io.WriteString(out, f.blobContent) + return err +} + +func (f fakeStorage) WriteBlob(ctx context.Context, storagePath string, r io.Reader) (string, int64, error) { + n, err := io.Copy(io.Discard, r) + if err != nil { + return "", 0, err + } + return f.writeBlobSHA, n, nil +} + +func (f fakeStorage) ApplyCommit(ctx context.Context, storagePath string, spec gitbackend.CommitSpec, ops []gitbackend.CommitOp, clean gitbackend.CleanFunc) (gitbackend.RefChange, error) { + if f.applyFn != nil { + if err := f.applyFn(spec, ops, clean); err != nil { + return gitbackend.RefChange{}, err + } + } + if f.applyErr != nil { + return gitbackend.RefChange{}, f.applyErr + } + return f.applyChange, nil +} + +type fakeLFS struct { + objects map[string]string // oid -> content + stored map[string]string // oid -> content received via StoreObject +} + +func (f fakeLFS) GetObject(ctx context.Context, repo domain.Repository, oid string) (io.ReadCloser, int64, error) { + content, ok := f.objects[oid] + if !ok { + return nil, 0, errors.New("object not found") + } + return io.NopCloser(strings.NewReader(content)), int64(len(content)), nil +} + +func (f fakeLFS) StoreObject(ctx context.Context, repo domain.Repository, uploaderID int64, oid string, size int64, r io.Reader) error { + body, err := io.ReadAll(r) + if err != nil { + return err + } + if f.stored != nil { + f.stored[oid] = string(body) + } + return nil +} + +type fakeDispatcher struct { + events *[]domain.RepositoryEvent +} + +func (f fakeDispatcher) DispatchEvent(ctx context.Context, event domain.RepositoryEvent) error { + *f.events = append(*f.events, event) + return nil +} + +func TestPrepareArchive(t *testing.T) { + row := gen.Repository{ID: 7, RepositoryName: "myrepo", StoragePath: "7/myrepo.git", Visibility: "private"} + + cases := []struct { + name string + registry Registry + storage RepositoryStorage + lfs LFSObjects + format string + includeLFS bool + wantErr error + }{ + {"unsupported format", fakeRegistry{repo: row}, fakeStorage{sha: testSHA}, nil, "rar", false, ErrUnsupportedFormat}, + {"lfs disabled", fakeRegistry{repo: row}, fakeStorage{sha: testSHA}, nil, "zip", true, ErrLFSNotEnabled}, + {"lfs enabled ok", fakeRegistry{repo: row}, fakeStorage{sha: testSHA}, fakeLFS{}, "zip", true, nil}, + {"repo not found", fakeRegistry{err: sql.ErrNoRows}, fakeStorage{sha: testSHA}, nil, "zip", false, ErrRepositoryNotFound}, + {"invalid ref", fakeRegistry{repo: row}, fakeStorage{resolveErr: gitbackend.ErrInvalidRev}, nil, "zip", false, ErrInvalidRef}, + {"ref not found", fakeRegistry{repo: row}, fakeStorage{resolveErr: gitbackend.ErrRevNotFound}, nil, "zip", false, ErrRefNotFound}, + {"ok", fakeRegistry{repo: row}, fakeStorage{sha: testSHA}, nil, "zip", false, nil}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc := NewService(zap.NewNop(), tc.registry, tc.storage, tc.lfs, nil) + req, err := svc.PrepareArchive(context.Background(), row.ID, "main", tc.format, tc.includeLFS) + if !errors.Is(err, tc.wantErr) { + t.Fatalf("PrepareArchive error = %v, want %v", err, tc.wantErr) + } + if tc.wantErr == nil { + if req.CommitSHA != testSHA || req.Repository.ID != row.ID || req.Format != domain.ArchiveFormatZip { + t.Errorf("PrepareArchive = %+v", req) + } + if want := "myrepo-aaaabbbbcccc.zip"; req.Filename() != want { + t.Errorf("Filename = %q, want %q", req.Filename(), want) + } + } + }) + } +} + +func TestStreamArchiveSmudgesLFS(t *testing.T) { + oid := strings.Repeat("ab", 32) + content := "REAL LFS CONTENT" + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(content)) + + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + for _, e := range []struct{ name, body string }{ + {"README.md", "hello\n"}, + {"big.bin", pointer}, + } { + if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeReg, Name: e.name, Mode: 0o644, Size: int64(len(e.body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + svc := NewService( + zap.NewNop(), + fakeRegistry{}, + fakeStorage{sha: testSHA, tarBytes: tarBuf.Bytes()}, + fakeLFS{objects: map[string]string{oid: content}}, + nil, + ) + + req := domain.ArchiveRequest{ + Repository: domain.Repository{ID: 7, RepositoryName: "myrepo", StoragePath: "7/myrepo.git"}, + CommitSHA: testSHA, + Format: domain.ArchiveFormatZip, + IncludeLFS: true, + } + + var out bytes.Buffer + if err := svc.StreamArchive(context.Background(), req, &out); err != nil { + t.Fatal(err) + } + + zr, err := zip.NewReader(bytes.NewReader(out.Bytes()), int64(out.Len())) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatal(err) + } + got[f.Name] = string(body) + } + + prefix := "myrepo-aaaabbbbcccc/" + if got[prefix+"big.bin"] != content { + t.Errorf("big.bin not smudged: %q", got[prefix+"big.bin"]) + } + if got[prefix+"README.md"] != "hello\n" { + t.Errorf("README.md = %q", got[prefix+"README.md"]) + } +} + +const blobSHA = "1111222233334444555566667777888899990000" + +func blobStorage(content string) fakeStorage { + return fakeStorage{ + blobInfo: gitbackend.BlobInfo{CommitSHA: testSHA, BlobSHA: blobSHA, Size: int64(len(content))}, + blobContent: content, + } +} + +func TestPrepareBlob(t *testing.T) { + row := gen.Repository{ID: 7, RepositoryName: "myrepo", StoragePath: "7/myrepo.git", Visibility: "private"} + oid := strings.Repeat("cd", 32) + content := "REAL LFS CONTENT" + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(content)) + + t.Run("raw file", func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, blobStorage("hello\n"), nil, nil) + req, err := svc.PrepareBlob(context.Background(), row.ID, "main", "README.md", false) + if err != nil { + t.Fatal(err) + } + if req.BlobSHA != blobSHA || req.CommitSHA != testSHA || req.Size != 6 || req.LFSOID != "" { + t.Errorf("PrepareBlob = %+v", req) + } + }) + + t.Run("pointer smudged", func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, blobStorage(pointer), fakeLFS{objects: map[string]string{oid: content}}, nil) + req, err := svc.PrepareBlob(context.Background(), row.ID, "main", "big.bin", true) + if err != nil { + t.Fatal(err) + } + if req.LFSOID != oid { + t.Errorf("LFSOID = %q", req.LFSOID) + } + if req.Size != int64(len(content)) { + t.Errorf("Size = %d, want object size %d", req.Size, len(content)) + } + }) + + t.Run("pointer without lfs flag stays raw", func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, blobStorage(pointer), fakeLFS{objects: map[string]string{oid: content}}, nil) + req, err := svc.PrepareBlob(context.Background(), row.ID, "main", "big.bin", false) + if err != nil { + t.Fatal(err) + } + if req.LFSOID != "" || req.Size != int64(len(pointer)) { + t.Errorf("PrepareBlob = %+v", req) + } + }) + + t.Run("large blob is never sniffed", func(t *testing.T) { + st := blobStorage(pointer) + st.blobInfo.Size = 5000 // over the pointer cap, content must not be read + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, st, fakeLFS{}, nil) + req, err := svc.PrepareBlob(context.Background(), row.ID, "main", "big.bin", true) + if err != nil { + t.Fatal(err) + } + if req.LFSOID != "" || req.Size != 5000 { + t.Errorf("PrepareBlob = %+v", req) + } + }) + + t.Run("missing lfs object fails loudly", func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, blobStorage(pointer), fakeLFS{}, nil) + if _, err := svc.PrepareBlob(context.Background(), row.ID, "main", "big.bin", true); !errors.Is(err, ErrLFSObjectNotFound) { + t.Errorf("want ErrLFSObjectNotFound, got %v", err) + } + }) + + t.Run("errors", func(t *testing.T) { + cases := []struct { + name string + storage RepositoryStorage + lfs LFSObjects + includeLFS bool + wantErr error + }{ + {"lfs disabled", blobStorage(""), nil, true, ErrLFSNotEnabled}, + {"not a file", fakeStorage{blobStatErr: gitbackend.ErrNotABlob}, nil, false, ErrNotAFile}, + {"path not found", fakeStorage{blobStatErr: gitbackend.ErrPathNotFound}, nil, false, ErrPathNotFound}, + {"ref not found", fakeStorage{blobStatErr: gitbackend.ErrRevNotFound}, nil, false, ErrRefNotFound}, + {"invalid ref", fakeStorage{blobStatErr: gitbackend.ErrInvalidRev}, nil, false, ErrInvalidRef}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, tc.storage, tc.lfs, nil) + if _, err := svc.PrepareBlob(context.Background(), row.ID, "main", "x", tc.includeLFS); !errors.Is(err, tc.wantErr) { + t.Errorf("PrepareBlob error = %v, want %v", err, tc.wantErr) + } + }) + } + }) +} + +func TestStreamBlob(t *testing.T) { + oid := strings.Repeat("cd", 32) + content := "REAL LFS CONTENT" + repo := domain.Repository{ID: 7, RepositoryName: "myrepo", StoragePath: "7/myrepo.git"} + + t.Run("raw", func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{}, blobStorage("hello\n"), nil, nil) + var out bytes.Buffer + if err := svc.StreamBlob(context.Background(), domain.BlobRequest{Repository: repo, BlobSHA: blobSHA}, &out); err != nil { + t.Fatal(err) + } + if out.String() != "hello\n" { + t.Errorf("content = %q", out.String()) + } + }) + + t.Run("smudged", func(t *testing.T) { + svc := NewService(zap.NewNop(), fakeRegistry{}, blobStorage(""), fakeLFS{objects: map[string]string{oid: content}}, nil) + var out bytes.Buffer + if err := svc.StreamBlob(context.Background(), domain.BlobRequest{Repository: repo, BlobSHA: blobSHA, LFSOID: oid}, &out); err != nil { + t.Fatal(err) + } + if out.String() != content { + t.Errorf("content = %q", out.String()) + } + }) +} + +func TestCommit(t *testing.T) { + row := gen.Repository{ID: 7, OwnerID: 3, RepositoryName: "myrepo", StoragePath: "7/myrepo.git", Visibility: "private"} + change := gitbackend.RefChange{Ref: "refs/heads/main", OldSHA: strings.Repeat("a", 40), NewSHA: testSHA} + req := domain.CommitRequest{ + Branch: "main", + Message: "update", + Author: domain.CommitIdentity{Name: "api-user", Email: "api@test"}, + ExpectedHeadSHA: strings.Repeat("a", 40), + PusherID: 42, + Operations: []domain.CommitFileOp{ + {Path: "run.sh", BlobSHA: blobSHA, Executable: true}, + {Path: "old.txt", Delete: true}, + }, + } + + t.Run("maps ops and dispatches the push event", func(t *testing.T) { + var events []domain.RepositoryEvent + var gotSpec gitbackend.CommitSpec + var gotOps []gitbackend.CommitOp + var gotClean gitbackend.CleanFunc + st := fakeStorage{applyChange: change, applyFn: func(spec gitbackend.CommitSpec, ops []gitbackend.CommitOp, clean gitbackend.CleanFunc) error { + gotSpec, gotOps, gotClean = spec, ops, clean + return nil + }} + + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, st, fakeLFS{}, fakeDispatcher{events: &events}) + res, err := svc.Commit(context.Background(), row.ID, req) + if err != nil { + t.Fatal(err) + } + + if res.CommitSHA != testSHA || res.Before != change.OldSHA || res.Branch != "main" { + t.Errorf("result = %+v", res) + } + if gotSpec.Branch != "main" || gotSpec.ExpectedOld != req.ExpectedHeadSHA || gotSpec.Author.Name != "api-user" { + t.Errorf("spec = %+v", gotSpec) + } + if len(gotOps) != 2 || gotOps[0].Mode != "100755" || !gotOps[1].Delete { + t.Errorf("ops = %+v", gotOps) + } + if gotClean == nil { + t.Error("clean must be set when lfs is enabled") + } + + if len(events) != 1 { + t.Fatalf("events = %+v", events) + } + e := events[0] + if e.Event != "push" || e.RepositoryFullName != "3/myrepo" || e.PusherID != 42 || + e.PusherUsername != "api-user" || e.Ref != change.Ref || e.OldSHA != change.OldSHA || e.NewSHA != change.NewSHA { + t.Errorf("event = %+v", e) + } + }) + + t.Run("nil clean when lfs disabled", func(t *testing.T) { + st := fakeStorage{applyChange: change, applyFn: func(_ gitbackend.CommitSpec, _ []gitbackend.CommitOp, clean gitbackend.CleanFunc) error { + if clean != nil { + t.Error("clean must be nil when lfs is disabled") + } + return nil + }} + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, st, nil, nil) + if _, err := svc.Commit(context.Background(), row.ID, req); err != nil { + t.Fatal(err) + } + }) + + t.Run("error mapping and no event on failure", func(t *testing.T) { + cases := []struct { + backend error + want error + }{ + {gitbackend.ErrInvalidBranch, ErrInvalidBranch}, + {gitbackend.ErrInvalidOps, ErrInvalidCommitOps}, + {gitbackend.ErrRevNotFound, ErrRefNotFound}, + {gitbackend.ErrPathNotFound, ErrPathNotFound}, + {gitbackend.ErrNotABlob, ErrNotAFile}, + {gitbackend.ErrHeadMismatch, ErrHeadMismatch}, + {gitbackend.ErrUnknownBlob, ErrUnknownBlob}, + {gitbackend.ErrNothingToCommit, ErrNothingToCommit}, + {gitbackend.ErrLFSRequired, ErrLFSNotEnabled}, + } + for _, tc := range cases { + var events []domain.RepositoryEvent + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, fakeStorage{applyErr: tc.backend}, nil, fakeDispatcher{events: &events}) + if _, err := svc.Commit(context.Background(), row.ID, req); !errors.Is(err, tc.want) { + t.Errorf("backend %v: got %v, want %v", tc.backend, err, tc.want) + } + if len(events) != 0 { + t.Errorf("backend %v: event dispatched on failure", tc.backend) + } + } + }) +} + +func TestCommitCleanClosure(t *testing.T) { + row := gen.Repository{ID: 7, OwnerID: 3, RepositoryName: "myrepo", StoragePath: "7/myrepo.git", Visibility: "private"} + pointerBlob := strings.Repeat("f", 40) + req := domain.CommitRequest{ + Branch: "main", + Message: "x", + Author: domain.CommitIdentity{Name: "t", Email: "t@t"}, + Operations: []domain.CommitFileOp{ + {Path: "big.bin", BlobSHA: blobSHA}, + }, + } + + t.Run("converts payload to lfs object and pointer", func(t *testing.T) { + payload := "REAL BINARY PAYLOAD" + wantOID := fmt.Sprintf("%x", sha256.Sum256([]byte(payload))) + + stored := map[string]string{} + st := fakeStorage{ + blobContent: payload, + writeBlobSHA: pointerBlob, + applyFn: func(_ gitbackend.CommitSpec, _ []gitbackend.CommitOp, clean gitbackend.CleanFunc) error { + got, err := clean("big.bin", blobSHA, int64(len(payload))) + if err != nil { + return err + } + if got != pointerBlob { + t.Errorf("clean returned %q, want pointer blob %q", got, pointerBlob) + } + return nil + }, + } + + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, st, fakeLFS{stored: stored}, nil) + if _, err := svc.Commit(context.Background(), row.ID, req); err != nil { + t.Fatal(err) + } + if stored[wantOID] != payload { + t.Errorf("stored objects = %v, want oid %s with payload", stored, wantOID) + } + }) + + t.Run("existing pointer passes through untouched", func(t *testing.T) { + pointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + strings.Repeat("ab", 32) + "\nsize 44\n" + + stored := map[string]string{} + st := fakeStorage{ + blobContent: pointer, + applyFn: func(_ gitbackend.CommitSpec, _ []gitbackend.CommitOp, clean gitbackend.CleanFunc) error { + got, err := clean("big.bin", blobSHA, int64(len(pointer))) + if err != nil { + return err + } + if got != blobSHA { + t.Errorf("clean returned %q, want passthrough %q", got, blobSHA) + } + return nil + }, + } + + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, st, fakeLFS{stored: stored}, nil) + if _, err := svc.Commit(context.Background(), row.ID, req); err != nil { + t.Fatal(err) + } + if len(stored) != 0 { + t.Errorf("pointer passthrough must not store objects, got %v", stored) + } + }) +}