diff --git a/README.md b/README.md index 19d1d00..d814642 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ Every request requires `Authorization: Bearer `. Responses are enve | `GET` | `/repositories/{id}/blob?ref=&path=&lfs=` | — | Stream one file's raw content. | | `GET` | `/repositories/{id}/archive?ref=&format=&lfs=&prefix=` | — | Stream a `zip` (default) or `tar.gz` archive of the tree. | | `POST` | `/repositories/{id}/blobs` | _raw bytes_ | Upload content into the repo's object database; returns `{sha, size}`. | -| `POST` | `/repositories/{id}/commits` | JSON | Create a commit on a branch from uploaded blobs. | +| `POST` | `/repositories/{id}/commits` | JSON | Create a commit from Git blobs or verified LFS objects. | ### Reading a repository @@ -290,6 +290,27 @@ curl -H "Authorization: Bearer $TOKEN" -X POST \ # -> 201 {"data": {"branch": "main", "commitSha": "...", "before": "9fb03799..."}} ``` +A `put` operation takes exactly one content source: + +```json +{ "op": "put", "path": "README.md", "blobSha": "" } +``` + +or a repository-scoped LFS object already uploaded and verified through the LFS Batch API: + +```json +{ + "op": "put", + "path": "models/model.bin", + "lfs": { + "oid": "...", + "size": 734003200 + } +} +``` + +`blobSha` and `lfs` are mutually exclusive. `executable` is optional for puts. A `delete` operation takes only `op` and `path`. + `expectedHeadSha` controls concurrency: | Value | Meaning | @@ -300,8 +321,6 @@ curl -H "Authorization: Bearer $TOKEN" -X POST \ Content is deduplicated by sha, so retrying an upload is free and a lost `409` race can be retried without re-uploading anything. Blobs that never get committed are garbage-collected after a grace period (see `REPO_GC_INTERVAL`). -**LFS is automatic**, the same way it is for a git client: if the repo's `.gitattributes` marks a path as `filter=lfs` (including attributes added in the very same commit), the server stores the content as an LFS object and commits a pointer instead. Content that already _is_ a valid pointer passes through untouched, so pre-uploading big files via the LFS API (presigned, straight to the bucket) and committing the pointer yourself remains the efficient path for large objects. - API commits dispatch the same signed [webhooks](#webhooks) as a `git push` — consumers can't tell them apart. ### Health diff --git a/internal/domain/commit.go b/internal/domain/commit.go index 6498135..c1c4666 100644 --- a/internal/domain/commit.go +++ b/internal/domain/commit.go @@ -16,10 +16,16 @@ type CommitDetails struct { CommittedAt time.Time } +type CommitFileLfsObject struct { + OID string + Size int64 +} + type CommitFileOp struct { Delete bool Path string - BlobSHA string + BlobSHA *string + Lfs *CommitFileLfsObject Executable bool } diff --git a/internal/gitbackend/errors.go b/internal/gitbackend/errors.go index 43351e8..757f055 100644 --- a/internal/gitbackend/errors.go +++ b/internal/gitbackend/errors.go @@ -16,4 +16,5 @@ var ( ErrHeadMismatch = errors.New("branch head mismatch") ErrNothingToCommit = errors.New("nothing to commit") ErrLFSRequired = errors.New("path is lfs-tracked but no clean filter is available") + ErrLFSNotTracked = errors.New("lfs object path is not lfs-tracked") ) diff --git a/internal/gitbackend/local.go b/internal/gitbackend/local.go index 0825f91..c2feabe 100644 --- a/internal/gitbackend/local.go +++ b/internal/gitbackend/local.go @@ -512,9 +512,12 @@ func (l *Local) ApplyCommit(ctx context.Context, storagePath string, spec Commit oldSHA = zeroSHA } - // one batch-check verifies every referenced blob (and captures sizes for - // the clean filter) plus the existence of every delete target - sizes, err := l.verifyCommitInputs(ctx, dir, oldSHA, unborn, ops) + ops, err = l.materializeLFSPointers(ctx, dir, ops) + if err != nil { + return RefChange{}, err + } + + ops, sizes, err := l.verifyCommitInputs(ctx, dir, oldSHA, unborn, ops) if err != nil { return RefChange{}, err } @@ -644,16 +647,37 @@ func validateCommitOps(ops []CommitOp) ([]CommitOp, error) { return nil, fmt.Errorf("%w: control character in path", ErrInvalidOps) } } - if seen[p] { - return nil, fmt.Errorf("%w: duplicate path %q", ErrInvalidOps, p) + for other := range seen { + if p == other { + return nil, fmt.Errorf("%w: duplicate path %q", ErrInvalidOps, p) + } + if strings.HasPrefix(p, other+"/") || strings.HasPrefix(other, p+"/") { + return nil, fmt.Errorf("%w: overlapping paths %q and %q", ErrInvalidOps, other, p) + } } seen[p] = true op.Path = p - if !op.Delete { - if !isHexSHA(op.BlobSHA) { + if op.Delete { + if op.BlobSHA != "" || op.Lfs != nil { + return nil, fmt.Errorf("%w: delete %q takes no object", ErrInvalidOps, p) + } + } else { + hasBlob := op.BlobSHA != "" + hasLFS := op.Lfs != nil + + if hasBlob == hasLFS { + return nil, fmt.Errorf("%w: put %q requires exactly one of blob sha or lfs object", ErrInvalidOps, p) + } + if hasBlob && !isHexSHA(op.BlobSHA) { return nil, fmt.Errorf("%w: blob sha %q", ErrInvalidOps, op.BlobSHA) } + if hasLFS && (!isLFSOID(op.Lfs.OID) || op.Lfs.Size < 0) { + return nil, fmt.Errorf("%w: invalid lfs object for %q", ErrInvalidOps, p) + } + if hasLFS && isAttributesPath(p) { + return nil, fmt.Errorf("%w: attributes file %q cannot be an lfs object", ErrInvalidOps, p) + } switch op.Mode { case "": op.Mode = "100644" @@ -667,10 +691,35 @@ func validateCommitOps(ops []CommitOp) ([]CommitOp, error) { return out, nil } +// transforms lfs objects to its pointers +func (l *Local) materializeLFSPointers(ctx context.Context, dir string, ops []CommitOp) ([]CommitOp, error) { + for i := range ops { + if ops[i].Lfs == nil { + continue + } + + pointer := fmt.Sprintf( + "version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", + ops[i].Lfs.OID, + ops[i].Lfs.Size, + ) + // generate sha for handcrafted pointer + sha, err := l.runGit(ctx, dir, nil, strings.NewReader(pointer), "hash-object", "-w", "--stdin") + if err != nil { + return nil, fmt.Errorf("write lfs pointer for %q: %w", ops[i].Path, err) + } + + if !isHexSHA(sha) { + return nil, fmt.Errorf("write lfs pointer for %q returned invalid sha %q", ops[i].Path, sha) + } + + ops[i].BlobSHA = sha + } + return ops, nil +} + // runs one cat-file --batch-check over every put blob sha (returning their sizes) -// and every delete target -// so unknown blobs and missing delete paths fail -func (l *Local) verifyCommitInputs(ctx context.Context, dir, oldSHA string, unborn bool, ops []CommitOp) (map[string]int64, error) { +func (l *Local) verifyCommitInputs(ctx context.Context, dir, oldSHA string, unborn bool, ops []CommitOp) ([]CommitOp, map[string]int64, error) { var in strings.Builder type query struct { op CommitOp @@ -680,7 +729,7 @@ func (l *Local) verifyCommitInputs(ctx context.Context, dir, oldSHA string, unbo for _, op := range ops { if op.Delete { if unborn { - return nil, fmt.Errorf("%w: %q", ErrPathNotFound, op.Path) + return nil, nil, fmt.Errorf("%w: %q", ErrPathNotFound, op.Path) } in.WriteString(oldSHA + ":" + op.Path + "\n") } else { @@ -691,14 +740,15 @@ func (l *Local) verifyCommitInputs(ctx context.Context, dir, oldSHA string, unbo out, err := l.runGit(ctx, dir, nil, strings.NewReader(in.String()), "cat-file", "--batch-check") if err != nil { - return nil, err + return nil, nil, err } lines := strings.Split(out, "\n") if len(lines) != len(queries) { - return nil, fmt.Errorf("unexpected batch-check output: %d lines for %d queries", len(lines), len(queries)) + return nil, nil, fmt.Errorf("unexpected batch-check output: %d lines for %d queries", len(lines), len(queries)) } + expanded := make([]CommitOp, 0, len(ops)) sizes := make(map[string]int64) for i, line := range lines { q := queries[i] @@ -706,34 +756,65 @@ func (l *Local) verifyCommitInputs(ctx context.Context, dir, oldSHA string, unbo switch { case len(fields) >= 2 && fields[len(fields)-1] == "missing": if q.isPut { - return nil, fmt.Errorf("%w: %s", ErrUnknownBlob, q.op.BlobSHA) + return nil, nil, fmt.Errorf("%w: %s", ErrUnknownBlob, q.op.BlobSHA) } - return nil, fmt.Errorf("%w: %q", ErrPathNotFound, q.op.Path) + return nil, nil, fmt.Errorf("%w: %q", ErrPathNotFound, q.op.Path) case len(fields) == 3 && fields[1] == "blob": size, err := strconv.ParseInt(fields[2], 10, 64) if err != nil { - return nil, fmt.Errorf("malformed batch-check size %q: %w", fields[2], err) + return nil, nil, fmt.Errorf("malformed batch-check size %q: %w", fields[2], err) } if q.isPut { sizes[q.op.BlobSHA] = size } + expanded = append(expanded, q.op) + case len(fields) == 3 && !q.isPut && fields[1] == "tree": + deletes, err := l.expandTreeDelete(ctx, dir, q.op.Path, fields[0]) + if err != nil { + return nil, nil, err + } + expanded = append(expanded, deletes...) + case len(fields) == 3 && !q.isPut && fields[1] == "commit": + expanded = append(expanded, q.op) case len(fields) == 3: // a delete target resolving to a tree or a put sha naming a non-blob if q.isPut { - return nil, fmt.Errorf("%w: %s is a %s", ErrUnknownBlob, q.op.BlobSHA, fields[1]) + return nil, nil, fmt.Errorf("%w: %s is a %s", ErrUnknownBlob, q.op.BlobSHA, fields[1]) } - return nil, fmt.Errorf("%w: %q is a %s", ErrNotABlob, q.op.Path, fields[1]) + return nil, nil, fmt.Errorf("%w: cannot delete %q of type %s", ErrInvalidOps, q.op.Path, fields[1]) default: - return nil, fmt.Errorf("malformed batch-check line: %q", line) + return nil, nil, fmt.Errorf("malformed batch-check line: %q", line) + } + } + return expanded, sizes, nil +} + +// recursive path listing for given directory, using "ls-tree -r" +func (l *Local) expandTreeDelete(ctx context.Context, dir, treePath, treeSHA string) ([]CommitOp, error) { + out, err := l.runGitBytes(ctx, dir, nil, nil, + "ls-tree", "-r", "-z", "--name-only", "--end-of-options", treeSHA, + ) + if err != nil { + return nil, fmt.Errorf("list delete tree %q: %w", treePath, err) + } + + var deletes []CommitOp + for name := range bytes.SplitSeq(out, []byte{0}) { + if len(name) == 0 { + continue } + deletes = append(deletes, CommitOp{ + Delete: true, + Path: treePath + "/" + string(name), + }) } - return sizes, nil + return deletes, nil } // separates .gitattributes changes from regular file changes func splitAttrOps(ops []CommitOp) (attr, files []CommitOp) { for _, op := range ops { - if op.Path == ".gitattributes" || strings.HasSuffix(op.Path, "/.gitattributes") { + if isAttributesPath(op.Path) { attr = append(attr, op) } else { files = append(files, op) @@ -747,10 +828,15 @@ func splitAttrOps(ops []CommitOp) (attr, files []CommitOp) { func (l *Local) cleanLFSTracked(ctx context.Context, dir string, env []string, ops []CommitOp, sizes map[string]int64, clean CleanFunc) ([]CommitOp, error) { var paths []string byPath := make(map[string]int) + pendingLFS := make(map[string]struct{}) + for i, op := range ops { if !op.Delete { paths = append(paths, op.Path) byPath[op.Path] = i + if op.Lfs != nil { + pendingLFS[op.Path] = struct{}{} + } } } if len(paths) == 0 { @@ -767,13 +853,23 @@ func (l *Local) cleanLFSTracked(ctx context.Context, dir string, env []string, o fields := strings.Split(out, "\x00") for i := 0; i+2 < len(fields); i += 3 { path, value := fields[i], fields[i+2] - if value != "lfs" { - continue - } idx, ok := byPath[path] if !ok { continue } + + if ops[idx].Lfs != nil { + delete(pendingLFS, path) + // do not allow commiting lfs objects, if they are not tracked as lfs + if value != "lfs" { + return nil, fmt.Errorf("%w: %q", ErrLFSNotTracked, path) + } + continue + } + + if value != "lfs" { + continue + } if clean == nil { return nil, fmt.Errorf("%w: %q", ErrLFSRequired, path) } @@ -787,6 +883,11 @@ func (l *Local) cleanLFSTracked(ctx context.Context, dir string, env []string, o } ops[idx].BlobSHA = pointerSHA } + + if len(pendingLFS) != 0 { + return nil, fmt.Errorf("check-attr omitted explicit lfs paths") + } + return ops, nil } @@ -1065,6 +1166,22 @@ func isHexSHA(s string) bool { return true } +func isLFSOID(oid string) bool { + if len(oid) != 64 { + return false + } + for _, c := range oid { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func isAttributesPath(treePath string) bool { + return treePath == ".gitattributes" || strings.HasSuffix(treePath, "/.gitattributes") +} + // just to keep track how much bytes were streamed type countingReader struct { r io.Reader diff --git a/internal/gitbackend/local_test.go b/internal/gitbackend/local_test.go index ec5fda3..22e11ec 100644 --- a/internal/gitbackend/local_test.go +++ b/internal/gitbackend/local_test.go @@ -1029,6 +1029,7 @@ func TestApplyCommit(t *testing.T) { first, err := l.ApplyCommit(ctx, repo, spec(zeroSHA, "init"), []CommitOp{ {Path: "README.md", BlobSHA: hello}, {Path: "src/run.sh", BlobSHA: script, Mode: "100755"}, + {Path: "src/config/default.txt", BlobSHA: hello}, }, nil) if err != nil { t.Fatal(err) @@ -1058,11 +1059,12 @@ func TestApplyCommit(t *testing.T) { t.Errorf("log = %q", logOut) } - // second commit: CAS on the known head, update one file, delete another + // second commit: CAS on the known head, update one file, and delete a + // directory with nested tracked content as one natural path operation v2 := blob("hello v2\n") second, err := l.ApplyCommit(ctx, repo, spec(first.NewSHA, "update"), []CommitOp{ {Path: "README.md", BlobSHA: v2}, - {Path: "src/run.sh", Delete: true}, + {Path: "src", Delete: true}, }, nil) if err != nil { t.Fatal(err) @@ -1075,8 +1077,8 @@ func TestApplyCommit(t *testing.T) { if readme, _ := os.ReadFile(filepath.Join(wt, "README.md")); string(readme) != "hello v2\n" { t.Errorf("README.md after pull = %q", readme) } - if _, err := os.Stat(filepath.Join(wt, "src", "run.sh")); !os.IsNotExist(err) { - t.Errorf("run.sh should be deleted, stat err = %v", err) + if _, err := os.Stat(filepath.Join(wt, "src")); !os.IsNotExist(err) { + t.Errorf("src directory should be deleted recursively, stat err = %v", err) } t.Run("errors", func(t *testing.T) { @@ -1095,6 +1097,12 @@ func TestApplyCommit(t *testing.T) { {"hostile branch", CommitSpec{Branch: "--help", Author: author, Message: "x"}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidBranch}, {"no ops", spec("", "x"), nil, ErrInvalidOps}, {"duplicate path", spec("", "x"), []CommitOp{{Path: "a", BlobSHA: hello}, {Path: "a", Delete: true}}, ErrInvalidOps}, + {"overlapping paths", spec("", "x"), []CommitOp{{Path: "README.md", Delete: true}, {Path: "README.md/child", BlobSHA: hello}}, ErrInvalidOps}, + {"put without object", spec("", "x"), []CommitOp{{Path: "a"}}, ErrInvalidOps}, + {"put with both objects", spec("", "x"), []CommitOp{{Path: "a", BlobSHA: hello, Lfs: &LfsObject{OID: strings.Repeat("a", 64), Size: 1}}}, ErrInvalidOps}, + {"invalid lfs oid", spec("", "x"), []CommitOp{{Path: "a", Lfs: &LfsObject{OID: "nope", Size: 1}}}, ErrInvalidOps}, + {"invalid lfs size", spec("", "x"), []CommitOp{{Path: "a", Lfs: &LfsObject{OID: strings.Repeat("a", 64), Size: -1}}}, ErrInvalidOps}, + {"lfs attributes file", spec("", "x"), []CommitOp{{Path: ".gitattributes", Lfs: &LfsObject{OID: strings.Repeat("a", 64), Size: 1}}}, ErrInvalidOps}, {"bad mode", spec("", "x"), []CommitOp{{Path: "a", BlobSHA: hello, Mode: "120000"}}, ErrInvalidOps}, {"missing author", CommitSpec{Branch: "main", Author: Identity{}, Message: "x"}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidOps}, {"missing message", CommitSpec{Branch: "main", Author: author}, []CommitOp{{Path: "a", BlobSHA: hello}}, ErrInvalidOps}, @@ -1156,6 +1164,34 @@ func TestApplyCommit(t *testing.T) { if notes.BlobSHA != hello { t.Errorf("notes.txt was cleaned but is not lfs-tracked") } + + // An explicit LFS object needs no clean callback: the backend writes its + // canonical pointer blob and still enforces the effective attributes. + explicitOID := strings.Repeat("cd", 32) + explicit, err := l.ApplyCommit(ctx, repo, spec(change.NewSHA, "explicit lfs"), []CommitOp{ + {Path: "direct.bin", Lfs: &LfsObject{OID: explicitOID, Size: 23}}, + }, nil) + if err != nil { + t.Fatal(err) + } + direct, err := l.StatBlob(ctx, repo, explicit.NewSHA, "direct.bin") + if err != nil { + t.Fatal(err) + } + var directPointer bytes.Buffer + if err := l.ReadBlob(ctx, repo, direct.BlobSHA, &directPointer); err != nil { + t.Fatal(err) + } + wantPointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + explicitOID + "\nsize 23\n" + if directPointer.String() != wantPointer { + t.Errorf("direct.bin pointer = %q, want %q", directPointer.String(), wantPointer) + } + + if _, err := l.ApplyCommit(ctx, repo, spec(explicit.NewSHA, "untracked lfs"), []CommitOp{ + {Path: "direct.dat", Lfs: &LfsObject{OID: explicitOID, Size: 23}}, + }, nil); !errors.Is(err, ErrLFSNotTracked) { + t.Fatalf("untracked explicit lfs: want ErrLFSNotTracked, got %v", err) + } }) } diff --git a/internal/gitbackend/types.go b/internal/gitbackend/types.go index 463955f..d604140 100644 --- a/internal/gitbackend/types.go +++ b/internal/gitbackend/types.go @@ -124,9 +124,15 @@ type BlobInfo struct { Size int64 } +type LfsObject struct { + OID string + Size int64 +} + type CommitOp struct { Delete bool Path string + Lfs *LfsObject BlobSHA string // puts only; must exist as a blob in this repo's odb Mode string // puts only: "100644" (default) or "100755" } diff --git a/internal/server/control/repositories/commits.go b/internal/server/control/repositories/commits.go index c82bc41..dc73b5e 100644 --- a/internal/server/control/repositories/commits.go +++ b/internal/server/control/repositories/commits.go @@ -57,6 +57,13 @@ func (h *handlers) createCommit(w http.ResponseWriter, r *http.Request) error { BlobSHA: op.BlobSHA, Executable: op.Executable, } + + if op.Lfs != nil { + ops[i].Lfs = &domain.CommitFileLfsObject{ + OID: op.Lfs.OID, + Size: op.Lfs.Size, + } + } } result, err := h.service.Commit(r.Context(), id, domain.CommitRequest{ @@ -78,6 +85,8 @@ func (h *handlers) createCommit(w http.ResponseWriter, r *http.Request) error { return response.NewError(http.StatusConflict, response.CodeHeadMismatch, "branch head does not match expectedHeadSha") case errors.Is(err, reposervice.ErrUnknownBlob): return response.NewError(http.StatusUnprocessableEntity, response.CodeUnknownBlob, "referenced blob not found, upload it first") + case errors.Is(err, reposervice.ErrLFSObjectNotFound): + return response.NewError(http.StatusUnprocessableEntity, response.CodeLFSObjectNotFound, "referenced lfs object not found or not verified, upload it first") case errors.Is(err, reposervice.ErrNothingToCommit): return response.NewError(http.StatusUnprocessableEntity, response.CodeNothingToCommit, "operations produce no change") case errors.Is(err, reposervice.ErrPathBlocked): diff --git a/internal/server/control/repositories/handlers_test.go b/internal/server/control/repositories/handlers_test.go index 63c57e9..c4c0ed1 100644 --- a/internal/server/control/repositories/handlers_test.go +++ b/internal/server/control/repositories/handlers_test.go @@ -883,7 +883,7 @@ func TestCreateCommit(t *testing.T) { t.Errorf("service request = %+v", req) } if len(req.Operations) != 2 || - req.Operations[0].Delete || !req.Operations[0].Executable || req.Operations[0].BlobSHA != strings.Repeat("b", 40) || + req.Operations[0].Delete || !req.Operations[0].Executable || req.Operations[0].BlobSHA == nil || *req.Operations[0].BlobSHA != strings.Repeat("b", 40) || !req.Operations[1].Delete || req.Operations[1].Path != "old.txt" { t.Errorf("service operations = %+v", req.Operations) } @@ -900,8 +900,11 @@ func TestCreateCommitValidation(t *testing.T) { {"missing author", `{"branch":"main","message":"x","operations":[{"op":"delete","path":"a"}]}`}, {"no operations", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[]}`}, {"bad op kind", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"move","path":"a"}]}`}, - {"put without blobSha", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"put","path":"a"}]}`}, + {"put without object", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"put","path":"a"}]}`}, + {"put with both sources", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"put","path":"a","blobSha":"abc","lfs":{"oid":"def","size":1}}]}`}, + {"put with zero lfs size", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"put","path":"a","lfs":{"oid":"def","size":0}}]}`}, {"delete with blobSha", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"delete","path":"a","blobSha":"abc"}]}`}, + {"delete with lfs", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"delete","path":"a","lfs":{"oid":"def","size":1}}]}`}, {"missing path", `{"branch":"main","message":"x","author":{"name":"a","email":"e"},"operations":[{"op":"delete"}]}`}, } @@ -933,6 +936,7 @@ func TestCreateCommitErrors(t *testing.T) { {"delete target missing", reposervice.ErrPathNotFound, http.StatusNotFound, "path_not_found"}, {"head mismatch", reposervice.ErrHeadMismatch, http.StatusConflict, "head_mismatch"}, {"unknown blob", reposervice.ErrUnknownBlob, http.StatusUnprocessableEntity, "unknown_blob"}, + {"unknown lfs object", reposervice.ErrLFSObjectNotFound, http.StatusUnprocessableEntity, "lfs_object_not_found"}, {"nothing to commit", reposervice.ErrNothingToCommit, http.StatusUnprocessableEntity, "nothing_to_commit"}, {"path blocked", reposervice.ErrPathBlocked, http.StatusUnprocessableEntity, "path_blocked"}, {"delete target is a dir", reposervice.ErrNotAFile, http.StatusBadRequest, "invalid_request"}, diff --git a/internal/server/control/repositories/types.go b/internal/server/control/repositories/types.go index b83641e..9cc7cbb 100644 --- a/internal/server/control/repositories/types.go +++ b/internal/server/control/repositories/types.go @@ -210,11 +210,17 @@ func newCommitDetails(commit domain.CommitDetails) CommitDetails { } } +type CommitObjectLfs struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + type CommitOperation struct { - Op string `json:"op"` // "put" | "delete" - Path string `json:"path"` - BlobSHA string `json:"blobSha,omitempty"` // puts only, from POST /blobs - Executable bool `json:"executable,omitempty"` // puts only + Op string `json:"op"` // "put" | "delete" + Path string `json:"path"` + Lfs *CommitObjectLfs `json:"lfs,omitempty"` // puts only, from POST .../lfs/objects/batch + BlobSHA *string `json:"blobSha,omitempty"` // puts only, from POST /blobs + Executable bool `json:"executable,omitempty"` // puts only } type CreateCommitRequest struct { @@ -245,12 +251,24 @@ func (r CreateCommitRequest) Validate() error { } switch op.Op { case "put": - if op.BlobSHA == "" { - return fmt.Errorf("operations[%d]: blobSha is required for put", i) + if (op.BlobSHA == nil) == (op.Lfs == nil) { + return fmt.Errorf("operations[%d]: exactly one of blobSha or lfs is required for put", i) + } + if op.BlobSHA != nil { + if *op.BlobSHA == "" { + return fmt.Errorf("operations[%d]: blobSha is required for put", i) + } + } else { + if op.Lfs.OID == "" { + return fmt.Errorf("operations[%d]: lfs.oid is required", i) + } + if op.Lfs.Size <= 0 { + return fmt.Errorf("operations[%d]: lfs.size must be positive", i) + } } case "delete": - if op.BlobSHA != "" || op.Executable { - return fmt.Errorf("operations[%d]: delete takes no blobSha or executable", i) + if op.BlobSHA != nil || op.Lfs != nil || op.Executable { + return fmt.Errorf("operations[%d]: delete takes no blobSha, lfs, or executable", i) } default: return fmt.Errorf("operations[%d]: op must be 'put' or 'delete'", i) diff --git a/internal/services/repositories/service.go b/internal/services/repositories/service.go index 210b37b..d5f9a43 100644 --- a/internal/services/repositories/service.go +++ b/internal/services/repositories/service.go @@ -17,6 +17,7 @@ import ( "github.com/Axenos-dev/HeadlessGit/internal/db/gen" "github.com/Axenos-dev/HeadlessGit/internal/domain" "github.com/Axenos-dev/HeadlessGit/internal/gitbackend" + lfsservice "github.com/Axenos-dev/HeadlessGit/internal/services/lfs" "go.uber.org/zap" ) @@ -527,10 +528,22 @@ func (s *Service) Commit(ctx context.Context, repositoryID int64, req domain.Com mode = "100755" } ops[i] = gitbackend.CommitOp{ - Delete: op.Delete, - Path: op.Path, - BlobSHA: op.BlobSHA, - Mode: mode, + Delete: op.Delete, + Path: op.Path, + Mode: mode, + } + if op.BlobSHA != nil { + ops[i].BlobSHA = *op.BlobSHA + } + + if op.Lfs != nil { + if err := s.validateLfsObject(ctx, repo, *op.Lfs); err != nil { + return domain.CommitResult{}, err + } + ops[i].Lfs = &gitbackend.LfsObject{ + OID: op.Lfs.OID, + Size: op.Lfs.Size, + } } } @@ -555,7 +568,7 @@ func (s *Service) Commit(ctx context.Context, repositoryID int64, req domain.Com switch { case errors.Is(err, gitbackend.ErrInvalidBranch): return domain.CommitResult{}, ErrInvalidBranch - case errors.Is(err, gitbackend.ErrInvalidOps), errors.Is(err, gitbackend.ErrInvalidPath), errors.Is(err, gitbackend.ErrInvalidRev): + case errors.Is(err, gitbackend.ErrInvalidOps), errors.Is(err, gitbackend.ErrInvalidPath), errors.Is(err, gitbackend.ErrInvalidRev), errors.Is(err, gitbackend.ErrLFSNotTracked): return domain.CommitResult{}, fmt.Errorf("%w: %s", ErrInvalidCommitOps, err) case errors.Is(err, gitbackend.ErrRevNotFound): return domain.CommitResult{}, ErrRefNotFound @@ -584,6 +597,30 @@ func (s *Service) Commit(ctx context.Context, repositoryID int64, req domain.Com }, nil } +// validates the LFS objects, checks if it exists and has a right size +func (s *Service) validateLfsObject(ctx context.Context, repo domain.Repository, lfsObject domain.CommitFileLfsObject) error { + if s.lfs == nil { + return ErrLFSNotEnabled + } + + // verify if it exists + object, size, err := s.lfs.GetObject(ctx, repo, lfsObject.OID) + switch { + case errors.Is(err, lfsservice.ErrObjectNotFound): + return ErrLFSObjectNotFound + case err != nil: + return err + } + defer object.Close() + + // verify its size + if size != lfsObject.Size { + return fmt.Errorf("%w: lfs object %s has size %d, requested %d", ErrInvalidCommitOps, lfsObject.OID, size, lfsObject.Size) + } + + return nil +} + func (s *Service) lfsCleanFunc(ctx context.Context, repo domain.Repository, pusherID int64) gitbackend.CleanFunc { uploaderID := pusherID if uploaderID == 0 { diff --git a/internal/services/repositories/service_test.go b/internal/services/repositories/service_test.go index d6c0164..5c30ed1 100644 --- a/internal/services/repositories/service_test.go +++ b/internal/services/repositories/service_test.go @@ -17,6 +17,7 @@ import ( "github.com/Axenos-dev/HeadlessGit/internal/db/gen" "github.com/Axenos-dev/HeadlessGit/internal/domain" "github.com/Axenos-dev/HeadlessGit/internal/gitbackend" + lfsservice "github.com/Axenos-dev/HeadlessGit/internal/services/lfs" "go.uber.org/zap" ) @@ -165,10 +166,12 @@ type fakeLFS struct { stored map[string]string // oid -> content received via StoreObject } +func stringPtr(value string) *string { return &value } + func (f fakeLFS) GetObject(ctx context.Context, repo domain.Repository, oid string) (io.ReadCloser, int64, error) { content, ok := f.objects[oid] if !ok { - return nil, 0, errors.New("object not found") + return nil, 0, lfsservice.ErrObjectNotFound } return io.NopCloser(strings.NewReader(content)), int64(len(content)), nil } @@ -677,7 +680,7 @@ func TestCommit(t *testing.T) { ExpectedHeadSHA: strings.Repeat("a", 40), PusherID: 42, Operations: []domain.CommitFileOp{ - {Path: "run.sh", BlobSHA: blobSHA, Executable: true}, + {Path: "run.sh", BlobSHA: stringPtr(blobSHA), Executable: true}, {Path: "old.txt", Delete: true}, }, } @@ -734,6 +737,56 @@ func TestCommit(t *testing.T) { } }) + t.Run("explicit lfs object requires lfs service", func(t *testing.T) { + lfsReq := req + lfsReq.Operations = []domain.CommitFileOp{{ + Path: "model.bin", + Lfs: &domain.CommitFileLfsObject{OID: strings.Repeat("a", 64), Size: 42}, + }} + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, fakeStorage{}, nil, nil) + if _, err := svc.Commit(context.Background(), row.ID, lfsReq); !errors.Is(err, ErrLFSNotEnabled) { + t.Fatalf("want ErrLFSNotEnabled, got %v", err) + } + }) + + t.Run("maps a verified explicit lfs object", func(t *testing.T) { + oid := strings.Repeat("a", 64) + lfsReq := req + lfsReq.Operations = []domain.CommitFileOp{{ + Path: "model.bin", + Lfs: &domain.CommitFileLfsObject{OID: oid, Size: 42}, + }} + st := fakeStorage{applyChange: change, applyFn: func(_ gitbackend.CommitSpec, ops []gitbackend.CommitOp, _ gitbackend.CleanFunc) error { + if len(ops) != 1 || ops[0].BlobSHA != "" || ops[0].Lfs == nil || ops[0].Lfs.OID != oid || ops[0].Lfs.Size != 42 { + t.Errorf("backend ops = %+v", ops) + } + return nil + }} + svc := NewService(zap.NewNop(), fakeRegistry{repo: row}, st, fakeLFS{objects: map[string]string{oid: strings.Repeat("x", 42)}}, nil) + if _, err := svc.Commit(context.Background(), row.ID, lfsReq); err != nil { + t.Fatal(err) + } + }) + + t.Run("rejects unavailable or mismatched lfs object", func(t *testing.T) { + oid := strings.Repeat("a", 64) + lfsReq := req + lfsReq.Operations = []domain.CommitFileOp{{ + Path: "model.bin", + Lfs: &domain.CommitFileLfsObject{OID: oid, Size: 42}, + }} + + missing := NewService(zap.NewNop(), fakeRegistry{repo: row}, fakeStorage{}, fakeLFS{}, nil) + if _, err := missing.Commit(context.Background(), row.ID, lfsReq); !errors.Is(err, ErrLFSObjectNotFound) { + t.Fatalf("missing object: want ErrLFSObjectNotFound, got %v", err) + } + + mismatch := NewService(zap.NewNop(), fakeRegistry{repo: row}, fakeStorage{}, fakeLFS{objects: map[string]string{oid: "short"}}, nil) + if _, err := mismatch.Commit(context.Background(), row.ID, lfsReq); !errors.Is(err, ErrInvalidCommitOps) { + t.Fatalf("size mismatch: want ErrInvalidCommitOps, got %v", err) + } + }) + t.Run("error mapping and no event on failure", func(t *testing.T) { cases := []struct { backend error @@ -748,6 +801,7 @@ func TestCommit(t *testing.T) { {gitbackend.ErrUnknownBlob, ErrUnknownBlob}, {gitbackend.ErrNothingToCommit, ErrNothingToCommit}, {gitbackend.ErrLFSRequired, ErrLFSNotEnabled}, + {gitbackend.ErrLFSNotTracked, ErrInvalidCommitOps}, } for _, tc := range cases { var events []domain.RepositoryEvent @@ -770,7 +824,7 @@ func TestCommitCleanClosure(t *testing.T) { Message: "x", Author: domain.CommitIdentity{Name: "t", Email: "t@t"}, Operations: []domain.CommitFileOp{ - {Path: "big.bin", BlobSHA: blobSHA}, + {Path: "big.bin", BlobSHA: stringPtr(blobSHA)}, }, } @@ -885,12 +939,12 @@ func TestCommitPathPolicies(t *testing.T) { ops []domain.CommitFileOp blocked bool }{ - {"put inside blocked dir", []domain.CommitFileOp{{Path: "runtime/state.json", BlobSHA: blobSHA}}, true}, - {"put blocked file", []domain.CommitFileOp{{Path: "config.lock", BlobSHA: blobSHA}}, true}, - {"dot-segment evasion", []domain.CommitFileOp{{Path: "./runtime/state.json", BlobSHA: blobSHA}}, true}, + {"put inside blocked dir", []domain.CommitFileOp{{Path: "runtime/state.json", BlobSHA: stringPtr(blobSHA)}}, true}, + {"put blocked file", []domain.CommitFileOp{{Path: "config.lock", BlobSHA: stringPtr(blobSHA)}}, true}, + {"dot-segment evasion", []domain.CommitFileOp{{Path: "./runtime/state.json", BlobSHA: stringPtr(blobSHA)}}, true}, {"delete of blocked path allowed", []domain.CommitFileOp{{Path: "runtime/state.json", Delete: true}}, false}, - {"unrelated put", []domain.CommitFileOp{{Path: "src/main.go", BlobSHA: blobSHA}}, false}, - {"sibling prefix not blocked", []domain.CommitFileOp{{Path: "runtimes/x", BlobSHA: blobSHA}}, false}, + {"unrelated put", []domain.CommitFileOp{{Path: "src/main.go", BlobSHA: stringPtr(blobSHA)}}, false}, + {"sibling prefix not blocked", []domain.CommitFileOp{{Path: "runtimes/x", BlobSHA: stringPtr(blobSHA)}}, false}, } for _, tc := range cases { @@ -927,7 +981,7 @@ func TestCommitPathPolicies(t *testing.T) { t.Run("reason is echoed", func(t *testing.T) { svc := NewService(zap.NewNop(), fakeRegistry{repo: row, policies: policies}, fakeStorage{}, nil, nil) req := base - req.Operations = []domain.CommitFileOp{{Path: "runtime/x", BlobSHA: blobSHA}} + req.Operations = []domain.CommitFileOp{{Path: "runtime/x", BlobSHA: stringPtr(blobSHA)}} _, err := svc.Commit(context.Background(), row.ID, req) if err == nil || !strings.Contains(err.Error(), "deploy-managed state") { t.Errorf("reason missing from error: %v", err)