From 5ec084ac8b3c7d018addfacec6aacc9989cc3122 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 19:06:49 +0800 Subject: [PATCH 01/10] feat(worker): add the modelartifact expecteddigest anchor - spec.expectedDigest asserts the manifest digest a hub source must resolve to; admission accepts it on hub sources only, refusing claim and image sources each with the reason their identity rules it out, and the whole-spec immutability ratchet covers it - status.resolved.digestSource records where a resolved digest came from: Hub, the hub's own listing, or Expected, the spec's anchor - the stale modelscope member comment, left reserved-shaped by #699, now describes the opened source - generated deepcopy, crd, protobuf, openapi and applyconfiguration artifacts Task 1 of model-artifact-expected-digest. Signed-off-by: thxCode --- api/worker/v1alpha1/generated.pb.go | 80 +++++++++++++++++ api/worker/v1alpha1/generated.proto | 32 +++++-- api/worker/v1alpha1/model_artifact.go | 44 +++++++-- api/worker/v1alpha1/zz_generated.crds.go | 12 ++- api/worker/zz_generated.openapi.go | 18 +++- .../worker/v1alpha1/modelartifactresolved.go | 13 +++ .../worker/v1alpha1/modelartifactsource.go | 11 +-- .../worker/v1alpha1/modelartifactspec.go | 17 ++++ pkg/modelartifact/modelscope.go | 10 +-- pkg/worker/webhooks/worker/model_artifact.go | 33 ++++++- .../webhooks/worker/model_artifact_test.go | 90 +++++++++++++++++++ .../worker/model_deployment_artifact_test.go | 6 +- 12 files changed, 340 insertions(+), 26 deletions(-) diff --git a/api/worker/v1alpha1/generated.pb.go b/api/worker/v1alpha1/generated.pb.go index 67379d06f..1651a4c4f 100644 --- a/api/worker/v1alpha1/generated.pb.go +++ b/api/worker/v1alpha1/generated.pb.go @@ -5315,6 +5315,11 @@ func (m *ModelArtifactResolved) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + i -= len(m.DigestSource) + copy(dAtA[i:], m.DigestSource) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.DigestSource))) + i-- + dAtA[i] = 0x3a if m.LastValidatedTime != nil { { size, err := m.LastValidatedTime.MarshalToSizedBuffer(dAtA[:i]) @@ -5447,6 +5452,11 @@ func (m *ModelArtifactSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + i -= len(m.ExpectedDigest) + copy(dAtA[i:], m.ExpectedDigest) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.ExpectedDigest))) + i-- + dAtA[i] = 0x22 if len(m.IgnorePatterns) > 0 { for iNdEx := len(m.IgnorePatterns) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.IgnorePatterns[iNdEx]) @@ -10544,6 +10554,8 @@ func (m *ModelArtifactResolved) Size() (n int) { l = m.LastValidatedTime.Size() n += 1 + l + sovGenerated(uint64(l)) } + l = len(m.DigestSource) + n += 1 + l + sovGenerated(uint64(l)) return n } @@ -10592,6 +10604,8 @@ func (m *ModelArtifactSpec) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + l = len(m.ExpectedDigest) + n += 1 + l + sovGenerated(uint64(l)) return n } @@ -13206,6 +13220,7 @@ func (this *ModelArtifactResolved) String() string { `SizeBytes:` + fmt.Sprintf("%v", this.SizeBytes) + `,`, `ResolvedTime:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.ResolvedTime), "Time", "v1.Time", 1), `&`, ``, 1) + `,`, `LastValidatedTime:` + strings.Replace(fmt.Sprintf("%v", this.LastValidatedTime), "Time", "v1.Time", 1) + `,`, + `DigestSource:` + fmt.Sprintf("%v", this.DigestSource) + `,`, `}`, }, "") return s @@ -13231,6 +13246,7 @@ func (this *ModelArtifactSpec) String() string { `Source:` + strings.Replace(strings.Replace(this.Source.String(), "ModelArtifactSource", "ModelArtifactSource", 1), `&`, ``, 1) + `,`, `AllowPatterns:` + fmt.Sprintf("%v", this.AllowPatterns) + `,`, `IgnorePatterns:` + fmt.Sprintf("%v", this.IgnorePatterns) + `,`, + `ExpectedDigest:` + fmt.Sprintf("%v", this.ExpectedDigest) + `,`, `}`, }, "") return s @@ -30262,6 +30278,38 @@ func (m *ModelArtifactResolved) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DigestSource", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DigestSource = ModelArtifactDigestSource(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -30603,6 +30651,38 @@ func (m *ModelArtifactSpec) Unmarshal(dAtA []byte) error { } m.IgnorePatterns = append(m.IgnorePatterns, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExpectedDigest", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ExpectedDigest = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/api/worker/v1alpha1/generated.proto b/api/worker/v1alpha1/generated.proto index cd49c37b5..df414839e 100644 --- a/api/worker/v1alpha1/generated.proto +++ b/api/worker/v1alpha1/generated.proto @@ -3019,6 +3019,13 @@ message ModelArtifactResolved { // // +optional optional .k8s.io.apimachinery.pkg.apis.meta.v1.Time lastValidatedTime = 6; + + // DigestSource says where ManifestDigest came from: Hub, the hub's own listing, or Expected, + // the spec's expectedDigest written after the hub's absence was confirmed. An Expected + // identity carries no Revision, FileCount or SizeBytes and never contacts the hub again. + // + // +optional + optional string digestSource = 7; } // ModelArtifactSource is a tagged union: exactly one member is set, webhook-enforced. @@ -3028,11 +3035,12 @@ message ModelArtifactSource { // +optional optional ModelArtifactHubSource huggingFace = 1; - // ModelScope is RESERVED AND REFUSED by admission in this version. Its shape is fixed so that - // opening it is a webhook change rather than a schema change, and the refusal names what opening - // it needs: branch resolution cross-checked against git, a listing that re-lists per directory at - // the API's silent truncation point, errors classified by the envelope code, and an engine runner - // whose ModelScope SDK accepts a commit as the revision. + // ModelScope is a second hub. Its repository and revision rules are the Hugging Face ones, + // the revision defaults to "master", and the patterns apply to it the same way. What + // differs is how it resolves: a full commit is taken as is, a branch or tag is resolved + // through the commits API and cross-checked against git, the file listing re-lists per + // directory at the API's silent truncation point, and the engine's runner needs a + // ModelScope SDK that accepts a commit as the revision. // // +optional optional ModelArtifactHubSource modelScope = 2; @@ -3087,6 +3095,20 @@ message ModelArtifactSpec { // +listType=atomic // +k8s:validation:maxItems=32 repeated string ignorePatterns = 3; + + // ExpectedDigest optionally asserts the manifest digest this artifact must resolve to: + // "sha256:" and 64 lowercase hex, the content address status.resolved.manifestDigest + // carries. Admission accepts it on a hub source only: a claim's content is whatever the + // volume holds at mount time — dynamically provisioned claims differ per provisioning — + // and its identity is the claim itself, which the user confirms; an image's identity is + // its reference's digest. A resolution whose digest differs is refused as DigestMismatch, + // and an artifact whose hub cannot be reached resolves to the anchor without the hub, + // recorded as status.resolved.digestSource "Expected". Immutable with the whole spec. + // + // +optional + // +k8s:validation:pattern="^sha256:[a-f0-9]{64}$" + // +k8s:validation:maxLength=71 + optional string expectedDigest = 4; } // ModelArtifactStatus defines the observed state of ModelArtifact. diff --git a/api/worker/v1alpha1/model_artifact.go b/api/worker/v1alpha1/model_artifact.go index fe8670549..9cfadc348 100644 --- a/api/worker/v1alpha1/model_artifact.go +++ b/api/worker/v1alpha1/model_artifact.go @@ -70,6 +70,20 @@ type ModelArtifactSpec struct { // +listType=atomic // +k8s:validation:maxItems=32 IgnorePatterns []string `json:"ignorePatterns,omitempty" protobuf:"bytes,3,rep,name=ignorePatterns"` + + // ExpectedDigest optionally asserts the manifest digest this artifact must resolve to: + // "sha256:" and 64 lowercase hex, the content address status.resolved.manifestDigest + // carries. Admission accepts it on a hub source only: a claim's content is whatever the + // volume holds at mount time — dynamically provisioned claims differ per provisioning — + // and its identity is the claim itself, which the user confirms; an image's identity is + // its reference's digest. A resolution whose digest differs is refused as DigestMismatch, + // and an artifact whose hub cannot be reached resolves to the anchor without the hub, + // recorded as status.resolved.digestSource "Expected". Immutable with the whole spec. + // + // +optional + // +k8s:validation:pattern="^sha256:[a-f0-9]{64}$" + // +k8s:validation:maxLength=71 + ExpectedDigest string `json:"expectedDigest,omitempty" protobuf:"bytes,4,opt,name=expectedDigest"` } // ModelArtifactSource is a tagged union: exactly one member is set, webhook-enforced. @@ -79,11 +93,12 @@ type ModelArtifactSource struct { // +optional HuggingFace *ModelArtifactHubSource `json:"huggingFace,omitempty" protobuf:"bytes,1,opt,name=huggingFace"` - // ModelScope is RESERVED AND REFUSED by admission in this version. Its shape is fixed so that - // opening it is a webhook change rather than a schema change, and the refusal names what opening - // it needs: branch resolution cross-checked against git, a listing that re-lists per directory at - // the API's silent truncation point, errors classified by the envelope code, and an engine runner - // whose ModelScope SDK accepts a commit as the revision. + // ModelScope is a second hub. Its repository and revision rules are the Hugging Face ones, + // the revision defaults to "master", and the patterns apply to it the same way. What + // differs is how it resolves: a full commit is taken as is, a branch or tag is resolved + // through the commits API and cross-checked against git, the file listing re-lists per + // directory at the API's silent truncation point, and the engine's runner needs a + // ModelScope SDK that accepts a commit as the revision. // // +optional ModelScope *ModelArtifactHubSource `json:"modelScope,omitempty" protobuf:"bytes,2,opt,name=modelScope"` @@ -266,8 +281,27 @@ type ModelArtifactResolved struct { // // +optional LastValidatedTime *meta.Time `json:"lastValidatedTime,omitempty" protobuf:"bytes,6,opt,name=lastValidatedTime"` + + // DigestSource says where ManifestDigest came from: Hub, the hub's own listing, or Expected, + // the spec's expectedDigest written after the hub's absence was confirmed. An Expected + // identity carries no Revision, FileCount or SizeBytes and never contacts the hub again. + // + // +optional + DigestSource ModelArtifactDigestSource `json:"digestSource,omitempty" protobuf:"bytes,7,opt,name=digestSource"` } +// ModelArtifactDigestSource is where a resolved manifest digest came from. +type ModelArtifactDigestSource string + +const ( + // ModelArtifactDigestSourceHub is a digest the hub's own listing produced. + ModelArtifactDigestSourceHub ModelArtifactDigestSource = "Hub" + + // ModelArtifactDigestSourceExpected is the spec's expectedDigest, written as the identity + // when the hub's absence was confirmed. + ModelArtifactDigestSourceExpected ModelArtifactDigestSource = "Expected" +) + // ModelArtifactList holds the list of ModelArtifact. // // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/worker/v1alpha1/zz_generated.crds.go b/api/worker/v1alpha1/zz_generated.crds.go index 45cf66052..91339123c 100644 --- a/api/worker/v1alpha1/zz_generated.crds.go +++ b/api/worker/v1alpha1/zz_generated.crds.go @@ -4204,6 +4204,12 @@ func crd_gpustack_api_worker_v1alpha1_ModelArtifact() *v1.CustomResourceDefiniti Nullable: true, XListType: ptr.To[string]("atomic"), }, + "expectedDigest": { + Description: "ExpectedDigest optionally asserts the manifest digest this artifact must resolve to:\n\"sha256:\" and 64 lowercase hex, the content address status.resolved.manifestDigest\ncarries. Admission accepts it on a hub source only: a claim's content is whatever the\nvolume holds at mount time — dynamically provisioned claims differ per provisioning —\nand its identity is the claim itself, which the user confirms; an image's identity is\nits reference's digest. A resolution whose digest differs is refused as DigestMismatch,\nand an artifact whose hub cannot be reached resolves to the anchor without the hub,\nrecorded as status.resolved.digestSource \"Expected\". Immutable with the whole spec.", + Type: "string", + MaxLength: ptr.To[int64](71), + Pattern: `^sha256:[a-f0-9]{64}$`, + }, "ignorePatterns": { Type: "array", MaxItems: ptr.To[int64](32), @@ -4271,7 +4277,7 @@ func crd_gpustack_api_worker_v1alpha1_ModelArtifact() *v1.CustomResourceDefiniti Nullable: true, }, "modelScope": { - Description: "ModelScope is RESERVED AND REFUSED by admission in this version. Its shape is fixed so that\nopening it is a webhook change rather than a schema change, and the refusal names what opening\nit needs: branch resolution cross-checked against git, a listing that re-lists per directory at\nthe API's silent truncation point, errors classified by the envelope code, and an engine runner\nwhose ModelScope SDK accepts a commit as the revision.", + Description: "ModelScope is a second hub. Its repository and revision rules are the Hugging Face ones,\nthe revision defaults to \"master\", and the patterns apply to it the same way. What\ndiffers is how it resolves: a full commit is taken as is, a branch or tag is resolved\nthrough the commits API and cross-checked against git, the file listing re-lists per\ndirectory at the API's silent truncation point, and the engine's runner needs a\nModelScope SDK that accepts a commit as the revision.", Type: "object", Required: []string{ "repository", @@ -4448,6 +4454,10 @@ func crd_gpustack_api_worker_v1alpha1_ModelArtifact() *v1.CustomResourceDefiniti "resolvedTime", }, Properties: map[string]v1.JSONSchemaProps{ + "digestSource": { + Description: "DigestSource says where ManifestDigest came from: Hub, the hub's own listing, or Expected,\nthe spec's expectedDigest written after the hub's absence was confirmed. An Expected\nidentity carries no Revision, FileCount or SizeBytes and never contacts the hub again.", + Type: "string", + }, "fileCount": { Description: "FileCount and SizeBytes are the manifest's file count and total size. Absent for a claim.", Type: "integer", diff --git a/api/worker/zz_generated.openapi.go b/api/worker/zz_generated.openapi.go index 0aef2d4a9..991ad52c3 100644 --- a/api/worker/zz_generated.openapi.go +++ b/api/worker/zz_generated.openapi.go @@ -8354,6 +8354,13 @@ func schema_gpustack_api_worker_v1alpha1_ModelArtifactResolved(ref common.Refere Ref: ref(metav1.Time{}.OpenAPIModelName()), }, }, + "digestSource": { + SchemaProps: spec.SchemaProps{ + Description: "DigestSource says where ManifestDigest came from: Hub, the hub's own listing, or Expected, the spec's expectedDigest written after the hub's absence was confirmed. An Expected identity carries no Revision, FileCount or SizeBytes and never contacts the hub again.", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"resolvedTime"}, }, @@ -8378,7 +8385,7 @@ func schema_gpustack_api_worker_v1alpha1_ModelArtifactSource(ref common.Referenc }, "modelScope": { SchemaProps: spec.SchemaProps{ - Description: "ModelScope is RESERVED AND REFUSED by admission in this version. Its shape is fixed so that opening it is a webhook change rather than a schema change, and the refusal names what opening it needs: branch resolution cross-checked against git, a listing that re-lists per directory at the API's silent truncation point, errors classified by the envelope code, and an engine runner whose ModelScope SDK accepts a commit as the revision.", + Description: "ModelScope is a second hub. Its repository and revision rules are the Hugging Face ones, the revision defaults to \"master\", and the patterns apply to it the same way. What differs is how it resolves: a full commit is taken as is, a branch or tag is resolved through the commits API and cross-checked against git, the file listing re-lists per directory at the API's silent truncation point, and the engine's runner needs a ModelScope SDK that accepts a commit as the revision.", Ref: ref(v1alpha1.ModelArtifactHubSource{}.OpenAPIModelName()), }, }, @@ -8457,6 +8464,15 @@ func schema_gpustack_api_worker_v1alpha1_ModelArtifactSpec(ref common.ReferenceC }, }, }, + "expectedDigest": { + SchemaProps: spec.SchemaProps{ + Description: "ExpectedDigest optionally asserts the manifest digest this artifact must resolve to: \"sha256:\" and 64 lowercase hex, the content address status.resolved.manifestDigest carries. Admission accepts it on a hub source only: a claim's content is whatever the volume holds at mount time — dynamically provisioned claims differ per provisioning — and its identity is the claim itself, which the user confirms; an image's identity is its reference's digest. A resolution whose digest differs is refused as DigestMismatch, and an artifact whose hub cannot be reached resolves to the anchor without the hub, recorded as status.resolved.digestSource \"Expected\". Immutable with the whole spec.", + MaxLength: ptr.To[int64](71), + Pattern: "^sha256:[a-f0-9]{64}$", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"source"}, }, diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactresolved.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactresolved.go index f1fb4f9a6..d5791f6c9 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactresolved.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactresolved.go @@ -3,6 +3,7 @@ package v1alpha1 import ( + workerv1alpha1 "gpustack.ai/gpustack/api/worker/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -28,6 +29,10 @@ type ModelArtifactResolvedApplyConfiguration struct { // LastValidatedTime is when access was last confirmed. Absent for a claim, which has no access // check of its own. LastValidatedTime *v1.Time `json:"lastValidatedTime,omitempty"` + // DigestSource says where ManifestDigest came from: Hub, the hub's own listing, or Expected, + // the spec's expectedDigest written after the hub's absence was confirmed. An Expected + // identity carries no Revision, FileCount or SizeBytes and never contacts the hub again. + DigestSource *workerv1alpha1.ModelArtifactDigestSource `json:"digestSource,omitempty"` } // ModelArtifactResolvedApplyConfiguration constructs a declarative configuration of the ModelArtifactResolved type for use with @@ -83,3 +88,11 @@ func (b *ModelArtifactResolvedApplyConfiguration) WithLastValidatedTime(value v1 b.LastValidatedTime = &value return b } + +// WithDigestSource sets the DigestSource field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DigestSource field is set to the value of the last call. +func (b *ModelArtifactResolvedApplyConfiguration) WithDigestSource(value workerv1alpha1.ModelArtifactDigestSource) *ModelArtifactResolvedApplyConfiguration { + b.DigestSource = &value + return b +} diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactsource.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactsource.go index 662bddde9..e383afac4 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactsource.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactsource.go @@ -9,11 +9,12 @@ package v1alpha1 type ModelArtifactSourceApplyConfiguration struct { // HuggingFace is a Hugging Face model repository at one revision. HuggingFace *ModelArtifactHubSourceApplyConfiguration `json:"huggingFace,omitempty"` - // ModelScope is RESERVED AND REFUSED by admission in this version. Its shape is fixed so that - // opening it is a webhook change rather than a schema change, and the refusal names what opening - // it needs: branch resolution cross-checked against git, a listing that re-lists per directory at - // the API's silent truncation point, errors classified by the envelope code, and an engine runner - // whose ModelScope SDK accepts a commit as the revision. + // ModelScope is a second hub. Its repository and revision rules are the Hugging Face ones, + // the revision defaults to "master", and the patterns apply to it the same way. What + // differs is how it resolves: a full commit is taken as is, a branch or tag is resolved + // through the commits API and cross-checked against git, the file listing re-lists per + // directory at the API's silent truncation point, and the engine's runner needs a + // ModelScope SDK that accepts a commit as the revision. ModelScope *ModelArtifactHubSourceApplyConfiguration `json:"modelScope,omitempty"` // PersistentVolumeClaim is a directory inside a claim in this namespace. The operator never // reads the claim's content, so the artifact has no revision and no digest, and what the diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactspec.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactspec.go index 90847931b..62bd424b2 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactspec.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modelartifactspec.go @@ -22,6 +22,15 @@ type ModelArtifactSpecApplyConfiguration struct { // each pattern is 1 to 256 characters without a control character. AllowPatterns []string `json:"allowPatterns,omitempty"` IgnorePatterns []string `json:"ignorePatterns,omitempty"` + // ExpectedDigest optionally asserts the manifest digest this artifact must resolve to: + // "sha256:" and 64 lowercase hex, the content address status.resolved.manifestDigest + // carries. Admission accepts it on a hub source only: a claim's content is whatever the + // volume holds at mount time — dynamically provisioned claims differ per provisioning — + // and its identity is the claim itself, which the user confirms; an image's identity is + // its reference's digest. A resolution whose digest differs is refused as DigestMismatch, + // and an artifact whose hub cannot be reached resolves to the anchor without the hub, + // recorded as status.resolved.digestSource "Expected". Immutable with the whole spec. + ExpectedDigest *string `json:"expectedDigest,omitempty"` } // ModelArtifactSpecApplyConfiguration constructs a declarative configuration of the ModelArtifactSpec type for use with @@ -57,3 +66,11 @@ func (b *ModelArtifactSpecApplyConfiguration) WithIgnorePatterns(values ...strin } return b } + +// WithExpectedDigest sets the ExpectedDigest field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ExpectedDigest field is set to the value of the last call. +func (b *ModelArtifactSpecApplyConfiguration) WithExpectedDigest(value string) *ModelArtifactSpecApplyConfiguration { + b.ExpectedDigest = &value + return b +} diff --git a/pkg/modelartifact/modelscope.go b/pkg/modelartifact/modelscope.go index 5ff00b992..2042fd450 100644 --- a/pkg/modelartifact/modelscope.go +++ b/pkg/modelartifact/modelscope.go @@ -245,7 +245,7 @@ func modelScopeManifestEntry(e modelScopeFileEntry) (ManifestEntry, bool, error) "entry %q is of type %q, which is neither a blob nor a tree", e.Path, e.Type) } if len(e.Sha256) != 64 || strings.ContainsFunc(e.Sha256, func(r rune) bool { - return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'f') + return (r < '0' || r > '9') && (r < 'a' || r > 'f') }) { return ManifestEntry{}, false, sourceErrorf(ReasonInvalidManifest, "entry %q names no usable Sha256, which every manifest line requires", e.Path) @@ -363,10 +363,10 @@ func (m *ModelScope) ValidToken(ctx context.Context, token string) (bool, error) return false, err } defer httpx.Close(resp) - switch { - case resp.StatusCode == http.StatusOK: + switch resp.StatusCode { + case http.StatusOK: return true, nil - case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden: + case http.StatusUnauthorized, http.StatusForbidden: return false, nil default: return false, sourceErrorf(ReasonSourceUnavailable, @@ -425,7 +425,7 @@ func (m *ModelScope) lsRemoteOverHTTP(ctx context.Context, gitURL, token string) resp, err := m.Client.Do(req) if err != nil { - return nil, fmt.Errorf("%s %s: %v", http.MethodGet, target, unwrapURLError(err)) + return nil, fmt.Errorf("%s %s: %w", http.MethodGet, target, unwrapURLError(err)) } defer httpx.Close(resp) switch { diff --git a/pkg/worker/webhooks/worker/model_artifact.go b/pkg/worker/webhooks/worker/model_artifact.go index 5c1a7ccb2..43743bea0 100644 --- a/pkg/worker/webhooks/worker/model_artifact.go +++ b/pkg/worker/webhooks/worker/model_artifact.go @@ -159,8 +159,37 @@ func validateModelArtifact(ma, old *workercore.ModelArtifact) field.ErrorList { errs = validateModelArtifactClaim(source.PersistentVolumeClaim, sourcePath.Child("persistentVolumeClaim")) } - return append(errs, validateModelArtifactPatterns(&ma.Spec, specPath, - source.HuggingFace != nil || source.ModelScope != nil, source.Image != nil)...) + hub := source.HuggingFace != nil || source.ModelScope != nil + errs = append(errs, validateModelArtifactExpectedDigest(ma.Spec.ExpectedDigest, &source, + specPath.Child("expectedDigest"))...) + + return append(errs, validateModelArtifactPatterns(&ma.Spec, specPath, hub, source.Image != nil)...) +} + +// validateModelArtifactExpectedDigest accepts the anchor on a hub source only, in the shape the +// manifest digest is written in. A claim's content is whatever the volume holds at mount time — +// dynamically provisioned, possibly different each time — and its identity is the claim itself, +// which the user confirms; an image's identity is its reference's digest. Neither can be anchored +// to a manifest digest, and an anchor that could never match would only invite a permanently +// unresolved artifact. +func validateModelArtifactExpectedDigest(digest string, source *workercore.ModelArtifactSource, path *field.Path) field.ErrorList { + switch { + case digest == "": + return nil + case source.HuggingFace != nil || source.ModelScope != nil: + if !modelArtifactDigestPattern.MatchString(digest) { + return field.ErrorList{field.Invalid(path, digest, `must be "sha256:" and 64 lowercase hex`)} + } + return nil + case source.Image != nil: + return field.ErrorList{field.Forbidden(path, + "expectedDigest asserts the manifest digest a hub source resolves to; an image's identity is "+ + "its reference's digest")} + default: + return field.ErrorList{field.Forbidden(path, + "expectedDigest asserts the manifest digest a hub source resolves to; a claim's content is "+ + "whatever it holds at mount time and its identity is the claim itself")} + } } // modelArtifactMaxPatterns and modelArtifactMaxPatternLength bound a list of patterns and each of diff --git a/pkg/worker/webhooks/worker/model_artifact_test.go b/pkg/worker/webhooks/worker/model_artifact_test.go index c40945072..bab02b07f 100644 --- a/pkg/worker/webhooks/worker/model_artifact_test.go +++ b/pkg/worker/webhooks/worker/model_artifact_test.go @@ -331,6 +331,96 @@ func TestModelArtifactWebhookUpdateRatchets(t *testing.T) { }) } +// TestModelArtifactWebhookExpectedDigest covers the anchor's admission rules: accepted on a hub +// source in the exact digest shape, refused on the sources whose identity does not come from a +// manifest, and immutable with the spec. +func TestModelArtifactWebhookExpectedDigest(t *testing.T) { + good := "sha256:" + strings.Repeat("a", 64) + // The image-volume gate would otherwise answer before the anchor rule does; the snapshot's + // Configure ignores later calls, so the seam is swapped here as the image test does. + defaultVersion := modelArtifactClusterVersion + t.Cleanup(func() { modelArtifactClusterVersion = defaultVersion }) + modelArtifactClusterVersion = func() kubediscovery.Version { + return kubediscovery.Version{GitVersion: "v1.35.5"} + } + + cases := []struct { + name string + in *workercore.ModelArtifact + wantField string + }{ + {name: "a Hugging Face artifact with an anchor", in: withDigest(newTestHubArtifact("owner/repo", "main"), good)}, + {name: "a ModelScope artifact with an anchor", in: withDigest(newTestModelScopeArtifact("qwen/Qwen2.5-7B-Instruct", "master"), good)}, + { + name: "a claim source is refused the anchor", wantField: "spec.expectedDigest", + in: withDigest(newTestClaimArtifact("models", ""), good), + }, + { + name: "an image source is refused the anchor", wantField: "spec.expectedDigest", + in: withDigest(newTestImageArtifact("registry/qwen@sha256:"+strings.Repeat("b", 64)), good), + }, + {name: "an uppercase digest", wantField: "spec.expectedDigest", in: withDigest(newTestHubArtifact("owner/repo", "main"), "sha256:"+strings.Repeat("A", 64))}, + {name: "a short digest", wantField: "spec.expectedDigest", in: withDigest(newTestHubArtifact("owner/repo", "main"), "sha256:"+strings.Repeat("a", 63))}, + {name: "no algorithm prefix", wantField: "spec.expectedDigest", in: withDigest(newTestHubArtifact("owner/repo", "main"), strings.Repeat("a", 64))}, + {name: "the wrong algorithm", wantField: "spec.expectedDigest", in: withDigest(newTestHubArtifact("owner/repo", "main"), "sha512:"+strings.Repeat("a", 128))}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, err := new(ModelArtifactWebhook).ValidateCreate(context.Background(), c.in) + if c.wantField == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + assertInvalidField(t, err, c.wantField) + }) + } + + t.Run("the claim refusal says the anchor is a hub source's assertion", func(t *testing.T) { + _, err := new(ModelArtifactWebhook).ValidateCreate(context.Background(), + withDigest(newTestClaimArtifact("models", ""), good)) + require.Error(t, err) + status, ok := err.(kerrors.APIStatus) + require.True(t, ok) + message := status.Status().Details.Causes[0].Message + assert.Contains(t, message, "mount time") + assert.Contains(t, message, "identity is the claim") + }) + + t.Run("the image refusal names the reference's digest as the identity", func(t *testing.T) { + _, err := new(ModelArtifactWebhook).ValidateCreate(context.Background(), + withDigest(newTestImageArtifact("registry/qwen@sha256:"+strings.Repeat("b", 64)), good)) + require.Error(t, err) + status, ok := err.(kerrors.APIStatus) + require.True(t, ok) + message := status.Status().Details.Causes[0].Message + assert.Contains(t, message, "reference's digest") + }) + + t.Run("an update changing only the anchor is a spec edit", func(t *testing.T) { + old := newTestHubArtifact("owner/repo", "main") + updated := old.DeepCopy() + updated.Spec.ExpectedDigest = good + + _, err := new(ModelArtifactWebhook).ValidateUpdate(context.Background(), old, updated) + require.Error(t, err) + assertInvalidField(t, err, "spec") + }) + t.Run("an update keeping the anchor is metadata only", func(t *testing.T) { + old := withDigest(newTestHubArtifact("owner/repo", "main"), good) + updated := old.DeepCopy() + updated.Labels = map[string]string{"a": "b"} + + _, err := new(ModelArtifactWebhook).ValidateUpdate(context.Background(), old, updated) + require.NoError(t, err) + }) +} + +func withDigest(ma *workercore.ModelArtifact, digest string) *workercore.ModelArtifact { + ma.Spec.ExpectedDigest = digest + return ma +} + func withPatterns(ma *workercore.ModelArtifact, allow, ignore []string) *workercore.ModelArtifact { ma.Spec.AllowPatterns, ma.Spec.IgnorePatterns = allow, ignore return ma diff --git a/pkg/worker/webhooks/worker/model_deployment_artifact_test.go b/pkg/worker/webhooks/worker/model_deployment_artifact_test.go index 2ca249b1b..9983c55ea 100644 --- a/pkg/worker/webhooks/worker/model_deployment_artifact_test.go +++ b/pkg/worker/webhooks/worker/model_deployment_artifact_test.go @@ -118,8 +118,10 @@ func TestValidateModelDeploymentArtifactOwnedKeys(t *testing.T) { { name: "the ModelScope environment", engine: workercore.ModelDeploymentEngineVLLM, env: []workercore.ModelDeploymentEnvVar{ - {Name: "MODELSCOPE_API_TOKEN", Value: "x"}, {Name: "MODELSCOPE_DOMAIN", Value: "ms"}, - {Name: "MODELSCOPE_CACHE", Value: "/c"}, {Name: "VLLM_USE_MODELSCOPE", Value: "true"}, + {Name: "MODELSCOPE_API_TOKEN", Value: "x"}, + {Name: "MODELSCOPE_DOMAIN", Value: "ms"}, + {Name: "MODELSCOPE_CACHE", Value: "/c"}, + {Name: "VLLM_USE_MODELSCOPE", Value: "true"}, }, wantField: []string{ "spec.roles[0].env[0]", "spec.roles[0].env[1]", "spec.roles[0].env[2]", "spec.roles[0].env[3]", From 6212820bea10d5f56e604e6f0939fcbc1ef8e625 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 20:07:24 +0800 Subject: [PATCH 02/10] feat(worker): assert the expectedDigest at resolution and revalidation - a resolution whose manifest digest differs from the anchor is refused as DigestMismatch with both digests in the message, and DigestMismatch joins the revoking reasons - the anchored revalidation re-lists the tree at the resolved commit and re-compares the digest instead of probing one file; the unanchored probe is unchanged - a SourceUnavailable confirmed on the refusal staircase writes the anchor as the identity (digestSource Expected, no revision or counts) and the artifact never contacts the hub again - the hub interface grows the manifest-returning ListManifest both clients already serve - mutations prove the comparison and the anchored revalidation tests go red Task 2 of model-artifact-expected-digest. Signed-off-by: thxCode --- pkg/modelartifact/client.go | 5 + .../controllers/worker/model_artifact.go | 82 ++++++- .../controllers/worker/model_artifact_test.go | 206 +++++++++++++++++- 3 files changed, 273 insertions(+), 20 deletions(-) diff --git a/pkg/modelartifact/client.go b/pkg/modelartifact/client.go index 80e8d8bf5..6edb92f2e 100644 --- a/pkg/modelartifact/client.go +++ b/pkg/modelartifact/client.go @@ -25,6 +25,11 @@ const ( // ReasonSourceUnavailable is a failure to get an answer at all, which says nothing about the // artifact and is retried. ReasonSourceUnavailable = "SourceUnavailable" + + // ReasonDigestMismatch is a source whose content digests to something other than the + // artifact's expectedDigest: the assertion the user made and the content the source + // serves disagree. + ReasonDigestMismatch = "DigestMismatch" // ReasonInvalidManifest is an answer that violates the manifest format. ReasonInvalidManifest = "InvalidManifest" // ReasonEmptyManifest is a commit with no files. diff --git a/pkg/worker/controllers/worker/model_artifact.go b/pkg/worker/controllers/worker/model_artifact.go index 29b0707ee..12ca79ca7 100644 --- a/pkg/worker/controllers/worker/model_artifact.go +++ b/pkg/worker/controllers/worker/model_artifact.go @@ -95,6 +95,7 @@ var _ ctrlreconcile.Reconciler = (*ModelArtifactReconciler)(nil) // staircase; only the endpoints and the reason classifier differ, and those live in the clients. type modelArtifactHub interface { Resolve(ctx context.Context, repository, revision, token string, filter modelartifact.Filter) (modelartifact.Resolution, error) + ListManifest(ctx context.Context, repository, commit, token string, filter modelartifact.Filter) (modelartifact.Manifest, error) Revalidate(ctx context.Context, repository, commit, token string) error ValidToken(ctx context.Context, token string) (bool, error) } @@ -358,7 +359,7 @@ func (r *ModelArtifactReconciler) reconcileHubSource( var after time.Duration if ma.Status.Resolved == nil { - after = r.resolveHubSource(ctx, ma, hub, source, token) + after = r.resolveHubSource(ctx, ma, hub, source, token, now) } else { after = r.revalidateHubSource(ctx, ma, hub, source, token, now, secretChanged) } @@ -369,29 +370,68 @@ func (r *ModelArtifactReconciler) reconcileHubSource( // resolveHubSource resolves the source once and reports when the source is to be asked next. func (r *ModelArtifactReconciler) resolveHubSource( ctx context.Context, ma *workercore.ModelArtifact, hub modelArtifactHub, source *workercore.ModelArtifactHubSource, - token string, + token string, now time.Time, ) time.Duration { resolution, err := hub.Resolve(ctx, source.Repository, source.Revision, token, modelartifact.Filter{ Allow: ma.Spec.AllowPatterns, Ignore: ma.Spec.IgnorePatterns, }) if err != nil { reason := modelartifact.ReasonOf(err) + // An anchor turns a confirmed unavailability into an identity: one blip is not a verdict, + // the same discipline a revocation follows, but a hub that could not be reached twice in a + // row never said no, so the anchor the user asserted becomes the resolution. A hub that + // answered — with content or with a refusal — is a hub whose answer decides. + if reason == modelartifact.ReasonSourceUnavailable && ma.Spec.ExpectedDigest != "" && + ModelArtifactConditionDegraded.IsTrue(ma) && ModelArtifactConditionDegraded.GetReason(ma) == reason { + if wait := modelArtifactConditionSince(ma, ModelArtifactConditionDegraded).Add(modelArtifactConfirmDelay).Sub(now); wait > 0 { + return wait + } + ma.Status.Resolved = &workercore.ModelArtifactResolved{ + ManifestDigest: ma.Spec.ExpectedDigest, + DigestSource: workercore.ModelArtifactDigestSourceExpected, + ResolvedTime: meta.NewTime(r.now()), + } + ModelArtifactConditionResolved.True(ma, modelArtifactReasonResolved, + fmt.Sprintf("resolved to the expected digest %s without the hub; the hub is not contacted again", ma.Spec.ExpectedDigest)) + ModelArtifactConditionDegraded.False(ma, modelArtifactReasonHealthy, "") + + return r.revalidateInterval(ctx) + } ModelArtifactConditionResolved.False(ma, reason, err.Error()) ModelArtifactConditionDegraded.True(ma, reason, err.Error()) if reason == modelartifact.ReasonSourceUnavailable { + // Stamped with this pass's time, as the revalidation staircase does: the anchor's + // confirm cadence runs from the outage this pass saw, not from a condition another + // failure left behind. + ModelArtifactConditionDegraded.LastTransitionTime(ma, now.UTC().Format(time.RFC3339)) return modelArtifactUnavailableRetry } return modelArtifactRefusedRetry } - now := meta.NewTime(r.now()) + // The anchor is an assertion about this very answer: a hub that serves different content than + // the user pinned is refused, both digests named for the ticket. + if anchor := ma.Spec.ExpectedDigest; anchor != "" && resolution.Manifest.Digest != anchor { + err = &modelartifact.SourceError{ + Reason: modelartifact.ReasonDigestMismatch, + Message: fmt.Sprintf("the artifact expects %s, the hub resolved %s@%s to %s", + anchor, source.Repository, source.Revision, resolution.Manifest.Digest), + } + ModelArtifactConditionResolved.False(ma, modelartifact.ReasonDigestMismatch, err.Error()) + ModelArtifactConditionDegraded.True(ma, modelartifact.ReasonDigestMismatch, err.Error()) + + return modelArtifactRefusedRetry + } + + resolvedAt := meta.NewTime(r.now()) ma.Status.Resolved = &workercore.ModelArtifactResolved{ Revision: resolution.Commit, ManifestDigest: resolution.Manifest.Digest, + DigestSource: workercore.ModelArtifactDigestSourceHub, FileCount: resolution.Manifest.FileCount, SizeBytes: resolution.Manifest.SizeBytes, - ResolvedTime: now, - LastValidatedTime: &now, + ResolvedTime: resolvedAt, + LastValidatedTime: &resolvedAt, } ModelArtifactConditionResolved.True(ma, modelArtifactReasonResolved, fmt.Sprintf("%s@%s resolved to commit %s", source.Repository, source.Revision, resolution.Commit)) @@ -412,6 +452,11 @@ func (r *ModelArtifactReconciler) revalidateHubSource( token string, now time.Time, secretChanged bool, ) time.Duration { interval := r.revalidateInterval(ctx) + if ma.Status.Resolved.DigestSource == workercore.ModelArtifactDigestSourceExpected { + // An anchored identity has no commit to probe and no hub assumed to exist; the pacing + // entry alone keeps the reconciler from spinning. + return interval + } if last := ma.Status.Resolved.LastValidatedTime; last != nil && ModelArtifactConditionResolved.IsTrue(ma) && !ModelArtifactConditionDegraded.IsTrue(ma) && now.Before(last.Add(interval)) { // Nothing asks for a check yet: the pacing entry was lost, or the Secret changed back. @@ -420,7 +465,26 @@ func (r *ModelArtifactReconciler) revalidateHubSource( } } - err := hub.Revalidate(ctx, source.Repository, ma.Status.Resolved.Revision, token) + // The anchor makes the periodic check an assertion: the tree is re-listed at the resolved + // commit and re-canonicalized, so a hub or a mirror that stops serving the pinned content + // fails the same way a resolution would. An unanchored artifact keeps the two-request probe. + var err error + if anchor := ma.Spec.ExpectedDigest; anchor != "" { + manifest, listErr := hub.ListManifest(ctx, source.Repository, ma.Status.Resolved.Revision, token, + modelartifact.Filter{Allow: ma.Spec.AllowPatterns, Ignore: ma.Spec.IgnorePatterns}) + switch { + case listErr != nil: + err = listErr + case manifest.Digest != anchor: + err = &modelartifact.SourceError{ + Reason: modelartifact.ReasonDigestMismatch, + Message: fmt.Sprintf("the artifact expects %s, the hub resolves %s@%s to %s", + anchor, source.Repository, ma.Status.Resolved.Revision, manifest.Digest), + } + } + } else { + err = hub.Revalidate(ctx, source.Repository, ma.Status.Resolved.Revision, token) + } if err == nil { validated := meta.NewTime(now) ma.Status.Resolved.LastValidatedTime = &validated @@ -459,9 +523,11 @@ func (r *ModelArtifactReconciler) revalidateHubSource( } // modelArtifactRevokingReason reports whether a failed access check can revoke: a refusal can, a -// failure to reach the source cannot. +// failure to reach the source cannot. A digest mismatch can, because the hub answered — its +// answer is just not the content the artifact asserts. func modelArtifactRevokingReason(reason string) bool { - return reason == modelartifact.ReasonAccessDenied || reason == modelartifact.ReasonRevisionNotFound + return reason == modelartifact.ReasonAccessDenied || reason == modelartifact.ReasonRevisionNotFound || + reason == modelartifact.ReasonDigestMismatch } func modelArtifactConditionSince(ma *workercore.ModelArtifact, c kubeapistatus.ConditionType) time.Time { diff --git a/pkg/worker/controllers/worker/model_artifact_test.go b/pkg/worker/controllers/worker/model_artifact_test.go index 9204bba4e..e15913d7b 100644 --- a/pkg/worker/controllers/worker/model_artifact_test.go +++ b/pkg/worker/controllers/worker/model_artifact_test.go @@ -173,18 +173,21 @@ func testModelScopeArtifact(secret string) *workercore.ModelArtifact { // testModelScopeHub is a fake ModelScope client: every answer swappable, every ask recorded. type testModelScopeHub struct { - mu sync.Mutex - resolution modelartifact.Resolution - resolveErr error - revalidate []error // one per Revalidate ask, the last repeating - tokenValid bool - tokenErr error - asks int - lastRepo string - lastRev string - lastToken string - lastFilter modelartifact.Filter - lastCommit string + mu sync.Mutex + resolution modelartifact.Resolution + resolveErr error + revalidate []error // one per Revalidate ask, the last repeating + list []modelartifact.Manifest // one per ListManifest ask, the last repeating + tokenValid bool + tokenErr error + asks int // every hub ask, however it is answered + lists int + revalidates int + lastRepo string + lastRev string + lastToken string + lastFilter modelartifact.Filter + lastCommit string } func (h *testModelScopeHub) Resolve(_ context.Context, repository, revision, token string, filter modelartifact.Filter) (modelartifact.Resolution, error) { @@ -203,6 +206,7 @@ func (h *testModelScopeHub) Revalidate(_ context.Context, repository, commit, to h.mu.Lock() defer h.mu.Unlock() h.asks++ + h.revalidates++ h.lastRepo, h.lastCommit, h.lastToken = repository, commit, token if len(h.revalidate) == 0 { return nil @@ -215,6 +219,23 @@ func (h *testModelScopeHub) Revalidate(_ context.Context, repository, commit, to return err } +func (h *testModelScopeHub) ListManifest(_ context.Context, repository, commit, token string, filter modelartifact.Filter) (modelartifact.Manifest, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.asks++ + h.lists++ + h.lastRepo, h.lastCommit, h.lastToken, h.lastFilter = repository, commit, token, filter + if len(h.list) == 0 { + return h.resolution.Manifest, nil + } + manifest := h.list[0] + if len(h.list) > 1 { + h.list = h.list[1:] + } + + return manifest, nil +} + func (h *testModelScopeHub) ValidToken(_ context.Context, _ string) (bool, error) { return h.tokenValid, h.tokenErr } @@ -967,3 +988,164 @@ func TestModelArtifactReconcileImage(t *testing.T) { assert.Equal(t, "Resolved", ModelArtifactConditionResolved.GetReason(got)) assert.Equal(t, "False", ModelArtifactConditionDegraded.GetStatus(got)) } + +// testAnchoredModelScopeArtifact is a ModelScope artifact carrying the anchor, with a Secret +// reference when one is named. +func testAnchoredModelScopeArtifact(digest, secret string) *workercore.ModelArtifact { + ma := testModelScopeArtifact(secret) + ma.Spec.ExpectedDigest = digest + + return ma +} + +func testManifest(path, hash string, size int64) (modelartifact.Manifest, error) { + return modelartifact.NewManifest([]modelartifact.ManifestEntry{ + {Path: path, Size: size, Digest: modelartifact.DigestSHA256 + ":" + hash}, + }) +} + +// TestModelArtifactReconcileAssertsTheExpectedDigest walks the anchor: a resolution whose digest +// matches stands; one that does not is refused with both digests and revokes on the staircase; and +// a confirmed outage writes the anchor as the identity, which asks the hub for nothing ever again. +func TestModelArtifactReconcileAssertsTheExpectedDigest(t *testing.T) { + good, err := testManifest("README.md", strings.Repeat("a", 64), 3) + require.NoError(t, err) + drifted, err := testManifest("README.md", strings.Repeat("f", 64), 3) + require.NoError(t, err) + + t.Run("a resolution matching the anchor stands", func(t *testing.T) { + hub := &testModelScopeHub{resolution: modelartifact.Resolution{Commit: testArtifactCommit, Manifest: good}} + env := newTestModelScopeEnv(t, hub, testAnchoredModelScopeArtifact(good.Digest, "")) + + ma, _ := env.reconcile(t, "qwen") + assert.True(t, ModelArtifactConditionResolved.IsTrue(ma)) + require.NotNil(t, ma.Status.Resolved) + assert.Equal(t, good.Digest, ma.Status.Resolved.ManifestDigest) + assert.Equal(t, testArtifactCommit, ma.Status.Resolved.Revision) + assert.Equal(t, workercore.ModelArtifactDigestSourceHub, ma.Status.Resolved.DigestSource) + }) + t.Run("a resolution missing the anchor is refused with both digests", func(t *testing.T) { + hub := &testModelScopeHub{resolution: modelartifact.Resolution{Commit: testArtifactCommit, Manifest: good}} + env := newTestModelScopeEnv(t, hub, testAnchoredModelScopeArtifact(drifted.Digest, "")) + + ma, _ := env.reconcile(t, "qwen") + assert.False(t, ModelArtifactConditionResolved.IsTrue(ma)) + assert.Equal(t, modelartifact.ReasonDigestMismatch, ModelArtifactConditionResolved.GetReason(ma)) + assert.Nil(t, ma.Status.Resolved, "a refused resolution writes no identity") + assert.True(t, ModelArtifactConditionDegraded.IsTrue(ma)) + message := ModelArtifactConditionResolved.GetMessage(ma) + assert.Contains(t, message, drifted.Digest) + assert.Contains(t, message, good.Digest) + }) + t.Run("a drift at the resolved commit walks the staircase", func(t *testing.T) { + hub := &testModelScopeHub{ + resolution: modelartifact.Resolution{Commit: testArtifactCommit, Manifest: good}, + list: []modelartifact.Manifest{drifted, drifted}, + } + env := newTestModelScopeEnv(t, hub, testAnchoredModelScopeArtifact(good.Digest, "")) + resolved, _ := env.reconcile(t, "qwen") + require.True(t, ModelArtifactConditionResolved.IsTrue(resolved)) + + day := 24 * time.Hour + env.clock.now = env.clock.now.Add(day) + first, _ := env.reconcile(t, "qwen") + assert.True(t, ModelArtifactConditionResolved.IsTrue(first), "the first mismatch degrades, not revokes") + assert.Equal(t, modelartifact.ReasonDigestMismatch, ModelArtifactConditionDegraded.GetReason(first)) + hub.mu.Lock() + assert.Equal(t, testArtifactCommit, hub.lastCommit, "the revalidation lists at the resolved commit") + assert.Equal(t, 1, hub.lists, "the anchored revalidation is one listing") + assert.Equal(t, 0, hub.revalidates, "the anchored revalidation is not the HEAD probe") + hub.mu.Unlock() + + env.clock.now = env.clock.now.Add(10 * time.Second) + env.reconcile(t, "qwen") + env.clock.now = env.clock.now.Add(time.Minute) + revoked, _ := env.reconcile(t, "qwen") + assert.False(t, ModelArtifactConditionResolved.IsTrue(revoked), "the confirmed mismatch revokes") + assert.Equal(t, modelartifact.ReasonDigestMismatch, ModelArtifactConditionResolved.GetReason(revoked)) + require.NotNil(t, revoked.Status.Resolved) + assert.Equal(t, testArtifactCommit, revoked.Status.Resolved.Revision, "the identity stays for audit") + }) + t.Run("a confirmed outage resolves to the anchor without the hub", func(t *testing.T) { + outage := &modelartifact.SourceError{Reason: modelartifact.ReasonSourceUnavailable, Message: "HTTP 503"} + hub := &testModelScopeHub{resolveErr: outage} + env := newTestModelScopeEnv(t, hub, testAnchoredModelScopeArtifact(good.Digest, "")) + + env.reconcile(t, "qwen") + got := mustGet(t, env.cli) + assert.False(t, ModelArtifactConditionResolved.IsTrue(got), "one blip is not a verdict") + assert.Nil(t, got.Status.Resolved) + + env.clock.now = env.clock.now.Add(2 * modelArtifactUnavailableRetry) + env.reconcile(t, "qwen") + got = mustGet(t, env.cli) + assert.True(t, ModelArtifactConditionResolved.IsTrue(got), "the confirmed outage anchors the identity") + require.NotNil(t, got.Status.Resolved) + assert.Equal(t, good.Digest, got.Status.Resolved.ManifestDigest) + assert.Equal(t, workercore.ModelArtifactDigestSourceExpected, got.Status.Resolved.DigestSource) + assert.Empty(t, got.Status.Resolved.Revision, "an Expected identity has no commit") + assert.Zero(t, got.Status.Resolved.FileCount) + assert.Zero(t, got.Status.Resolved.SizeBytes) + + hub.mu.Lock() + asksBefore := hub.asks + hub.mu.Unlock() + env.clock.now = env.clock.now.Add(25 * time.Hour) + env.reconcile(t, "qwen") + hub.mu.Lock() + defer hub.mu.Unlock() + assert.Equal(t, asksBefore, hub.asks, "an Expected artifact asks the hub for nothing, ever") + }) + t.Run("a hub that answered is never anchored around", func(t *testing.T) { + refused := &modelartifact.SourceError{Reason: modelartifact.ReasonAccessDenied, Message: "private"} + hub := &testModelScopeHub{resolveErr: refused} + env := newTestModelScopeEnv(t, hub, testAnchoredModelScopeArtifact(good.Digest, "")) + + env.reconcile(t, "qwen") + env.clock.now = env.clock.now.Add(2 * modelArtifactUnavailableRetry) + env.reconcile(t, "qwen") + + ma := new(workercore.ModelArtifact) + require.NoError(t, env.cli.Get(context.Background(), + ctrlcli.ObjectKey{Namespace: "team-a", Name: "qwen"}, ma)) + assert.False(t, ModelArtifactConditionResolved.IsTrue(ma)) + assert.Equal(t, modelartifact.ReasonAccessDenied, ModelArtifactConditionResolved.GetReason(ma)) + assert.Nil(t, ma.Status.Resolved, "an answered hub's refusal is the verdict") + }) + t.Run("a single blip does not freeze the identity", func(t *testing.T) { + outage := &modelartifact.SourceError{Reason: modelartifact.ReasonSourceUnavailable, Message: "HTTP 503"} + hub := &testModelScopeHub{resolveErr: outage} + env := newTestModelScopeEnv(t, hub, testAnchoredModelScopeArtifact(good.Digest, "")) + + env.reconcile(t, "qwen") + hub.mu.Lock() + hub.resolveErr = nil + hub.resolution = modelartifact.Resolution{Commit: testArtifactCommit, Manifest: good} + hub.mu.Unlock() + env.clock.now = env.clock.now.Add(2 * modelArtifactUnavailableRetry) + ma, _ := env.reconcile(t, "qwen") + + assert.True(t, ModelArtifactConditionResolved.IsTrue(ma)) + require.NotNil(t, ma.Status.Resolved) + assert.Equal(t, testArtifactCommit, ma.Status.Resolved.Revision, "the hub answered, so the hub decides") + assert.Equal(t, workercore.ModelArtifactDigestSourceHub, ma.Status.Resolved.DigestSource) + }) + t.Run("an unanchored artifact revalidates with the HEAD probe, not a listing", func(t *testing.T) { + hub := &testModelScopeHub{resolution: modelartifact.Resolution{Commit: testArtifactCommit, Manifest: good}} + env := newTestModelScopeEnv(t, hub, testModelScopeArtifact("")) + env.reconcile(t, "qwen") + + env.clock.now = env.clock.now.Add(25 * time.Hour) + env.reconcile(t, "qwen") + assert.True(t, ModelArtifactConditionResolved.IsTrue(mustGet(t, env.cli))) + }) +} + +func mustGet(t *testing.T, cli ctrlcli.Client) *workercore.ModelArtifact { + t.Helper() + ma := new(workercore.ModelArtifact) + require.NoError(t, cli.Get(context.Background(), + ctrlcli.ObjectKey{Namespace: "team-a", Name: "qwen"}, ma)) + + return ma +} From e48a5226563e41ab42ca8424d7d90bce5edc7c85 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 21:29:27 +0800 Subject: [PATCH 03/10] feat(worker): block engine delivery of an anchored artifact, peers-only chain - an anchored artifact under Engine delivery waits with AnchorNeedsNodeDelivery, the message naming why: an engine downloads by repository and revision and cannot anchor-verify what it fetched, and an Expected identity has no commit - an Expected identity's node chain is peers only: the manifest comes from a peer's published listing bound to the digest (Puller.FetchManifest), and with no peer holding the tree the mount fails naming the digest nothing holds - the hub-identity chain is unchanged, still listing anchor-checked before any byte moves Task 3 of model-artifact-expected-digest. Signed-off-by: thxCode --- pkg/modelmanager/materialize/materialize.go | 47 ++++++++++++++ .../materialize/materialize_test.go | 26 ++++++++ pkg/modelmanager/peer/client.go | 35 ++++++++++ pkg/modelmanager/peer/client_test.go | 20 ++++++ .../worker/model_artifact_placement.go | 15 +++++ .../worker/model_artifact_placement_test.go | 64 +++++++++++++++++++ 6 files changed, 207 insertions(+) diff --git a/pkg/modelmanager/materialize/materialize.go b/pkg/modelmanager/materialize/materialize.go index 56228102b..712877453 100644 --- a/pkg/modelmanager/materialize/materialize.go +++ b/pkg/modelmanager/materialize/materialize.go @@ -361,6 +361,12 @@ func (m *Materializer) attempt(ctx context.Context, j *job) error { var lastErr error for _, src := range j.snapshotSources() { + // An Expected identity has no commit to list at and no hub to ask: the peers holding the + // published tree are the only source there is. + if src.commit == "" { + lastErr = m.fromPeersOnly(ctx, j, a, src) + break + } hub, dl, err := m.Environment(ctx, src.kind) if err != nil { // One kind's configuration failure says nothing about another kind's: try the @@ -382,6 +388,35 @@ func (m *Materializer) attempt(ctx context.Context, j *job) error { return nil } +// errNoSourceForAnchored is the loud failure an Expected identity mounts with when no peer holds +// the tree: the digest is named in the tenant-free detail, and the way out is in the plugin's log. +func errNoSourceForAnchored(hex string) *download.Error { + return &download.Error{ + Reason: download.ReasonSourceUnavailable, + Message: fmt.Sprintf("no node holds sha256:%s and the artifact is anchored to its digest without "+ + "a hub: seed one node while the hub is reachable, or have a peer serve the tree", hex), + Detail: "no peer holds sha256:" + hex, + } +} + +// fromPeersOnly materializes an Expected identity from the peers alone: the manifest comes from a +// peer's published listing, bound to the digest before it is trusted, and every byte comes from a +// peer — there is no hub to fall back to. +func (m *Materializer) fromPeersOnly(ctx context.Context, j *job, a *store.Attempt, src *source) error { + if !m.Peers.Enabled() { + noSource := errNoSourceForAnchored(j.hex) + noSource.Detail += ", and peer pulling is not configured" + + return noSource + } + manifest, err := m.Peers.FetchManifest(ctx, "sha256:"+j.hex) + if err != nil { + return errNoSourceForAnchored(j.hex) + } + + return m.fetchTree(ctx, j, a, manifest, nil, nil, src) +} + func (m *Materializer) fromSource( ctx context.Context, j *job, a *store.Attempt, hub Hub, dl *download.Downloader, src *source, ) error { @@ -394,6 +429,13 @@ func (m *Materializer) fromSource( "the hub lists %s at commit %s as %s, the artifact's digest is sha256:%s", src.repository, src.commit, manifest.Digest, j.hex), Detail: "the hub lists " + manifest.Digest} } + + return m.fetchTree(ctx, j, a, manifest, hub, dl, src) +} + +func (m *Materializer) fetchTree( + ctx context.Context, j *job, a *store.Attempt, manifest modelartifact.Manifest, hub Hub, dl *download.Downloader, src *source, +) error { j.total.Store(manifest.SizeBytes) stateDir := filepath.Join(a.Dir(), "state") @@ -455,6 +497,11 @@ func (m *Materializer) fromSource( if m.pullFromPeers(gctx, j, manifestDigest, f) { return nil } + // An Expected identity's fallback is not the hub — there is none; the file is on no + // peer, which is the no-source failure, not an integrity one. + if hub == nil { + return errNoSourceForAnchored(j.hex) + } return dl.Fetch(gctx, f) }) diff --git a/pkg/modelmanager/materialize/materialize_test.go b/pkg/modelmanager/materialize/materialize_test.go index 079c1bac8..76dd278bd 100644 --- a/pkg/modelmanager/materialize/materialize_test.go +++ b/pkg/modelmanager/materialize/materialize_test.go @@ -195,6 +195,32 @@ func (e *testEnv) request(repo, uid, token string) driver.Request { return driver.Request{Hex: store.HexOf(digest), Artifact: e.artifact(repo, uid, digest), Token: token} } +// TestEnsureExpectedIdentityIsPeersOnly pins an Expected identity's chain: no commit exists to list +// at, so the hub is never asked — a peer holding the published tree is the only source, and without +// one the mount fails loudly naming the missing source, not as a hub error dressed as an integrity +// one. +func TestEnsureExpectedIdentityIsPeersOnly(t *testing.T) { + env := newTestEnv(t, repoFiles()) + digest := env.hub.manifest("owner/repo").Digest + ma := env.artifact("owner/repo", "uid-a", digest) + ma.Status.Resolved.Revision = "" + ma.Status.Resolved.DigestSource = workercore.ModelArtifactDigestSourceExpected + req := driver.Request{Hex: store.HexOf(digest), Artifact: ma} + + env.m.Ensure(context.Background(), req) + env.waitIdle(t, req.Hex) + + assert.Empty(t, env.hub.recorded(), "an Expected identity asks the hub for nothing") + require.False(t, env.store.IsPublished(req.Hex)) + rec, err := env.store.ReadDigest(req.Hex) + require.NoError(t, err) + assert.Equal(t, download.ReasonSourceUnavailable, rec.Reason) + // The ledger carries the tenant-free public message: the reason's meaning and the digest + // nothing holds. The full wording, the anchoring and the way out, is the plugin log's. + assert.Contains(t, rec.Message, "sha256:"+req.Hex) + assert.Contains(t, rec.Message, "peer pulling is not configured") +} + // waitIdle waits until no attempt runs for hex. func (e *testEnv) waitIdle(t *testing.T, hex string) { t.Helper() diff --git a/pkg/modelmanager/peer/client.go b/pkg/modelmanager/peer/client.go index 10a66922f..e77565e4c 100644 --- a/pkg/modelmanager/peer/client.go +++ b/pkg/modelmanager/peer/client.go @@ -94,6 +94,41 @@ func (p *Puller) FetchFile(ctx context.Context, digest string, file download.Fil return fmt.Errorf("%s: %w", file.Path, lastErr) } +// FetchManifest returns the tree's manifest from a peer that holds it published, reassembled +// into the canonical manifest and bound to the digest before it is trusted. A caller with no +// other way to the manifest — an artifact anchored to its digest without a hub — asks this +// first; a failure here is the loud no-source failure, not a hub error to dress up. +func (p *Puller) FetchManifest(ctx context.Context, digest string) (modelartifact.Manifest, error) { + treeHex, ok := HexOf(digest) + if !ok { + return modelartifact.Manifest{}, fmt.Errorf("digest %q is not a manifest digest", digest) + } + sources, err := p.Discover(ctx, digest) + if err != nil { + return modelartifact.Manifest{}, err + } + var lastErr error + for _, src := range sources { + if p.sitsOut(src) { + continue + } + manifest, err := p.fetchListing(ctx, src, treeHex) + if err == nil { + return manifest, nil + } + if ctx.Err() != nil { + return modelartifact.Manifest{}, context.Cause(ctx) + } + p.setAside(src) + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("no candidate was usable") + } + + return modelartifact.Manifest{}, fmt.Errorf("no node holds %s: %w", digest, lastErr) +} + // fetchFrom pulls the whole file from one source: its listing is reassembled into a canonical // manifest and bound to the tree's digest before a byte is trusted, then the file's ranges are // fetched and hashed in byte order, checkpointing as they go. diff --git a/pkg/modelmanager/peer/client_test.go b/pkg/modelmanager/peer/client_test.go index 7df80199e..97b7def6a 100644 --- a/pkg/modelmanager/peer/client_test.go +++ b/pkg/modelmanager/peer/client_test.go @@ -137,6 +137,26 @@ func TestPullerFailsWithoutCandidates(t *testing.T) { assert.Contains(t, err.Error(), "no candidate was usable") } +func TestPullerFetchManifest(t *testing.T) { + ps := newPeerServer(t, strings.Repeat("abcdef", 100)) + source := ps.source(t, "peer-1") + p := &Puller{Discover: func(context.Context, string) ([]*Source, error) { + return []*Source{source}, nil + }} + manifest, err := p.FetchManifest(context.Background(), "sha256:"+ps.hex) + require.NoError(t, err) + assert.Equal(t, "sha256:"+ps.hex, manifest.Digest, "the listing is bound to the tree's digest") + require.Len(t, manifest.Entries, 1) + assert.Equal(t, "weights.bin", manifest.Entries[0].Path) + assert.Equal(t, int64(len(ps.content)), manifest.Entries[0].Size) + + dead := newPeerServer(t, strings.Repeat("abcdef", 100)) + dead.Close() + p = &Puller{Discover: func(context.Context, string) ([]*Source, error) { return nil, nil }} + _, err = p.FetchManifest(context.Background(), "sha256:"+dead.hex) + require.Error(t, err, "no peer holding the tree is the loud no-source failure") +} + func TestPullerFailsWithoutADigest(t *testing.T) { ps := newPeerServer(t, "0123456789") p := &Puller{} diff --git a/pkg/worker/controllers/worker/model_artifact_placement.go b/pkg/worker/controllers/worker/model_artifact_placement.go index 6d457a8eb..4d97ffaec 100644 --- a/pkg/worker/controllers/worker/model_artifact_placement.go +++ b/pkg/worker/controllers/worker/model_artifact_placement.go @@ -43,6 +43,7 @@ const ( modelWeightsReasonDownloaded = "Downloaded" modelWeightsReasonNodeUnavailable = "NodeDeliveryUnavailable" modelWeightsReasonFilterNeedsNode = "FilterNeedsNodeDelivery" + modelWeightsReasonAnchorNeedsNode = "AnchorNeedsNodeDelivery" modelWeightsReasonMaterializing = "Materializing" modelWeightsReasonMaterializeFail = "MaterializationFailed" @@ -203,6 +204,20 @@ func resolveModelArtifactWeights( w.Blocked, w.Reason = true, modelWeightsReasonFilterNeedsNode w.Message = fmt.Sprintf("ModelArtifact %q selects files with allow or ignore patterns, which only node "+ "delivery honors: set model-artifact-delivery-mode to Node", name) + break + } + // An anchor is an assertion the engine cannot keep: it downloads by repository and revision + // and cannot verify what it fetched against the digest, and an Expected identity has no + // commit to pin the revision with at all. + if ma.Spec.ExpectedDigest != "" { + w.Blocked, w.Reason = true, modelWeightsReasonAnchorNeedsNode + message := fmt.Sprintf("ModelArtifact %q carries expectedDigest, and an engine downloads by "+ + "repository and revision, so it cannot anchor-verify what it fetched: set "+ + "model-artifact-delivery-mode to Node", name) + if resolved.DigestSource == workercore.ModelArtifactDigestSourceExpected { + message += "; the artifact resolved to its anchor without the hub and has no resolved commit" + } + w.Message = message } } diff --git a/pkg/worker/controllers/worker/model_artifact_placement_test.go b/pkg/worker/controllers/worker/model_artifact_placement_test.go index 3e1f7b8eb..6e7b4017c 100644 --- a/pkg/worker/controllers/worker/model_artifact_placement_test.go +++ b/pkg/worker/controllers/worker/model_artifact_placement_test.go @@ -268,6 +268,70 @@ func TestModelDeploymentArtifactNodeDelivery(t *testing.T) { } } +// TestModelDeploymentArtifactAnchorNeedsNodeDelivery pins that an anchored artifact never delivers +// through an engine: the engine downloads by repository and revision and cannot anchor-verify what +// it fetched, and an Expected identity has no commit to pin at all. +func TestModelDeploymentArtifactAnchorNeedsNodeDelivery(t *testing.T) { + anchored := func() *workercore.ModelArtifact { + ma := artifactFixture("", true, true) + ma.Spec.ExpectedDigest = testArtifactDigest + + return ma + } + expected := func() *workercore.ModelArtifact { + ma := anchored() + ma.Status.Resolved.Revision = "" + ma.Status.Resolved.DigestSource = workercore.ModelArtifactDigestSourceExpected + + return ma + } + driver := &storage.CSIDriver{ObjectMeta: meta.ObjectMeta{Name: "model.csi.gpustack.ai"}} + cases := []struct { + name string + delivery string + artifact func() *workercore.ModelArtifact + driver bool + wantPods int + wantReason string + wantInMsg string + }{ + { + name: "a hub-resolved anchor under Engine delivery creates nothing", delivery: "Engine", + artifact: anchored, wantReason: "AnchorNeedsNodeDelivery", wantInMsg: "cannot anchor-verify", + }, + { + name: "an Expected identity under Engine delivery names the missing commit", delivery: "Engine", + artifact: expected, wantReason: "AnchorNeedsNodeDelivery", wantInMsg: "no resolved commit", + }, + { + name: "a hub-resolved anchor under Node delivery creates the replica", delivery: "Node", + artifact: anchored, driver: true, wantPods: 1, wantReason: "WeightsNotMounted", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + orig := modelArtifactDeliveryMode + t.Cleanup(func() { modelArtifactDeliveryMode = orig }) + modelArtifactDeliveryMode = func(context.Context) string { return c.delivery } + objs := []ctrlcli.Object{artifactDeploymentFixture(1), newRenderInstanceType(), c.artifact()} + if c.driver { + objs = append(objs, driver) + } + cli := newModelDeploymentClient(objs...) + + _, err := reconcileModelDeployment(t, cli) + require.NoError(t, err) + + assert.Len(t, listReplicas(t, cli), c.wantPods) + md := getModelDeployment(t, cli) + assert.Equal(t, c.wantReason, ModelDeploymentConditionWeightsReady.GetReason(md)) + if c.wantInMsg != "" { + assert.Contains(t, ModelDeploymentConditionWeightsReady.GetMessage(md), c.wantInMsg) + } + }) + } +} + // TestModelDeploymentArtifactDeliverySwitchRolls pins that switching the delivery Setting changes the // replicas' fingerprint once: the render differs, so each replica is replaced like an image change. func TestModelDeploymentArtifactDeliverySwitchRolls(t *testing.T) { From cf6dfb4009de65259658f287fed5043703fb9c60 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 21:34:04 +0800 Subject: [PATCH 04/10] docs(model-store): document the expectedDigest anchor and its delivery - artifact.md: the expectedDigest field and its hub-only admission, the anchored resolution and revalidation states, digestSource, the confirmed SourceUnavailable fallback and what an Expected identity delivers, the shipped access model for Expected identities, the AnchorNeedsNodeDelivery row, the migration contract, and the downgrade-window note - node-store.md: the Expected identity's peers-only manifest and the SourceUnavailable row's anchored case - README index: the artifact page's description carries the anchor Task 4 of model-artifact-expected-digest. Signed-off-by: thxCode --- docs/README.md | 2 +- docs/model-store/artifact.md | 49 ++++++++++++++++++++++++++++++++++ docs/model-store/node-store.md | 8 ++++-- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 498093514..82ad34656 100644 --- a/docs/README.md +++ b/docs/README.md @@ -73,7 +73,7 @@ Everything written about GPUStack Operator, and the order to read it in. Start a | [KV Cache on Disk-Heavy Nodes](kv-cache/disk-heavy-nodes.md) | What to configure on a node whose capacity is disk rather than memory: why no member group can be disk alone, and how small its memory segment may be | operators | ~5 min | | [KV Cache Pool](kv-cache/pool.md) | How a namespace is granted a quota on a store, what a quota ceiling buys, and why a full quota discards data instead of refusing writes | operators, contributors | ~12 min | | [KV Cache Walkthrough](kv-cache/walkthrough.md) | The shortest path from nothing to a ModelDeployment on a shared cache: the four objects in order, the pasteable manifests, and the three things that go wrong | operators, users | ~11 min | -| [Model Artifact](model-store/artifact.md) | How a `ModelArtifact` names a model's weights, how it is resolved and revalidated, the manifest digest, file patterns, and how a `ModelDeployment` or an `Instance` mounts or downloads them under each delivery | users, operators | ~16 min | +| [Model Artifact](model-store/artifact.md) | How a `ModelArtifact` names a model's weights, how it is resolved and revalidated — with an optional `expectedDigest` asserting the manifest digest — how a `ModelDeployment` or an `Instance` mounts or downloads them under each delivery, and what an air-gapped cluster can still deliver | users, operators | ~16 min | | [Model Image Source](model-store/image-source.md) | The `ModelArtifact` image source: the digest-pinned reference and what it does and does not promise, building weights into an image, image-volume delivery to a `ModelDeployment` or an `Instance`, the version floors, double storage, kubelet image GC, and registry mirrors | users, operators | ~8 min | | [Model Prefetch](model-store/prefetch.md) | Warming a model onto nodes before any Pod asks: the `ModelStore` / `ModelStoreBinding` / `ModelPrefetch` objects, placement, the warm-up pod and why it carries no queue-name label, budgets, pinning and expiry, and the status views | users, operators | ~9 min | | [Node Model Store](model-store/node-store.md) | The `NodeModelStore` resource and the `model-manager` plugin: every field and its writer, the status guard, mount authorization, how content is downloaded, verified and published, failure reasons, collection and metrics | operators, contributors | ~12 min | diff --git a/docs/model-store/artifact.md b/docs/model-store/artifact.md index d0603d96b..426df6391 100644 --- a/docs/model-store/artifact.md +++ b/docs/model-store/artifact.md @@ -47,10 +47,13 @@ spec: # immutable after creation # reference: registry.example.com/team/qwen@sha256:669ed7b1...48 # digest-pinned allowPatterns: ["*.safetensors", "*.json", "tokenizer*"] # optional; hub sources only ignorePatterns: ["original/"] # optional; wins over allowPatterns + expectedDigest: sha256:669ed7b128b6ad1658d735326bd172a33497ecdb8bbd72dd0b23c98b58469448 + # optional; hub sources only — the digest the source must resolve to status: resolved: revision: 7ae557604adf67be50417f59c2c2f167def9a775 manifestDigest: sha256:669ed7b128b6ad1658d735326bd172a33497ecdb8bbd72dd0b23c98b58469448 + digestSource: Hub # Hub, the hub's listing | Expected, the spec's anchor fileCount: 10 sizeBytes: 999604126 nodes: # hub sources only; where the content is across nodes @@ -81,6 +84,12 @@ status: refused on claim and image sources. A filter that keeps no file is `Resolved=False`, `EmptyManifest`. A filtered artifact needs [Node delivery](#referencing-it-from-a-modeldeployment). +- **`expectedDigest` asserts what must resolve.** Optional, immutable with the spec, `"sha256:"` and + 64 lowercase hex — the manifest digest, exactly what `status.resolved.manifestDigest` + carries. Admission accepts it on a hub source only: a claim's content is whatever the volume + holds at mount time — dynamically provisioned claims differ per provisioning — and its identity is + the claim itself, which the user confirms; an image's identity is its reference's digest. What the + assertion does is under [Resolution and revalidation](#resolution-and-revalidation). - **`status.nodes` counts the content, not the artifact.** Nodes whose `NodeModelStore` lists the digest `Ready`, `Downloading` or `Failed`; artifacts with the same digest see the same nodes, and only numbers cross namespaces. The mean covers the downloading nodes only, each a whole copy. It @@ -148,6 +157,35 @@ The HEAD is `/api/models//resolve//` on Hugging Face and `/api/v1/models//repo?Revision=&FilePath=` on ModelScope, where a 200 — an LFS file's too — confirms and every 404 is an access refusal. +**An `expectedDigest` turns both passes into assertions.** At resolution the digest is compared +with the anchor; a mismatch refuses with `DigestMismatch`, both digests in the message. At +revalidation an anchored artifact does not stop at the one-file `HEAD`: the tree is re-listed at +the resolved commit, so a hub or mirror that stops serving the pinned content fails the same +staircase — first `Degraded`, then `Resolved=False`. Unanchored artifacts keep the two-request check. + +**A hub that cannot be reached, confirmed, hands the identity to the anchor.** One +`SourceUnavailable` is not a verdict; the artifact asks again a minute later. When the second pass +fails the same way, the anchor becomes the resolution: `Resolved=True`, `manifestDigest` = the +anchor, `digestSource: Expected`, no revision, file count or size, and the hub is never contacted +again. A hub that *answered* is never anchored around; only an unanswered hub falls back. + +| `digestSource` | Meaning | +| --- | --- | +| `Hub` | the digest came from the hub's own listing, checked against the anchor when one is set | +| `Expected` | the digest is the spec's anchor, written after the hub's absence was confirmed | + +**What an `Expected` identity delivers.** No commit exists, so [Engine +delivery](#engine-delivery) is refused (`AnchorNeedsNodeDelivery`) and +[Node delivery](node-store.md#materialization) draws the manifest and the bytes from the other +nodes; with no peer holding the tree the mount fails naming the digest nothing holds. Seed one +node while the hub is reachable — the identity stands without it. + +**The shipped access model for `Expected` identities.** Integrity is never at risk: only +anchor-named, verified bytes enter a published set or a mount. What relaxes is secrecy — an +artifact whose anchor names a digest already published on a node can mount that tree with no +credential, and digests are visible cluster-wide on `NodeModelStore` status. Private content must +not rely on digest secrecy; hub-verified identities keep resolving with the namespace's token. + `Resolved=False` stops **new** consumption: no new replica, replacement or scale-up. It never deletes a running Pod. A claim source is resolved by the claim existing; the operator never reads its content, so it has no revision and no digest. @@ -179,6 +217,12 @@ Two consequences to keep in mind: files have the same digest; authorization always comes from resolving with the namespace's token. - The same files on Hugging Face and ModelScope have different digests, because non-LFS files are hashed differently there. Content is not deduplicated across hubs. +- **The anchor is the migration's acceptance contract.** A future import of a legacy GPUStack + cache ([gpustack/gpustack-operator#693](https://github.com/gpustack/gpustack-operator/issues/693), + deferred) may publish only trees whose computed manifest digest equals the artifact's + `expectedDigest`, through the node store's own pipeline, lazily and one-shot — never an online + migration of a running install. This format and that pipeline are what such an import verifies + against. ## Referencing it from a ModelDeployment @@ -343,6 +387,7 @@ being the identity. `WeightsReady` says whether every engine role's weights are | False | `ClaimNotBound`, `AccessModeConflict` | no new Pod is created; see the placement table | | False | `NodeDeliveryUnavailable` | `Node` delivery, and the CSIDriver `model.csi.gpustack.ai` does not exist; no new Pod is created | | False | `FilterNeedsNodeDelivery` | an artifact with patterns under `Engine` delivery, which cannot honor them; no new Pod is created | +| False | `AnchorNeedsNodeDelivery` | an artifact carrying `expectedDigest` under `Engine` delivery: the engine downloads by repository and revision and cannot anchor-verify what it fetched, and an `Expected` identity has no resolved commit to pin; no new Pod is created | | False | `Materializing` | a node Pod is not mounted yet and its node lists the digest `Downloading` | | False | `MaterializationFailed` | the same, and the node lists it `Failed`; the message carries the node's reason and retry time | | False | `WeightsNotMounted` | a claim, node or image Pod's `PodReadyToStartContainers` is not True yet; for an image Pod the Pod's events carry the pull error | @@ -379,6 +424,10 @@ waits instead when that node cannot run one, naming the node and the floor - **Kubernetes 1.29**, the floor the bundled Kueue already sets. `PodReadyToStartContainers` (beta, on by default since 1.29) feeds `WeightsReady`; with it off, a claim deployment's `WeightsReady` stays `WeightsNotMounted` while its replicas run. +- **A downgrade window weakens the anchor's admission.** An old webhook cannot see + `expectedDigest`, so an update changing it is not refused while the old webhook serves, and an + old worker drops `digestSource` from the status it writes. The CRD serves the field throughout, + and re-upgrading recomputes the status on the next pass. - **A ModelScope Engine download needs the runner's ModelScope SDK at 1.39.1 or later**, the version that accepts a commit as the revision. The operator renders the environment whatever the runner holds and cannot see into it: on a runner below the floor the engine fails with the SDK's diff --git a/docs/model-store/node-store.md b/docs/model-store/node-store.md index 24ffab172..ccb96392f 100644 --- a/docs/model-store/node-store.md +++ b/docs/model-store/node-store.md @@ -183,7 +183,11 @@ call after publication mounts. After a download completes, a Pod starts at kubel to about two minutes later. 1. **Manifest.** The tree at `status.resolved.revision` is listed with the mount's credential and - filtered by the artifact's patterns; its canonical digest must equal `manifestDigest`. + filtered by the artifact's patterns; its canonical digest must equal `manifestDigest`. An + artifact whose `digestSource` is `Expected` resolved to its anchor without the hub and has no + commit to list at: the manifest comes from a peer's published listing instead, reassembled and + bound to the digest before it is trusted, and the chain is peers only — there is no hub to fall + back to. 2. **Capacity.** The rest of the manifest's size, together with what every other running download has yet to write, is reserved against the high watermark, collecting first when it does not fit; two downloads that each fit and together do not are never both admitted. The reservation is held @@ -217,7 +221,7 @@ stay for a resume. | --- | --- | --- | | `InvalidRequest` | the configuration cannot be executed: an invalid endpoint, proxy or CA | waits for the configuration to change | | `AccessDenied` | the Hub refused the credential | backoff | -| `SourceUnavailable` | the Hub is unreachable, answers 5xx or 429, or a file's ranges keep failing | backoff | +| `SourceUnavailable` | the Hub is unreachable, answers 5xx or 429, a file's ranges keep failing, or no node holds an anchored artifact's digest | backoff | | `IntegrityMismatch` | a file's hash or size, or the manifest's digest, does not match | the file is discarded; backoff | | `InsufficientCapacity` | the reservation does not fit under the high watermark after collection | backoff | | `Canceled` | no mount asked for the digest for five minutes | resumes on the next mount | From c60f1460fe8d2935df4f1b8fc5aa74a1869d2c2d Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 21:40:51 +0800 Subject: [PATCH 05/10] test(e2e): case 114 asserts the expectedDigest anchor end to end - a match resolves with digestSource Hub; a foreign anchor is refused DigestMismatch with both digests in the message - a twice-unreachable hub writes the anchor as the identity with the hub's request log flat, and the Expected identity mounts from the peer with the hub log still flat - Engine delivery holds both anchored artifacts with AnchorNeedsNodeDelivery and no Pod - the peer leg skips on one worker or with E2E_C114_OFFLINE=1; the run restores both Settings it flips Task 5 of model-artifact-expected-digest. Signed-off-by: thxCode --- .agents/skills/gpustack-operator-e2e/SKILL.md | 1 + .../gpustack-operator-e2e/cases/case-114.sh | 240 ++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-114.sh diff --git a/.agents/skills/gpustack-operator-e2e/SKILL.md b/.agents/skills/gpustack-operator-e2e/SKILL.md index e06653ea3..42694636e 100644 --- a/.agents/skills/gpustack-operator-e2e/SKILL.md +++ b/.agents/skills/gpustack-operator-e2e/SKILL.md @@ -158,6 +158,7 @@ Each case is self-contained; its header (see **Case header contract**) states go | 111 | Node-to-node sync: a cold node materializes a digest from a peer's published tree with source=Peer and its hub byte counter at zero; the seed's plugin pod dying mid-pull still ends Ready (checkpoint resume); a tenant pod cannot reach the peer port | `pkg/modelmanager/peer/**`, `pkg/modelmanager/materialize/materialize.go`, `pkg/modelmanager/config.go`, `deploy/gpustack-operator/chart/templates/model-manager/**`, `docs/model-store/peer-sync.md` | yes (confirm) | two schedulable workers (a seed and a puller), the chart installed with `modelManager.port` non-zero and the peer NetworkPolicy enabled, `model-store-peer-sync` at its `true` default | | 112 | Image source: a tag-only reference is refused naming the digest contract; an image artifact resolves claim-shaped (no manifest digest, no revision, no `status.nodes`); an Instance pinned to the node mounts the pinned image read-only through an image volume and reads the fixture byte for byte; a `ModelPrefetch` naming the artifact is refused | `pkg/worker/webhooks/worker/{model_artifact,model_prefetch}.go`, `pkg/worker/controllers/worker/{model_artifact_placement,model_deployment_artifact,instance,model_placement_preference}.go`, `pkg/kubediscovery/feature.go`, `cases/case-112.sh` | yes (confirm) | one worker whose kubelet/containerd serve image volumes (kubelet 1.35+, containerd 2.1+), docker on the runner, the stock `registry:2`, `python:3.12-slim` and `busybox:1.36` images pullable | | 113 | ModelScope source: a `modelScope` artifact resolves to the commit `git ls-remote` also names and materializes on a cold node hashing to the hub's own sha256; a second node pulls from the peer with its hub byte counter flat; a `modelscope` SDK at the floor downloads at the resolved commit; a well-formed `modelScope` source is admitted and a three-part repository refused | `pkg/modelartifact/modelscope.go`, `pkg/worker/controllers/worker/model_artifact.go`, `pkg/modelmanager/{driver/authorize.go,materialize/materialize.go,report/report.go}`, `pkg/worker/webhooks/worker/model_artifact.go`, `cases/case-113.sh` | yes (confirm) | two schedulable workers, the chart with `modelManager.enabled`, the stock python image pullable, and an internet path to www.modelscope.cn and pypi.org from the nodes and the runner (`E2E_C113_OFFLINE=1` skips) | +| 114 | Expected digest: a match resolves with `digestSource Hub`, a foreign anchor is refused `DigestMismatch` naming both digests, a twice-unreachable hub writes the anchor as the identity (`digestSource Expected`, no revision) with the hub's request log flat, `Engine` delivery holds both anchored artifacts with `AnchorNeedsNodeDelivery` and no Pod, and an `Expected` identity mounts on a second node from the peer, hub log still flat | `api/worker/v1alpha1/model_artifact.go`, `pkg/worker/webhooks/worker/model_artifact.go`, `pkg/worker/controllers/worker/{model_artifact,model_artifact_placement,model_deployment}.go`, `pkg/modelmanager/materialize/materialize.go`, `pkg/modelmanager/peer/client.go`, `cases/case-114.sh` | yes (confirm) | one schedulable worker for the anchor legs, two for the peer leg (`E2E_C114_OFFLINE=1` skips that leg), the chart with `modelManager.enabled`, the stock python image pullable, one InstanceType for the engine-block rows; no internet — the hub is the case's own test hub | Each note below is something the **lead** must act on before or around a run. What a case *does* — its goal, environment, inputs, assertions and cleanup — lives in its own header, which the **Case header contract** below requires to be readable on its own; the index never restates it. diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh new file mode 100755 index 000000000..1780a018a --- /dev/null +++ b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# +# CASE 114 — An expectedDigest asserts the content: a match resolves, a mismatch refuses naming both +# digests, a twice-unreachable hub hands the identity to the anchor, and an anchored +# artifact is delivered by a node alone (MUTATING, self-recovering; the peer leg +# AUTO-SKIPS on one worker or with E2E_C114_OFFLINE=1) +# +# case-114.sh +# +# Goal: Pin the anchor end to end against a controlled hub: the digest a resolution +# produces is exactly the assertion, a hub serving other content is refused with both +# digests in the message, a hub that twice cannot be reached writes the anchor as the +# identity (digestSource Expected, no revision) and is never asked again, and an +# anchored artifact is held under Engine delivery and served from the peer's published +# tree under Node delivery. +# Environment: A cluster installed from this chart with modelManager.enabled (the default), the +# stock python image pullable, and two schedulable workers for the peer leg. NO GPU +# and no internet: the hub is this case's own in-cluster test hub, so the refusal and +# the confirmed outage are exact. +# Inputs: MOCKED: the hub (_model-hub.py) serving one small repository, and the worker's hub +# endpoint Setting pointed at it for the run and restored afterwards. Everything the +# anchor reads — resolution, the staircase, the ledger — is the real code path. +# Expected: - an unanchored twin resolves, and its digest is published on the first worker; +# - an artifact anchored to that digest resolves with digestSource Hub and a commit; +# - an artifact anchored to another digest stays Resolved=False, reason +# DigestMismatch, message carrying both digests; +# - with the hub unreachable, an artifact anchored to the first digest waits out one +# blip, then resolves to the anchor (digestSource Expected, revision empty), while +# the hub's request log stays flat from the flip onward; +# - under Engine delivery both anchored artifacts are held with +# AnchorNeedsNodeDelivery — the Expected one's message naming the missing commit — +# and no Pod is created; +# - a consumer on the second worker mounts the Expected identity, the hub's request +# log still flat, its peer bytes risen (or the node pre-warmed). +# Cleanup: Trap restores both Settings, deletes the Pods, artifacts, deployments and the hub. +set -uo pipefail + +E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" +[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +. "${CASES_DIR}/_rows-lib.sh" +# shellcheck source=/dev/null +. "${CASES_DIR}/_model-hub-lib.sh" + +NS="${1:?usage: case-114.sh }" +if [ "$NS" = "$SYSTEM_NS" ]; then + echo "[case-114] refusing system namespace ${SYSTEM_NS}; usage: case-114.sh " >&2 + exit 2 +fi +P=c114 +COMMIT_BOUND=120 +ANCHOR_BOUND=240 +POD_BOUND=300 +WRONG="sha256:$(printf 'f%.0s' $(seq 64))" +FAILS=0 +ROWS=() +record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } + +WORKERS=() +for _w in $(model_workers); do WORKERS+=("$_w"); done +if [ "${#WORKERS[@]}" -lt 1 ]; then + echo "[case-114] SKIP: needs a schedulable worker, found none" >&2 + exit 0 +fi + +# The two Settings this case flips, saved so the trap restores whatever the cluster carried. +EP_SAVED="$(setting_get model-artifact-huggingface-endpoint)" +DELIVERY_SAVED="$(setting_get model-artifact-delivery-mode)" + +# shellcheck disable=SC2317 # the trap keeps it reachable; the checker cannot see the EXIT +cleanup() { + echo + echo "[case-114] cleanup" + if [ -n "$EP_SAVED" ]; then setting_set model-artifact-huggingface-endpoint "$EP_SAVED" >/dev/null; else setting_unset model-artifact-huggingface-endpoint >/dev/null; fi + if [ -n "$DELIVERY_SAVED" ]; then setting_set model-artifact-delivery-mode "$DELIVERY_SAVED" >/dev/null; else setting_unset model-artifact-delivery-mode >/dev/null; fi + kubectl -n "$NS" delete pod "${P}-seed" "${P}-peer" --ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 + kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "${P}-md-match" "${P}-md-expected" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai "${P}-twin" "${P}-match" "${P}-mismatch" "${P}-expected" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete deploy "${P}-hub" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete svc "${P}-hub" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 +} +trap cleanup EXIT + +REPOS="$(python3 -c " +import json +print(json.dumps({ + 'e2e/anchored': {'files': {'config.json': {'size': 300}, 'tokenizer.json': {'size': 3000}}}, +}))")" +HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" +setting_set model-artifact-huggingface-endpoint "$HUB_URL" >/dev/null +setting_set model-artifact-delivery-mode Node >/dev/null +settings_settle +echo "[case-114] hub at ${HUB_URL}; workers ${WORKERS[*]}" + +echo "== 1. the twin resolves, its digest publishes on the first worker ==" +artifact "$NS" "${P}-twin" e2e/anchored "" "$P" +DIGEST="$(wait_resolved "$NS" "${P}-twin" "$COMMIT_BOUND")" +if [ -n "$DIGEST" ]; then + record PASS "twin resolved" "digest ${DIGEST:0:19}…" +else + record FAIL "twin resolved" "no digest within ${COMMIT_BOUND}s: $(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-twin" -o jsonpath='{.status.conditions[?(@.type=="Resolved")].message}' 2>/dev/null)" +fi +[ -n "$DIGEST" ] || { print_rows "${ROWS[@]}"; exit 1; } + +TWIN_UID=$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-twin" -o jsonpath='{.metadata.uid}') +consumer "$NS" "${P}-seed" "${WORKERS[0]}" "${P}-twin" "$TWIN_UID" "$DIGEST" +if pod_ready "$NS" "${P}-seed" "$POD_BOUND"; then + record PASS "seed node" "the first worker holds the published tree" +else + record FAIL "seed node" "not Running within ${POD_BOUND}s: $(pod_mount_events "$NS" "${P}-seed" | head -1)" +fi + +echo "== 2. a matching anchor resolves as a hub identity ==" +artifact "$NS" "${P}-match" e2e/anchored "" "$P" " + expectedDigest: $DIGEST" +MATCH_DIGEST="$(wait_resolved "$NS" "${P}-match" "$COMMIT_BOUND")" +MATCH_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-match" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" +MATCH_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-match" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" +if [ "$MATCH_DIGEST" = "$DIGEST" ] && [ "$MATCH_SRC" = Hub ] && [ -n "$MATCH_REV" ]; then + record PASS "match resolves" "digestSource Hub at commit ${MATCH_REV:0:12}…" +else + record FAIL "match resolves" "digest '${MATCH_DIGEST:0:19}…' source '$MATCH_SRC' revision '$MATCH_REV'" +fi + +echo "== 3. a foreign anchor is refused with both digests ==" +artifact "$NS" "${P}-mismatch" e2e/anchored "" "$P" " + expectedDigest: $WRONG" +MISMATCH_MSG="" +for _ in $(seq 1 45); do + R="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-mismatch" \ + -o jsonpath='{.status.conditions[?(@.type=="Resolved")].reason}' 2>/dev/null)" + [ "$R" = DigestMismatch ] && break + sleep 2 +done +MISMATCH_MSG="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-mismatch" -o jsonpath='{.status.conditions[?(@.type=="Resolved")].message}' 2>/dev/null)" +if [ "$R" = DigestMismatch ] && printf '%s' "$MISMATCH_MSG" | grep -qF "$WRONG" && printf '%s' "$MISMATCH_MSG" | grep -qF "$DIGEST"; then + record PASS "mismatch refused" "DigestMismatch names the anchor and the hub's digest" +else + record FAIL "mismatch refused" "reason '$R', message '${MISMATCH_MSG:0:160}'" +fi + +echo "== 4. a twice-unreachable hub hands the identity to the anchor ==" +LOG_BEFORE="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" +setting_set model-artifact-huggingface-endpoint "http://${P}-absent.${NS}.svc:8080" >/dev/null +settings_settle +artifact "$NS" "${P}-expected" e2e/anchored "" "$P" " + expectedDigest: $DIGEST" +EXPECTED_DIGEST="$(wait_resolved "$NS" "${P}-expected" "$ANCHOR_BOUND")" +EXPECTED_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" +EXPECTED_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" +LOG_DURING="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" +if [ "$EXPECTED_DIGEST" = "$DIGEST" ] && [ "$EXPECTED_SRC" = Expected ] && [ -z "$EXPECTED_REV" ] && [ "$LOG_DURING" = "$LOG_BEFORE" ]; then + record PASS "anchored identity" "digestSource Expected, no revision, hub log flat (${LOG_BEFORE} → ${LOG_DURING})" +else + record FAIL "anchored identity" "digest '${EXPECTED_DIGEST:0:19}…' source '$EXPECTED_SRC' revision '$EXPECTED_REV' log ${LOG_BEFORE} → ${LOG_DURING}" +fi +if [ -n "$EP_SAVED" ]; then setting_set model-artifact-huggingface-endpoint "$EP_SAVED" >/dev/null; else setting_unset model-artifact-huggingface-endpoint >/dev/null; fi +settings_settle + +echo "== 5. Engine delivery is refused for anchored artifacts ==" +IT="$(kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" +if [ -z "$IT" ]; then + record SKIP "engine block" "no InstanceType exists to reference; the render is unit-covered" +else + setting_set model-artifact-delivery-mode Engine >/dev/null + settings_settle + for pair in "${P}-match:${P}-md-match:cannot anchor-verify" "${P}-expected:${P}-md-expected:no resolved commit"; do + ART="${pair%%:*}"; REST="${pair#*:}"; MD="${REST%%:*}"; WANT="${REST#*:}" + kubectl apply -f - >/dev/null </dev/null)" + [ "$REASON" = AnchorNeedsNodeDelivery ] && break + sleep 2 + done + MSG="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai "$MD" -o jsonpath='{.status.conditions[?(@.type=="WeightsReady")].message}' 2>/dev/null)" + PODS="$(kubectl -n "$NS" get pods -l "worker.gpustack.ai/model-deployment=$MD" --no-headers 2>/dev/null | wc -l | tr -d ' ')" + if [ "$REASON" = AnchorNeedsNodeDelivery ] && printf '%s' "$MSG" | grep -qF "$WANT" && [ "$PODS" = 0 ]; then + record PASS "engine block" "$MD held with AnchorNeedsNodeDelivery ('$WANT'), no Pod" + else + record FAIL "engine block" "$MD reason '$REASON', pods $PODS, message '${MSG:0:120}'" + fi + kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$MD" --ignore-not-found >/dev/null 2>&1 + done + if [ -n "$DELIVERY_SAVED" ]; then setting_set model-artifact-delivery-mode "$DELIVERY_SAVED" >/dev/null; else setting_unset model-artifact-delivery-mode >/dev/null; fi + settings_settle +fi + +echo "== 6. the Expected identity is served from the peer ==" +if [ "${#WORKERS[@]}" -lt 2 ]; then + record SKIP "peer mount" "needs two schedulable workers, found ${#WORKERS[@]}" +elif [ "${E2E_C114_OFFLINE:-0}" = 1 ]; then + record SKIP "peer mount" "E2E_C114_OFFLINE=1" +else + EXPECTED_UID=$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.metadata.uid}') + kubectl -n "$SYSTEM_NS" delete pod -l app.kubernetes.io/component=model-manager \ + --field-selector "spec.nodeName=${WORKERS[1]}" --wait=true --timeout=120s >/dev/null 2>&1 + for _ in $(seq 1 30); do + PP="$(plugin_pod "${WORKERS[1]}")" + [ -n "$PP" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$PP" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break + sleep 3 + done + LOG_BEFORE="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" + PEER_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" + HUB_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" + consumer "$NS" "${P}-peer" "${WORKERS[1]}" "${P}-expected" "$EXPECTED_UID" "$DIGEST" + if pod_ready "$NS" "${P}-peer" "$POD_BOUND"; then + LOG_AFTER="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" + PEER_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" + HUB_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" + if [ "$LOG_AFTER" != "$LOG_BEFORE" ]; then + record FAIL "peer mount" "the hub was asked (${LOG_BEFORE} → ${LOG_AFTER}) though the identity is Expected" + elif [ "$PEER_AFTER" -gt "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then + record PASS "peer mount" "the second node took ${PEER_AFTER} peer bytes and ${HUB_AFTER} hub bytes" + elif [ "$PEER_AFTER" -le "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then + record PASS "peer mount" "the mount is a hit on pre-warmed content (bytes flat); the hub log stayed flat" + else + record FAIL "peer mount" "the second node pulled from the hub (${HUB_BEFORE} → ${HUB_AFTER})" + fi + else + record FAIL "peer mount" "not Running within ${POD_BOUND}s: $(pod_mount_events "$NS" "${P}-peer" | head -1)" + fi +fi + +print_rows "${ROWS[@]}" +exit "$FAILS" From 03e49b292165e0c5357936229c90b071417f5ae8 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 21:44:24 +0800 Subject: [PATCH 06/10] fix(worker): an Expected identity's files carry no hub URL, and the KV pin - fetchTree built every file's URL by calling the hub, which is nil on the peers-only path and would panic before a peer byte was pulled; the URL is now empty when there is no hub, and the per-file loop's no-hub branch was already the loud no-source failure - the KV identity pin the spec's acceptance names: the digest's leading digits for a hub artifact and for an Expected identity alike, the UID's hash only for a claim Task 5 of model-artifact-expected-digest. Signed-off-by: thxCode --- pkg/modelmanager/materialize/materialize.go | 13 ++++++++++- .../controllers/worker/model_artifact_test.go | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/pkg/modelmanager/materialize/materialize.go b/pkg/modelmanager/materialize/materialize.go index 712877453..a2cf13c0e 100644 --- a/pkg/modelmanager/materialize/materialize.go +++ b/pkg/modelmanager/materialize/materialize.go @@ -417,6 +417,17 @@ func (m *Materializer) fromPeersOnly(ctx context.Context, j *job, a *store.Attem return m.fetchTree(ctx, j, a, manifest, nil, nil, src) } +// hubFileURL is a hub download address, or empty when there is no hub: an Expected identity's +// files carry no fallback URL, and the per-file loop goes to the peers or fails naming the +// digest nothing holds. +func hubFileURL(hub Hub, src *source, path string) string { + if hub == nil { + return "" + } + + return hub.FileURL(src.repository, src.commit, path) +} + func (m *Materializer) fromSource( ctx context.Context, j *job, a *store.Attempt, hub Hub, dl *download.Downloader, src *source, ) error { @@ -484,7 +495,7 @@ func (m *Materializer) fetchTree( for i, e := range manifest.Entries { f := download.File{ Path: e.Path, Size: e.Size, Digest: e.Digest, - URL: hub.FileURL(src.repository, src.commit, e.Path), + URL: hubFileURL(hub, src, e.Path), Dest: dests[i], Checkpoint: checkpoints[i], Token: src.currentToken, diff --git a/pkg/worker/controllers/worker/model_artifact_test.go b/pkg/worker/controllers/worker/model_artifact_test.go index e15913d7b..dce6fb4fd 100644 --- a/pkg/worker/controllers/worker/model_artifact_test.go +++ b/pkg/worker/controllers/worker/model_artifact_test.go @@ -2,6 +2,8 @@ package worker import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "net/http" @@ -1141,6 +1143,26 @@ func TestModelArtifactReconcileAssertsTheExpectedDigest(t *testing.T) { }) } +// TestModelArtifactKVIdentity pins the identity a KV store's keys carry: the manifest digest's +// leading digits wherever the artifact resolved to one — an anchored artifact included, its +// Expected identity being a digest like any other — and the UID's hash only where nothing did. +func TestModelArtifactKVIdentity(t *testing.T) { + ma := artifactFixture("", true, true) + assert.Equal(t, "m-"+strings.Repeat("1", 32), modelArtifactKVIdentity(ma), + "a hub artifact's identity is its digest's leading digits") + + ma.Spec.ExpectedDigest = testArtifactDigest + ma.Status.Resolved.Revision = "" + ma.Status.Resolved.DigestSource = workercore.ModelArtifactDigestSourceExpected + assert.Equal(t, "m-"+strings.Repeat("1", 32), modelArtifactKVIdentity(ma), + "an Expected identity keys on the digest it carries") + + claim := artifactFixture("models", true, true) + sum := sha256.Sum256([]byte(claim.UID)) + assert.Equal(t, "m-"+hex.EncodeToString(sum[:])[:32], modelArtifactKVIdentity(claim), + "a claim's identity is the artifact, not its content") +} + func mustGet(t *testing.T, cli ctrlcli.Client) *workercore.ModelArtifact { t.Helper() ma := new(workercore.ModelArtifact) From 70fd2feba42746ada4fe02d2bd396e92f7ea7bb1 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 22:26:11 +0800 Subject: [PATCH 07/10] fix(e2e): case 114 passes the anchor at the spec's indentation The artifact helper appends its last argument under spec:, six spaces landed the field inside the huggingFace member and strict decoding refused every anchored artifact the case created. Task 5 of model-artifact-expected-digest. Signed-off-by: thxCode --- .agents/skills/gpustack-operator-e2e/cases/case-114.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh index 1780a018a..40ce47e5d 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh @@ -114,7 +114,7 @@ fi echo "== 2. a matching anchor resolves as a hub identity ==" artifact "$NS" "${P}-match" e2e/anchored "" "$P" " - expectedDigest: $DIGEST" + expectedDigest: $DIGEST" MATCH_DIGEST="$(wait_resolved "$NS" "${P}-match" "$COMMIT_BOUND")" MATCH_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-match" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" MATCH_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-match" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" @@ -126,7 +126,7 @@ fi echo "== 3. a foreign anchor is refused with both digests ==" artifact "$NS" "${P}-mismatch" e2e/anchored "" "$P" " - expectedDigest: $WRONG" + expectedDigest: $WRONG" MISMATCH_MSG="" for _ in $(seq 1 45); do R="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-mismatch" \ @@ -146,7 +146,7 @@ LOG_BEFORE="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" setting_set model-artifact-huggingface-endpoint "http://${P}-absent.${NS}.svc:8080" >/dev/null settings_settle artifact "$NS" "${P}-expected" e2e/anchored "" "$P" " - expectedDigest: $DIGEST" + expectedDigest: $DIGEST" EXPECTED_DIGEST="$(wait_resolved "$NS" "${P}-expected" "$ANCHOR_BOUND")" EXPECTED_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" EXPECTED_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" From 42c34e1fe224946d7bf2fb2705095c9ed77adc54 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 22:41:23 +0800 Subject: [PATCH 08/10] fix(e2e): case 114's engine-block roles name an image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The replica render synthesizes the runner image from the instance type's observed hardware; the cluster's generic type has none, so the render failed before WeightsReady could carry the anchor block. Name the stock python image on the role — the block creates no Pod whatever the image is. Task 5 of model-artifact-expected-digest. Signed-off-by: thxCode --- .agents/skills/gpustack-operator-e2e/cases/case-114.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh index 40ce47e5d..f1a96a995 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh @@ -178,6 +178,7 @@ spec: roles: - name: server instanceType: ${IT} + image: ${MH_PYTHON} replicas: 1 YAML REASON="" From 355129ad1d78477edc77c54d62f6ee529a1a3238 Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 23:09:53 +0800 Subject: [PATCH 09/10] docs(specs): ship the ModelArtifact expected digest T1-T6 delivered: the anchor's admission, assertion at resolution and revalidation, the confirmed-outage Expected identity, the peers-only chain, the Engine block, the documentation, and the e2e case. The plan-gate adjudications are recorded in the spec's Open Questions and Alternatives. Task 6 of model-artifact-expected-digest. Signed-off-by: thxCode --- ...26-09-29-model-artifact-expected-digest.md | 520 ++++++++++++++++++ 1 file changed, 520 insertions(+) create mode 100644 specs/2026-09-29-model-artifact-expected-digest.md diff --git a/specs/2026-09-29-model-artifact-expected-digest.md b/specs/2026-09-29-model-artifact-expected-digest.md new file mode 100644 index 000000000..a0b0ea95b --- /dev/null +++ b/specs/2026-09-29-model-artifact-expected-digest.md @@ -0,0 +1,520 @@ +# Spec: ModelArtifact Expected Digest + +Status: Shipped +Shipped on: 2026-09-29, as the pull request this commit lands with. T1–T6 delivered; the plan-gate +adjudications (the claim anchor excluded by the user, digest-as-capability accepted and documented, +the #693 migration contract) are recorded in Open Questions and Alternatives. +Type: Feature + +## Summary + +Add an optional `expectedDigest` to the `ModelArtifact` spec: the manifest digest the user asserts +this artifact resolves to. The manifest format v1 was published precisely so that such an assertion +could later be compared with it; this spec opens that comparison. On a hub source the resolution +proceeds as today and a mismatch refuses it with a new `DigestMismatch` reason; the periodic +revalidation re-computes the manifest instead of probing one file, so a hub that stops serving the +asserted content is caught too. When the hub cannot be reached at all, an anchored hub artifact +resolves to its anchor without contacting the hub (`digestSource: Expected`) and delivers from a +peer that holds the published tree — an air-gapped cluster's working delivery path. The same anchor +is the acceptance contract for the deferred legacy-cache migration +([#693](https://github.com/gpustack/gpustack-operator/issues/693)): whatever form the future import +takes, it may publish only trees whose computed manifest digest equals the artifact's anchor; the +tool itself is out of scope this cycle and the issue stays open. + +## Motivation + +### Goals + +`expectedDigest` is an identity anchor on top of the published `gpustack-manifest v1` format. It +changes neither the format nor the digest algorithm; it asserts which manifest digest this artifact +is, and the system refuses anything else. Two scenarios each get an end-to-end answer: + +1. **Hub sources assert what they resolve to.** A user who resolved the artifact once — on another + cluster, from a mirror, in a change ticket — pins the digest they saw. Resolution still talks to + the hub; a hub that answers with different content than the anchor asserts fails loudly + (`Resolved=False`, `DigestMismatch`, both digests in the message) instead of delivering content + the user never asserted. The revalidation upgrades from a single-file `HEAD` to a full + manifest re-computation, so the same refusal catches a source that drifts after resolution. +2. **A pre-resolved artifact works without the hub.** An air-gapped cluster cannot resolve: today + the artifact never resolves and nothing delivers. With an anchor, the controller writes the + identity from the spec alone (`digestSource: Expected`) after the hub's absence is confirmed, + and delivery draws its bytes from a peer that holds them; the spec states exactly which + sub-cases remain unreachable and does not pretend otherwise. + +The anchor is also the contract the deferred migration (#693) will be held to: verifying a +directory of unknown-provenance bytes against a manifest digest before anything is published is the +one operation any future import must perform, and this spec publishes the format and the pipeline +it runs through. The tool itself is another cycle's work. + +### Non-Goals + +- **No change to the manifest format v1, the digest algorithms, or the canonical bytes.** The + anchor is an assertion over the published format, not a format extension. Any change to the + format would change digests and strand every anchor. +- **No anchor on claim or image sources.** A claim's content is whatever the provisioner + materialized at mount time — like an image volume, the consumer only knows what it got when it + mounts — and a claim's identity is the claim itself, as the documentation has always said; the + user confirms their claim's content with their own tooling. An image is already pinned by its + reference's OCI digest. Admission refuses `expectedDigest` on both, so no claim path changes in + any way: same delivery, same placement, same status, no bridge, no Verified-for-claims. +- **No cross-source digest deduplication.** Digests stay incomparable across hubs (non-LFS files + hash as `gitsha1` on Hugging Face, `sha256` on ModelScope). The anchor says nothing about content + equivalence across sources. +- **No engine delivery for anchored artifacts.** An engine downloads by repository+revision and + cannot anchor-verify what it fetched (an `Expected`-identity artifact additionally has no commit + to pin). Anchored artifacts deliver through the node only. +- **No automatic upgrade of an anchored artifact to a hub-resolved one.** Once an artifact resolves + to its anchor (`digestSource: Expected`) it never contacts the hub — not for resolution, not for + revalidation, not for the token probe. A user who wants the hub-verified form recreates the + artifact while connected. +- **No anchor-less adoption.** A hub artifact without an anchor resolves and revalidates exactly + as today; its status, chain and delivery are unchanged. +- **No `Verified` condition.** Its only novel scenario was the claim anchor; for hub sources + `status.nodes.ready` already answers whether any node holds a published copy. The condition + stays deferred, as the format's design note left it. +- **No legacy-cache migration tool this cycle.** #693 is deferred by adjudication (2026-09-29): + the issue stays open and is arranged separately. This spec publishes only the acceptance anchor + such a migration must satisfy — a future import, whatever its form, verifies a candidate tree's + computed manifest digest against `expectedDigest` and publishes only through the store's own + pipeline — and designs no tool body: no CLI, no Job, no import RPC, no `NodeModelStore` section. +- **No DeliveryClass or external provider work** (the reserved S8), and no change to the CSI volume + contract. + +## Proposal + +### The API (additive, one generation run) + +```yaml +spec: + source: {...} # unchanged, exactly one member + expectedDigest: sha256:669ed7b1… # optional; "sha256:" + 64 lowercase hex; hub sources only +status: + resolved: + revision: 7ae557604adf67be50417f59c2c2f167def9a775 # absent for an Expected-digestSource artifact + manifestDigest: sha256:669ed7b1… + digestSource: Hub # Hub (the hub's listing) | Expected (the spec's anchor) + fileCount: 10 # absent for an Expected-digestSource artifact + sizeBytes: 999604126 + nodes: {...} + conditions: # unchanged: Resolved, Degraded + - {type: Resolved, status: "True", reason: Resolved} + - {type: Degraded, status: "False", reason: Healthy} +``` + +- `spec.expectedDigest` is optional, validated as `sha256:` + 64 lowercase hex, and immutable with + the rest of the spec (the existing whole-spec identity rule already covers it: the webhook's + deep-equality check sees the new field like any other). +- `status.resolved.digestSource` is written whenever `manifestDigest` is: `Hub` when the digest came + from a hub listing, `Expected` when it came from the spec's anchor. It makes the two kinds of + identity visible instead of inferable, which matters to everything downstream that decides + whether a revision exists. +- The `status.nodes` aggregation, the conditions and every claim/image behavior are untouched. + +### Admission + +- `expectedDigest` is accepted on hub sources and **refused on claim and image sources**: a claim's + content is whatever the claim holds at mount time (dynamically provisioned claims differ per + provisioning — the user confirms it, not the operator), and an image is already pinned by its + reference's OCI digest; an anchor on either would invite a field that can never say anything. + The refusal messages follow the existing image-source refusal's shape. +- The shape rule is the image reference's digest rule reused: `^sha256:[a-f0-9]{64}$`. +- Source immutability is unchanged; the spec's deep-equality now covers the new field. An update + that changes only `expectedDigest` is refused with the identity message. + +### Resolution (controller) + +**Hub source with an anchor.** Resolution is today's: resolve the revision to a commit, list the +tree, canonicalize, digest. Then compare: equal to `expectedDigest` → `Resolved=True`, +`digestSource: Hub`, exactly as an unanchored artifact; unequal → `Resolved=False` with the new +reason `DigestMismatch` and a message carrying both digests ("expected …, the hub resolved …"). +`DigestMismatch` joins the revoking reasons, so it rides the existing staircase: retried on the +refused cadence, because the fix is a corrected anchor — a new artifact, since the spec is +immutable. + +**Revalidation with an anchor.** The anchor makes the periodic check stronger: instead of one +`HEAD` of a file, the controller re-lists the tree at the resolved commit, recomputes the digest, +and compares it with the anchor. A hub (or mirror endpoint) that now serves different content at the +same commit fails the same `DigestMismatch` staircase — first refusal sets `Degraded`, the confirmed +refusal sets `Resolved=False`. The cost is one listing per interval for anchored artifacts only; +unanchored artifacts keep the two-request check. + +**Hub source, anchor, no hub.** If the resolution fails with `SourceUnavailable` **on the +confirmed cadence** — the refused staircase's second pass, one minute after the first, the same +"a single refused request is not yet a verdict" discipline the revocation staircase uses — and the +artifact carries an anchor, the controller writes the identity from the spec: +`resolved{manifestDigest: expectedDigest, digestSource: Expected}`, `Resolved=True`, no revision, +no `fileCount`/`sizeBytes`. One transient blip at creation therefore does not freeze the +artifact's identity; a genuinely hub-less environment waits one extra minute. Revalidation does +nothing for such an artifact — there is no commit to probe and no hub assumed to exist, including +the token probe (no `ValidToken` call is made on an anchored pass). The state is visible +(`digestSource: Expected`, `revision` absent), never inferred after the fact. A refusal that is +*not* `SourceUnavailable` (`AccessDenied`, `RevisionNotFound`, `DigestMismatch` from a reachable +hub) never falls back: a hub that answered is a hub whose answer decides. + +### Node delivery (plugin) + +- **Renderer.** An anchored hub artifact renders Node delivery as hub artifacts do. Under Engine + delivery it is blocked with a new `WeightsReady` reason `AnchorNeedsNodeDelivery`, whose message + names the honest reason: the engine downloads by repository and revision and cannot anchor-verify + what it fetched (for an `Expected`-identity artifact there is additionally no commit to pin). + The mount rule is unchanged — a claim source is still not a hub artifact, and no claim carries an + anchor anymore. +- **Source order.** A hub-identity artifact's chain is today's: peers first as a byte source, then + the hub listing + download — the listing carries the manifest and is anchor-checked before any + byte moves. An `Expected`-identity artifact has no commit to list at, so its chain is **peers + only**; no peer holding the tree is a loud failure whose message names the missing source ("no + node holds sha256: and the artifact is anchored without a hub"), not a hub error dressed + up as an integrity mismatch. An unanchored artifact keeps today's chain and today's behavior in + full. + +### The migration contract (#693, deferred) + +The legacy-cache migration is out of scope this cycle (adjudicated 2026-09-29; the issue stays +open). This spec fixes the interface any future import is held to: + +- The import verifies the candidate tree against the artifact's `expectedDigest` — computing the + canonical manifest from the actual bytes and comparing digests — before anything is published. +- The import publishes only through the node store's own pipeline (verify, seal, one rename), so + the single-writer invariant and the nothing-unverified-in-`published/` rule hold whatever the + import's trigger is. +- The import is lazy and per artifact, never a bulk copy; and it is a one-shot import of files at + rest, never an online migration of a running legacy install. + +Everything the future tool needs — the published format and the pipeline — exists today; the +anchor gives it the verdict to publish by. + +### Free behaviors + +Everything downstream keys on the manifest digest and needs no new mechanism once an anchored hub +artifact resolves: **delivery** (the mount is the same bind of a published tree), **peer sync**, +**progress aggregation** (`status.nodes` counts the digest), **prefetch** (a `ModelPrefetch` warms +the digest through the existing hub warm-up path), **placement preference**, and the **KV reuse +identity** (already keyed on `status.resolved.manifestDigest`). Each is pinned by a test that +proves the mechanism unchanged (AC1, AC5); none is code. + +### Documentation + +- `docs/model-store/artifact.md`: the `expectedDigest` field in the resource example and its rules + (shape, immutability, hub sources only — the claim and image refusals and why); `digestSource` in + the status section; the `DigestMismatch` row in the reason table; the anchored-resolution states + (hub-verified and Expected) and what revalidation does for each; the `AnchorNeedsNodeDelivery` + row in the delivery table; the air-gapped matrix (an `Expected` artifact delivers from peers; no + peer and no hub is a loud failure); the **shipped access model for `Expected` identities** — + digest knowledge is the capability, digests being cluster-visible on `NodeModelStore` status, + integrity unaffected, hub-verified identities unchanged — per the plan-gate adjudication; the + migration-contract paragraph (#693 deferred, the anchor + as the acceptance contract); the **downgrade note** (in a downgrade window the old webhook cannot + see `expectedDigest`, so its immutability is unenforced and `digestSource` is dropped by old + workers; re-upgrading recomputes it). +- `docs/model-store/node-store.md`: the `Expected`-identity chain (peers only) and the no-source + failure message. +- `docs/README.md` and any cross-references follow the docs skill's routing; the reason tables and + the troubleshooting entries gain the new reasons. `docs/settings.md` is untouched: this spec adds + no Setting. + +### Version floors + +Unchanged: Kubernetes 1.29 for the feature path, 1.23 install. The spec adds no Kubernetes +capability requirement: one optional spec field, one optional status field, no new feature gate, +no new admission mechanism, no new component. + +## User Stories + +### Story 1 + +As a platform operator, I resolved a model on a connected workstation and now create its +`ModelArtifact` in the cluster with `expectedDigest` set to the digest I saw. If the cluster's hub +(or mirror) serves those bytes, the artifact resolves; if anything upstream drifted, the artifact +refuses with both digests in the message instead of delivering content I did not assert — and the +periodic check keeps asserting, so a mirror that drifts later is caught too. + +### Story 2 + +As an operator of an air-gapped cluster, I create a hub artifact with the anchor I resolved before +disconnecting; after the hub's absence is confirmed the artifact resolves to its anchor without +the hub, the one node I seeded while connected serves every other node over peer sync, and no byte +moves that the digest has not verified. + +## Core Features & Acceptance Criteria + +- **AC1 — Hub resolution asserts the anchor (match).** A hub artifact with `expectedDigest` equal + to the resolved digest writes `Resolved=True` (`digestSource: Hub`), the same revision, digest, + counts and delivery as an unanchored artifact; every digest-keyed behavior (peers, nodes, + prefetch, preference, KV identity) sees nothing new — pinned by tests that prove it. +- **AC2 — Hub resolution refuses a mismatch (red then green).** A fake hub whose listing digests to + something other than `expectedDigest` produces `Resolved=False`, reason `DigestMismatch`, message + carrying the expected and the actual digest, riding the revoking staircase. Shown red by + neutering the comparison (accepting any digest), then green again. +- **AC3 — Revalidation catches post-resolution drift (red then green).** With an anchor set, the + revalidation pass re-lists and re-compares: a fake hub that answers the first resolution with the + anchored tree and a later revalidation with a different tree walks the staircase + (`Degraded`, then `Resolved=False`, `DigestMismatch`). Shown red by reducing the anchored + revalidation to the single-file `HEAD`, then green. An unanchored artifact's revalidation stays + the two-request check (request count asserted). +- **AC4 — Anchored identity without a hub.** A hub artifact whose resolution fails + `SourceUnavailable` twice (the confirmed cadence) with an anchor set writes + `resolved{manifestDigest: anchor, digestSource: Expected}`, `Resolved=True`, no revision or + counts; a single blip does not fall back; no hub request of any kind — listing, revalidation, or + token probe — is made on any later pass (asserted against the fake hub's request log); a hub + refusal that is not `SourceUnavailable` does not fall back. +- **AC5 — Anchored delivery.** An anchored hub artifact under Engine delivery is blocked with + `AnchorNeedsNodeDelivery` and creates no Pods. With `Expected` identity on a hub-less cluster: a + node holding the published tree (seeded while connected) serves a second node over peer sync — + the second node materializes and mounts with zero hub bytes; with no peer holding the tree, the + mount fails loudly naming the missing source. A hub-identity artifact's chain is unchanged + (hub listing anchor-checked before bytes). +- **AC6 — Admission.** `expectedDigest` is accepted on hub sources with the exact shape + (`sha256:` + 64 lowercase hex), refused on claim and image sources each with its reason, refused + malformed, and immutable (an update changing only it is refused with the identity message). All + pre-existing cases stay green. +- **AC7 — Documentation.** The new reasons (`DigestMismatch`, `AnchorNeedsNodeDelivery`) are in + the docs' reason tables and troubleshooting; `digestSource`, the anchored-resolution states, the + air-gapped matrix, the migration-contract paragraph and the downgrade note exist as described + above. + +## Notes / Constraints / Caveats + +- **The anchor never becomes authorization — with one declared, adjudicated exception.** For + **hub-verified** identities, nothing changes: a namespace consumes content only through its own + artifact, resolved with its own credential. For an **`Expected`** identity, no credential + participates: an artifact whose anchor names a digest already published on a node can mount that + tree, so digest knowledge becomes the capability. **Adjudicated (plan gate, 2026-09-29): + accepted and documented** — integrity is never at risk, because only anchor-named, verified + bytes ever enter `published/` or a mount; what relaxes is the secrecy of content whose digest + leaked, and private content must not rely on digest secrecy (digests are visible cluster-wide on + `NodeModelStore` status, a cluster-scoped resource); without the anchor the air-gapped scenario + cannot exist at all. The docs state the shipped model and this premise; hub-verified identities + keep today's credential-resolved path. Should the user overturn this later, the `Expected` + cross-namespace sharing surface comes back for re-adjudication. +- **The manifest is still not stored in the artifact.** An anchored artifact's manifest lives where + manifests live: recomputed from the source, or served by a peer's published listing. The + controller holds only the digest. +- **`make generate` runs for the API additions** (`ModelArtifact.spec.expectedDigest`, + `ModelArtifactResolved.digestSource`), in the private gen tree per the task standards; the chart + ships the changed CRDs, so the local seven-image chart matrix runs before my-ship (REQUIRED for a + Go change). The stale ModelScope comment in `api/worker/v1alpha1/model_artifact.go` is corrected + in the same run (pure comment; S10 opened the source without updating it). +- **Measured conventions restated:** hub digests are hub knowledge (`gitsha1` for non-LFS files on + Hugging Face, `sha256` everywhere on ModelScope); the mount rule compares the volume attribute + with `status.resolved.manifestDigest`; the store publishes only through seal-and-rename. +- English code, comments, docs and commit messages; progress and gates to the coordinator in + Chinese. + +## Boundaries + +- **Always:** the anchor comparison on every path that produces or re-produces a digest; tests + proven able to fail (mutation of exactly the guard each test pins) for AC2 and AC3; per-module + squashed commits with `--signoff`, spec last; rebase on `origin/main` at each task start, before + every e2e and before my-ship; the file-intersection discipline (name the touched files in the + day's first progress message before editing existing files). +- **Ask first:** any API change beyond the two named above (`expectedDigest`, `digestSource`); + any new Setting; any chart change beyond the CRD regeneration ride-along; reviving the #693 + migration or the claim anchor inside this spec (both are adjudicated out; a change of course goes + back to the coordinator). +- **Never:** a manifest format or digest-algorithm change; cross-source dedup; delivery of an + unverified tree (nothing enters `published/` without a digest match); following a branch after + resolution; an anchored artifact contacting the hub after it resolved to `Expected`; touching + crew-line files (`pkg/setting/types.go`, `node_queue.go`, the MD webhook barrier); the S8 + DeliveryClass surface; any change to claim or image source behavior. +- Review round budget: 4; after that, findings are listed, not fixed — except data loss, a broken + security guarantee (read-only), or cross-tenant leakage, each raised with the coordinator first. + +## Risks and Mitigations + +- **An anchored artifact pinned to the wrong digest** fails at first use, loudly, at resolution — + the failure the feature exists to produce; the message carries both digests for the ticket. +- **Revalidation listing cost** → one listing per interval per anchored artifact; unanchored + artifacts keep the two-request probe. The anchor is opt-in per artifact. +- **Digest knowledge becomes a capability for `Expected` identities** (see the Notes caveat) → + flagged for the plan gate: accept-and-document (recommended) or restrict `Expected` mounts to + self-verified trees. Integrity is unaffected either way. +- **A downgrade window drops the new fields** (the old webhook cannot enforce the anchor's + immutability; old workers drop `digestSource`) → documented in the docs' downgrade note; + re-upgrading recomputes. The spec field itself is CRD-served and survives the window. +- **The confirmed-cadence fallback delays an air-gapped artifact by one minute** → deliberate: one + blip must not freeze an identity; the staircase's own cadence is the established pattern. + +## Design Details + +### Commands + +Everything runs locally on the seat's macOS worktree; kind clusters come from the e2e skill's own +provision scripts, images from the xbuild-and-verify skill, and code generation from the private +gen tree (`gen-trees/mam-s11`, branch `mam-s11-gen`, replayed onto the task branch and required to +replay zero-diff). Per task, the focused form first and the wide sweep before each commit: + +```bash +go test ./pkg/worker/webhooks/worker/ -run ModelArtifact # admission +go test ./pkg/worker/controllers/worker/ -run ModelArtifact # resolution, rendering +go test ./pkg/modelmanager/... # the plugin's Expected chain +go test ./api/worker/... # types, round-trip +make lint # golangci-lint + the rest +make lint docs && make lint chart # docs pages; CRD-bearing chart +bash .agents/skills/gpustack-operator-docs/scripts/check-specs.sh +make generate # in the private gen tree only; replay zero-diff onto the branch +``` + +The e2e case runs through the e2e skill's full harness (preflight, `case-1.sh` mandatory first), +with the case number taken at my-ship time as the lowest unused. + +### Project structure + +| Path | Role in this spec | +| --- | --- | +| `api/worker/v1alpha1/model_artifact.go` | `spec.expectedDigest`; `status.resolved.digestSource`; the stale ModelScope comment corrected | +| `pkg/worker/webhooks/worker/model_artifact.go` (+`_test.go`) | anchor shape, hub-only acceptance (claim/image refusals), immutability message | +| `pkg/worker/controllers/worker/model_artifact.go` (+`_test.go`) | hub anchor compare, `DigestMismatch` in the revoking reasons, the revalidation upgrade, the confirmed-cadence `Expected` fallback, `digestSource` on every write | +| `pkg/worker/controllers/worker/model_artifact_placement.go`, `model_deployment.go` (+tests) | `AnchorNeedsNodeDelivery` under Engine delivery | +| `pkg/modelmanager/materialize/materialize.go` (+ its test) | the `Expected`-identity chain: peers only, the no-source message | +| `docs/model-store/artifact.md`, `docs/model-store/node-store.md`, `docs/README.md` | the documented behavior listed under Documentation | + +### Code style + +Follows the tree's standing conventions: English everywhere; reasons are constants mapped verbatim +into condition reasons (`modelartifact.SourceError` is the shape); table-driven tests with a +shared loop and declarative cases; fake clients over real dependencies; errors wrapped with +`fmt.Errorf("...: %w")`; snake_case file names. New condition reasons join the existing reason +blocks, not ad-hoc strings. + +### Implementation Plan + +- [x] **T1 · API + admission tracer** + Blocked by: None + Owns: `api/worker/v1alpha1/model_artifact.go`, `pkg/worker/webhooks/worker/model_artifact.go` + (+ its test), generated artifacts via the gen tree + Acceptance: `expectedDigest` validates (`sha256:` + 64 lowercase hex), is accepted on hub + sources, refused on claim sources ("the claim's content is whatever it holds at mount time; + its identity is the claim") and on image sources ("the reference's digest is the identity"), + and is immutable with the spec; `digestSource` exists; CRD + deepcopy + protobuf regenerate + zero-diff through the gen tree; the stale ModelScope comment corrected. (AC6) + Verify: `go test ./pkg/worker/webhooks/worker/ -run ModelArtifact && go test ./api/worker/...` +- [x] **T2 · Controller: hub anchoring, anchored identity, digestSource** + Blocked by: T1 + Owns: `pkg/worker/controllers/worker/model_artifact.go` (+ its test) + Acceptance: resolution compares the anchor (`DigestMismatch`, both digests in the message, + joining the revoking reasons); revalidation with an anchor re-lists and re-compares (the hub + interface grows the manifest-returning form; the revalidation and the token probe both branch + on `digestSource`, so an `Expected` artifact makes no hub call on any pass); the + `SourceUnavailable` fallback fires only on the confirmed cadence and is skipped for any hub + that answered; `digestSource` is written on every resolved write. Mutations prove AC2 and + AC3 red. (AC1–AC4) + Verify: `go test ./pkg/worker/controllers/worker/ -run ModelArtifact` +- [x] **T3 · Delivery surface: Engine block and the Expected chain** + Blocked by: T2 + Owns: `pkg/worker/controllers/worker/model_artifact_placement.go`, `model_deployment.go` + (+ their tests), `pkg/modelmanager/materialize/materialize.go` (+ its test) + Acceptance: an anchored hub artifact under Engine delivery is blocked with + `AnchorNeedsNodeDelivery` (the anchor-verification message) and creates no Pods; the + plugin's chain for an `Expected`-identity artifact is peers only and its no-peer failure is + the loud no-source message; a hub-identity artifact's chain and its anchor-checked listing + are unchanged. (AC5) + Verify: `go test ./pkg/worker/controllers/worker/ -run ModelArtifact && + go test ./pkg/modelmanager/materialize/` +- [x] **T4 · Documentation** + Blocked by: T2, T3 (docs describe landed behavior) + Owns: `docs/model-store/artifact.md`, `docs/model-store/node-store.md`, `docs/README.md` + Acceptance: every Documentation bullet landed; `make lint docs` and the cross-reference + checks green. (AC7) + Verify: `make lint docs && + bash .agents/skills/gpustack-operator-docs/scripts/check-docs.sh . && + bash .agents/skills/gpustack-operator-docs/scripts/check-crossrefs.sh .` +- [x] **T5 · The e2e case** + Blocked by: T3; branch rebased on `origin/main` first + Owns: `.agents/skills/gpustack-operator-e2e/cases/case-.sh` (N = lowest unused at + my-ship) and its SKILL.md index row + Acceptance: on kind — a hub anchor match resolves and delivers as an unanchored artifact; + a hub anchor mismatch refuses resolution (`DigestMismatch`, both digests); an anchored + artifact under Engine delivery creates no Pods (`AnchorNeedsNodeDelivery`); the air-gap leg + (Expected identity, peer-served delivery with the hub unreachable) runs where the + environment allows, with an offline skip guard in the case-113 style. (AC1, AC2, AC5) + Verify: the e2e skill's full run on the case, green; evidence under the task's report + directory. +- [x] **T6 · Ship** + Blocked by: T4, T5 + Owns: the branch as a whole; the PR + Acceptance: all gates green locally (`make lint`, `make lint docs`, `make lint chart` for + the CRD ride-along, the agents-shell gate for the new case, the gen-tree replay zero-diff), + the spec's status advanced and committed last, the PR opened per the issue-PR conventions, + review rounds within budget. Mutation results recorded in the build handback. + Verify: the gates' transcripts; `git log` module-squashed with `--signoff`, spec last. + +### Test Plan +[ ] I/we understand the owners of the involved components may require updates to existing tests to make this +code solid enough prior to committing the changes necessary to implement this enhancement. + +#### Prerequisite testing updates +Existing suites that assert today's shapes need updating in the tasks that change them: the +webhook's immutability cases (T1) and the controller's resolution/revalidation reason tables (T2). +No test is deleted; every changed expectation is changed in the commit that changes the behavior. + +#### Unit tests +- `pkg/worker/webhooks/worker`: `2026-09-29 (T1)` — anchor shape accept/refuse, claim-source + refusal, image-source refusal, immutable update; the existing ModelArtifact admission table + grows the rows. +- `pkg/worker/controllers/worker`: `2026-09-29 (T2–T3)` — anchor compare (match/mismatch), + `DigestMismatch` in the revoking staircase, revalidation re-list vs HEAD with request counts, + confirmed-cadence fallback vs single blip vs answered refusals, the zero-hub-request pin across + Secret- and NodeModelStore-triggered passes, `digestSource` on every write, `AnchorNeedsNodeDelivery`. +- `pkg/modelmanager/materialize`: `2026-09-29 (T3)` — the `Expected` chain: peers-only, the + no-source message; a hub-identity chain unchanged. +- Mutation checks (recorded in the build handback): AC2 — neuter the comparison, the mismatch case + goes red; AC3 — reduce the anchored revalidation to the HEAD probe, the drift case goes red. + Revert and re-verify. + +#### Integration tests +The controller's fake-hub suites are the integration layer this feature has; the peer-pull leg of +AC5 runs against the existing peer test harness. + +#### e2e tests +One new case (T5): hub anchor match and mismatch, the Engine block, and the air-gap peer leg with +an offline skip guard. case-113 (ModelScope) and the existing artifact cases re-run as regression +in the same gate run. + +## Alternatives + +- **The claim anchor** (`expectedDigest` accepted on claim sources, the node plugin + reverse-computing the manifest from the claim's bytes): **rejected by the user at the plan gate + (2026-09-29)** — a claim may be dynamically provisioned, so its content is only known at mount + time, like an image; the claim's identity is the claim itself, and the user confirms their + claim's content with their own tooling. The anchor there would have made digest knowledge a + cross-tenant capability and dragged a kubelet pod-volume bridge behind it. If claim verification + is ever wanted, it is a new spec. +- **`expectedDigest` on the image source**: rejected at admission — the reference's OCI digest + already pins an image; an allowed-but-meaningless field invites a permanently failing artifact. +- **The `Verified` condition**: dropped this revision — its only novel scenario was the claim + anchor; for hub sources `status.nodes.ready` already answers whether a published copy exists. + The condition stays deferred, as the format's design note left it. +- **Silent anchored fallback for every resolution failure**: rejected — a hub that answered + (`AccessDenied`, `RevisionNotFound`) is a hub whose answer decides; only `SourceUnavailable`, the + one outcome that carries no verdict, falls back to the anchor, and only on the confirmed cadence. +- **Upgrading an `Expected` artifact to hub-resolved when the hub appears**: rejected for this spec — + it would rewrite a frozen identity's revision after the fact; recreate the artifact instead. +- **Carrying the manifest itself in the API** (a ConfigMap reference or a spec field) for the + air-gapped case: deferred — it is real API surface serving one combination, and the declared + out-of-scope keeps this spec's API additive and small. Revisit if the combination matters to a + user. +- **The legacy-cache migration in this spec (#693)**: excluded by adjudication (2026-09-29) — the + anchor this spec publishes is the acceptance contract the future import must satisfy; the tool + itself is that issue's own design work. + +## Open Questions + +1. **#693 inclusion** — **resolved, excluded (spec gate, 2026-09-29)**: the migration tool is + another cycle's work under its own issue; this spec carries the acceptance contract above. +2. **The claim anchor and the placement question** — **resolved, excluded (plan gate, + 2026-09-29)**: the user adjudicated the claim anchor out (dynamically provisioned claims hold + whatever the mount brings; the user confirms their own claim), which moots the PVC + reverse-computation placement, the kind PoC and the `Verified` condition. Claims and images are + refused an anchor at admission; no claim path changes. +3. **Air-gapped boundary** — **resolved, confirmed (spec gate, 2026-09-29)**: an anchored artifact + that resolved to `Expected` never contacts the hub again; no Engine delivery for anchored + artifacts; an anchored hub artifact offline delivers only from a peer holding the published + tree; no automatic `Expected` → hub-resolved upgrade; behavior without an anchor is unchanged. +4. **Digest-as-capability for `Expected` identities** — **resolved, accepted and documented (plan + gate, 2026-09-29)**: integrity is unaffected (only anchor-named verified bytes mount), secrecy + of leaked digests is explicitly not a control (digests are cluster-visible on `NodeModelStore` + status), hub-verified identities keep credential resolution, and the docs state the shipped + model and its premise. An overturn reopens the `Expected` sharing surface for re-adjudication. + +No question blocks the build; the plan gate confirmed the revised scope (T1–T6) on 2026-09-29 and +my-build starts at T1. From f91eb787e166e6d575cb7d5b1df815d888860cff Mon Sep 17 00:00:00 2001 From: thxCode Date: Tue, 29 Sep 2026 23:28:28 +0800 Subject: [PATCH 10/10] fix(worker): an Expected identity's listing failure keeps its cause and class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fromPeersOnly replaced FetchManifest's error wholesale: a cancellation under the listing — the waiter window firing — was misclassified as SourceUnavailable and consumed the digest's backoff, and every other underlying cause never reached the plugin's log. A cancellation now keeps its canceled class (no backoff, the shape fetchTree's wait already uses), and any other failure logs its cause beside the loud no-source. The ledger's tenant-free message is unchanged. Task 5 of model-artifact-expected-digest. Signed-off-by: thxCode --- pkg/modelmanager/materialize/materialize.go | 10 +++++++ .../materialize/materialize_test.go | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/pkg/modelmanager/materialize/materialize.go b/pkg/modelmanager/materialize/materialize.go index a2cf13c0e..576142fba 100644 --- a/pkg/modelmanager/materialize/materialize.go +++ b/pkg/modelmanager/materialize/materialize.go @@ -411,6 +411,16 @@ func (m *Materializer) fromPeersOnly(ctx context.Context, j *job, a *store.Attem } manifest, err := m.Peers.FetchManifest(ctx, "sha256:"+j.hex) if err != nil { + // A cancellation under us — the waiter window firing, the plugin stopping — is the + // outcome, not a source verdict: it must not consume the digest's backoff. + if cause := context.Cause(ctx); errors.Is(cause, errNoWaiters) { + const msg = "no mount asked for it within the waiter window" + return &download.Error{Reason: download.ReasonCanceled, Message: msg, Detail: msg} + } + // The classified failure stays the loud no-source; the underlying cause reaches the + // plugin's log, which is where the message points the diagnosing reader. + klog.V(1).InfoS("the peers-only listing found no source", "digest", "sha256:"+j.hex, "error", err.Error()) + return errNoSourceForAnchored(j.hex) } diff --git a/pkg/modelmanager/materialize/materialize_test.go b/pkg/modelmanager/materialize/materialize_test.go index 76dd278bd..46162cd13 100644 --- a/pkg/modelmanager/materialize/materialize_test.go +++ b/pkg/modelmanager/materialize/materialize_test.go @@ -221,6 +221,34 @@ func TestEnsureExpectedIdentityIsPeersOnly(t *testing.T) { assert.Contains(t, rec.Message, "peer pulling is not configured") } +// TestEnsureExpectedIdentityPeersListingFailure pins the classification when peer discovery +// itself fails: the ledger keeps the loud no-source with the digest, tenant-free, and the +// underlying cause goes to the plugin's log — it never replaces the classified failure, and a +// cancellation under the listing keeps its own canceled class (the same shape fetchTree's +// wait uses). +func TestEnsureExpectedIdentityPeersListingFailure(t *testing.T) { + env := newTestEnv(t, repoFiles()) + digest := env.hub.manifest("owner/repo").Digest + ma := env.artifact("owner/repo", "uid-a", digest) + ma.Status.Resolved.Revision = "" + ma.Status.Resolved.DigestSource = workercore.ModelArtifactDigestSourceExpected + env.m.Peers = &peer.Puller{Discover: func(context.Context, string) ([]*peer.Source, error) { + return nil, fmt.Errorf("discovery refused") + }} + req := driver.Request{Hex: store.HexOf(digest), Artifact: ma} + + env.m.Ensure(context.Background(), req) + env.waitIdle(t, req.Hex) + + assert.Empty(t, env.hub.recorded(), "the hub is still never asked") + rec, err := env.store.ReadDigest(req.Hex) + require.NoError(t, err) + assert.Equal(t, download.ReasonSourceUnavailable, rec.Reason) + assert.Contains(t, rec.Message, "sha256:"+req.Hex) + assert.NotContains(t, rec.Message, "discovery refused", + "the public message stays tenant-free; the cause is the plugin log's") +} + // waitIdle waits until no attempt runs for hex. func (e *testEnv) waitIdle(t *testing.T, hex string) { t.Helper()