From e3961da1f05f57f18830dec1fd66a2a8b7e18c2c Mon Sep 17 00:00:00 2001 From: Adam Hevenor Date: Wed, 6 May 2026 06:12:18 -0600 Subject: [PATCH 1/3] Schema tools improvements and query consistency - Add eventual consistency to all query calls (CLI commands and TUI) - Fix FTS detection: use ftsEnabled() helper instead of key-existence check so zeroed-out API objects no longer show as FTS-enabled - Expand schema apply to handle full FTS config objects, ann, glob, regex - Add schema diff support for property modifications (not just additions/conflicts) - Include zero vector in schema placeholder write for vector attributes - Add unindexed bytes column to TUI namespace list - Expand valid schema types ([]string, int, float) and validation rules - Validate id filterable and vector ann requirements Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/edit.go | 3 + cmd/get.go | 3 + cmd/list.go | 3 + cmd/scan.go | 3 + cmd/schema.go | 144 ++++++++++++++++++++---- cmd/search.go | 3 + internal/schema/schema.go | 224 ++++++++++++++++++++++++++++++++++--- internal/tui/documents.go | 6 + internal/tui/helpers.go | 26 ++++- internal/tui/namespaces.go | 16 ++- internal/tui/schema.go | 4 +- 11 files changed, 386 insertions(+), 49 deletions(-) diff --git a/cmd/edit.go b/cmd/edit.go index 4b1d949..af57e89 100644 --- a/cmd/edit.go +++ b/cmd/edit.go @@ -47,6 +47,9 @@ func runEdit(cmd *cobra.Command, args []string) error { Filters: turbopuffer.NewFilterEq("id", id), TopK: turbopuffer.Int(1), IncludeAttributes: turbopuffer.IncludeAttributesParam{Bool: turbopuffer.Bool(true)}, + Consistency: turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + }, }) if err != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", err) diff --git a/cmd/get.go b/cmd/get.go index 3dc379c..185d17c 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -50,6 +50,9 @@ func runGet(cmd *cobra.Command, args []string) error { Filters: turbopuffer.NewFilterEq("id", id), TopK: turbopuffer.Int(1), IncludeAttributes: turbopuffer.IncludeAttributesParam{Bool: turbopuffer.Bool(true)}, + Consistency: turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + }, }) if err != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", err) diff --git a/cmd/list.go b/cmd/list.go index 4cb4ba3..15658a8 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -135,6 +135,9 @@ func displayNamespaceDocuments(ctx context.Context, namespace string, topK int, RankBy: turbopuffer.NewRankByVector(vecAttr, zeroVec), TopK: turbopuffer.Int(int64(topK)), ExcludeAttributes: []string{vecAttr}, + Consistency: turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + }, }) if err != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", err) diff --git a/cmd/scan.go b/cmd/scan.go index 70f99c9..1d82e6d 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -59,6 +59,9 @@ func runScan(cmd *cobra.Command, args []string) error { RankBy: turbopuffer.NewRankByAttribute("id", turbopuffer.RankByAttributeOrderAsc), IncludeAttributes: turbopuffer.IncludeAttributesParam{StringArray: []string{field}}, TopK: turbopuffer.Int(int64(pageSize)), + Consistency: turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + }, } if lastID != "" { diff --git a/cmd/schema.go b/cmd/schema.go index d93f744..ee0f079 100644 --- a/cmd/schema.go +++ b/cmd/schema.go @@ -6,12 +6,14 @@ import ( "fmt" "os" "sort" + "strings" "github.com/hev/tpuff/internal/client" "github.com/hev/tpuff/internal/output" "github.com/hev/tpuff/internal/schema" "github.com/spf13/cobra" "github.com/turbopuffer/turbopuffer-go" + "github.com/turbopuffer/turbopuffer-go/packages/param" ) var schemaCmd = &cobra.Command{ @@ -198,9 +200,15 @@ func getCurrentSchema(ctx context.Context, ns *turbopuffer.Namespace) map[string if len(md.Schema) == 0 { return nil } - result := make(map[string]any) - for k, v := range md.Schema { - result[k] = v + // Marshal SDK structs to JSON, then unmarshal back to map[string]any + // so they're comparable to schemas loaded from JSON files. + b, err := json.Marshal(md.Schema) + if err != nil { + return nil + } + var result map[string]any + if err := json.Unmarshal(b, &result); err != nil { + return nil } return result } @@ -220,6 +228,9 @@ func displaySchemaDiff(diff schema.SchemaDiff, namespace string) { for k := range diff.Additions { allAttrs[k] = true } + for k := range diff.Modifications { + allAttrs[k] = true + } for k := range diff.Conflicts { allAttrs[k] = true } @@ -231,12 +242,17 @@ func displaySchemaDiff(diff schema.SchemaDiff, namespace string) { sort.Strings(sorted) for _, attr := range sorted { - if v, ok := diff.Unchanged[attr]; ok { - fmt.Printf(" %s: %s\n", attr, v) + if _, ok := diff.Unchanged[attr]; ok { + fmt.Printf(" %s\n", attr) } else if v, ok := diff.Additions[attr]; ok { - fmt.Printf("+%s: %s (new)\n", attr, v) + fmt.Printf("+ %s: %s (new)\n", attr, v) + } else if m, ok := diff.Modifications[attr]; ok { + fmt.Printf("~ %s (%s):\n", attr, m.BaseType) + for _, c := range m.Changes { + fmt.Printf(" %s: %s -> %s\n", c.Key, c.OldValue, c.NewValue) + } } else if c, ok := diff.Conflicts[attr]; ok { - fmt.Printf("!%s: %s -> %s (type change not allowed)\n", attr, c[0], c[1]) + fmt.Printf("! %s: %s -> %s (type change not allowed)\n", attr, c[0], c[1]) } } fmt.Println() @@ -282,11 +298,12 @@ func applySchemaToSingle(ctx context.Context, namespace string, newSchema map[st } type batchApplyResult struct { - namespace string - success bool - additions int - conflicts int - err string + namespace string + success bool + additions int + modifications int + conflicts int + err string } func applySchemaToMultiple(ctx context.Context, namespaces []string, newSchema map[string]any, region string, dryRun, yes, continueOnError bool) { @@ -307,9 +324,10 @@ func applySchemaToMultiple(ctx context.Context, namespaces []string, newSchema m diff := schema.ComputeSchemaDiff(currentSchema, newSchema) r := batchApplyResult{ - namespace: nsName, - additions: len(diff.Additions), - conflicts: len(diff.Conflicts), + namespace: nsName, + additions: len(diff.Additions), + modifications: len(diff.Modifications), + conflicts: len(diff.Conflicts), } if diff.HasConflicts() { @@ -349,7 +367,7 @@ func applySchemaToMultiple(ctx context.Context, namespaces []string, newSchema m // Find namespaces to update var toUpdate []int for i, r := range results { - if r.additions > 0 && r.conflicts == 0 && r.err == "" { + if (r.additions > 0 || r.modifications > 0) && r.conflicts == 0 && r.err == "" { toUpdate = append(toUpdate, i) } } @@ -413,7 +431,7 @@ func displayBatchSummary(results []batchApplyResult, dryRun bool) { } else if r.err != "" { changes = "N/A" status = fmt.Sprintf("error: %s", r.err) - } else if r.additions == 0 { + } else if r.additions == 0 && r.modifications == 0 { changes = "no changes" if dryRun { status = "would skip" @@ -421,7 +439,14 @@ func displayBatchSummary(results []batchApplyResult, dryRun bool) { status = "up-to-date" } } else { - changes = fmt.Sprintf("+%d attribute(s)", r.additions) + var parts []string + if r.additions > 0 { + parts = append(parts, fmt.Sprintf("+%d new", r.additions)) + } + if r.modifications > 0 { + parts = append(parts, fmt.Sprintf("~%d modified", r.modifications)) + } + changes = strings.Join(parts, ", ") if dryRun { status = "would apply" } else if r.success { @@ -450,11 +475,23 @@ func writeSchemaErr(ctx context.Context, ns *turbopuffer.Namespace, s map[string schemaParam[k] = toAttributeSchemaParam(v) } + row := turbopuffer.RowParam{"id": "__schema_placeholder__"} + + // If schema has a vector field, include a zero vector to satisfy the constraint. + for attrName, attrType := range s { + baseType := schema.ExtractBaseType(attrType) + if schema.VectorTypePattern.MatchString(baseType) { + dims := schema.ParseVectorDims(baseType) + if dims > 0 { + vec := make([]float32, dims) + row[attrName] = vec + } + } + } + _, err := ns.Write(ctx, turbopuffer.NamespaceWriteParams{ - UpsertRows: []turbopuffer.RowParam{ - {"id": "__schema_placeholder__"}, - }, - Schema: schemaParam, + UpsertRows: []turbopuffer.RowParam{row}, + Schema: schemaParam, }) return err } @@ -471,13 +508,72 @@ func toAttributeSchemaParam(v any) turbopuffer.AttributeSchemaConfigParam { p.Type = t } if fts, ok := val["full_text_search"]; ok { - if b, ok := fts.(bool); ok && b { - p.FullTextSearch = &turbopuffer.FullTextSearchConfigParam{} + switch ftsVal := fts.(type) { + case bool: + if ftsVal { + p.FullTextSearch = &turbopuffer.FullTextSearchConfigParam{} + } + case map[string]any: + ftsParam := &turbopuffer.FullTextSearchConfigParam{} + if v, ok := ftsVal["ascii_folding"].(bool); ok { + ftsParam.AsciiFolding = turbopuffer.Bool(v) + } + if v, ok := ftsVal["b"].(float64); ok && v != 0 { + ftsParam.B = turbopuffer.Float(v) + } + if v, ok := ftsVal["case_sensitive"].(bool); ok { + ftsParam.CaseSensitive = turbopuffer.Bool(v) + } + if v, ok := ftsVal["k1"].(float64); ok && v != 0 { + ftsParam.K1 = turbopuffer.Float(v) + } + if v, ok := ftsVal["remove_stopwords"].(bool); ok { + ftsParam.RemoveStopwords = turbopuffer.Bool(v) + } + if v, ok := ftsVal["stemming"].(bool); ok { + ftsParam.Stemming = turbopuffer.Bool(v) + } + if v, ok := ftsVal["language"].(string); ok && v != "" { + ftsParam.Language = turbopuffer.Language(v) + } + if v, ok := ftsVal["max_token_length"].(float64); ok && v != 0 { + ftsParam.MaxTokenLength = turbopuffer.Int(int64(v)) + } + if v, ok := ftsVal["tokenizer"].(string); ok && v != "" { + ftsParam.Tokenizer = turbopuffer.Tokenizer(v) + } + // Only set if there's actual config (not all zeroes) + hasConfig := ftsParam.B.Valid() || ftsParam.K1.Valid() || + ftsParam.Language != "" || ftsParam.Tokenizer != "" || + ftsParam.MaxTokenLength.Valid() + if hasConfig { + p.FullTextSearch = ftsParam + } } } if f, ok := val["filterable"].(bool); ok { p.Filterable = turbopuffer.Bool(f) } + if ann, ok := val["ann"]; ok { + switch a := ann.(type) { + case bool: + if a { + p.Ann = param.Override[turbopuffer.AttributeSchemaConfigAnnParam](true) + } + case map[string]any: + if dm, ok := a["distance_metric"].(string); ok && dm != "" { + p.Ann = turbopuffer.AttributeSchemaConfigAnnParam{ + DistanceMetric: turbopuffer.DistanceMetric(dm), + } + } + } + } + if g, ok := val["glob"].(bool); ok { + p.Glob = turbopuffer.Bool(g) + } + if r, ok := val["regex"].(bool); ok { + p.Regex = turbopuffer.Bool(r) + } return p default: return turbopuffer.AttributeSchemaConfigParam{ diff --git a/cmd/search.go b/cmd/search.go index 92c6636..2328ee7 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -82,6 +82,9 @@ func runSearch(cmd *cobra.Command, args []string) error { var queryParams turbopuffer.NamespaceQueryParams queryParams.TopK = turbopuffer.Int(int64(topK)) + queryParams.Consistency = turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + } if useFTS { output.StatusPrint(fmt.Sprintf("Mode: Full-text search (BM25) on field \"%s\"\n", ftsField)) diff --git a/internal/schema/schema.go b/internal/schema/schema.go index ca3ea0a..7798774 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -11,10 +11,13 @@ import ( // Valid simple schema types. var ValidSimpleTypes = map[string]bool{ - "string": true, - "uint64": true, - "uuid": true, - "bool": true, + "string": true, + "[]string": true, + "uint64": true, + "int": true, + "float": true, + "uuid": true, + "bool": true, } // VectorTypePattern matches vector types like [1536]f32 or [768]f16. @@ -26,13 +29,30 @@ var ValidTypeKeys = map[string]bool{ "full_text_search": true, "regex_index": true, "filterable": true, + "ann": true, + "glob": true, + "regex": true, +} + +// PropertyChange represents a single property change within an attribute. +type PropertyChange struct { + Key string + OldValue string + NewValue string +} + +// AttrModification holds the details of a modified attribute. +type AttrModification struct { + BaseType string + Changes []PropertyChange } // SchemaDiff holds the result of comparing two schemas. type SchemaDiff struct { - Unchanged map[string]string // attr -> display type - Additions map[string]string // attr -> display type - Conflicts map[string][2]string // attr -> [old_type, new_type] + Unchanged map[string]string // attr -> display type + Additions map[string]string // attr -> display type (new fields) + Modifications map[string]AttrModification // attr -> modification details + Conflicts map[string][2]string // attr -> [old_type, new_type] (base type change — blocked) } // HasConflicts returns true if there are any type conflicts. @@ -40,9 +60,14 @@ func (d *SchemaDiff) HasConflicts() bool { return len(d.Conflicts) > 0 } -// HasChanges returns true if there are any additions or conflicts. +// HasModifications returns true if there are any property modifications. +func (d *SchemaDiff) HasModifications() bool { + return len(d.Modifications) > 0 +} + +// HasChanges returns true if there are any additions, modifications, or conflicts. func (d *SchemaDiff) HasChanges() bool { - return len(d.Additions) > 0 || len(d.Conflicts) > 0 + return len(d.Additions) > 0 || len(d.Modifications) > 0 || len(d.Conflicts) > 0 } // NormalizeSchemaType normalizes a schema type to a comparable string. @@ -130,9 +155,14 @@ func ValidateSchemaType(attrName string, attrType any) []string { // Validate specific option types if fts, ok := v["full_text_search"]; ok { - if _, isBool := fts.(bool); !isBool { + switch fts.(type) { + case bool: + // ok + case map[string]any: + // ok — full config object from API + default: errors = append(errors, fmt.Sprintf( - "Attribute '%s': 'full_text_search' must be a boolean", attrName)) + "Attribute '%s': 'full_text_search' must be a boolean or object", attrName)) } } if ri, ok := v["regex_index"]; ok { @@ -169,16 +199,163 @@ func ValidateSchema(schemaData map[string]any) []string { errors = append(errors, ValidateSchemaType(attrName, attrType)...) } + // The "id" attribute must have filterable: true if present. + if idType, ok := schemaData["id"]; ok { + if m, isMap := idType.(map[string]any); isMap { + if f, hasFilt := m["filterable"]; hasFilt { + if b, isBool := f.(bool); isBool && !b { + errors = append(errors, "Attribute 'id': must have \"filterable\" set to true") + } + } + } + } + + // Vector attributes must have ann: true. + for attrName, attrType := range schemaData { + m, isMap := attrType.(map[string]any) + if !isMap { + continue + } + baseType, _ := m["type"].(string) + if !VectorTypePattern.MatchString(baseType) { + continue + } + ann, hasAnn := m["ann"] + if !hasAnn { + errors = append(errors, fmt.Sprintf( + "Attribute '%s': vector type must have \"ann\" set to true", attrName)) + } else if b, isBool := ann.(bool); isBool && !b { + errors = append(errors, fmt.Sprintf( + "Attribute '%s': vector type must have \"ann\" set to true", attrName)) + } + } + return errors } +// ExtractBaseType returns the base type string from a schema attribute value. +// For simple strings like "string", returns the string itself. +// For complex objects like {"type":"string","filterable":true}, returns "string". +func ExtractBaseType(attrType any) string { + switch v := attrType.(type) { + case string: + return v + case map[string]any: + if t, ok := v["type"].(string); ok { + return t + } + return "" + default: + return fmt.Sprintf("%v", v) + } +} + +// ParseVectorDims extracts the dimension count from a vector type string like "[768]f32". +func ParseVectorDims(vectorType string) int { + matches := VectorTypePattern.FindStringSubmatch(vectorType) + if matches == nil { + return 0 + } + // Extract number between brackets + start := 1 // skip '[' + end := strings.Index(vectorType, "]") + if end <= start { + return 0 + } + var dims int + fmt.Sscanf(vectorType[start:end], "%d", &dims) + return dims +} + +// annEnabled returns true if the ann value represents "enabled" (either bool true or a config object). +func annEnabled(v any) bool { + switch a := v.(type) { + case bool: + return a + case map[string]any: + // Any non-empty object means ann is enabled + return len(a) > 0 + } + return false +} + +// computePropertyChanges compares two attribute maps and returns the changed properties. +// attrName is used for special-case handling (e.g. "id"). +func computePropertyChanges(attrName string, oldMap, newMap map[string]any) []PropertyChange { + var changes []PropertyChange + // Collect all keys + allKeys := make(map[string]bool) + for k := range oldMap { + allKeys[k] = true + } + for k := range newMap { + allKeys[k] = true + } + + sorted := make([]string, 0, len(allKeys)) + for k := range allKeys { + sorted = append(sorted, k) + } + sort.Strings(sorted) + + for _, k := range sorted { + if k == "type" { + continue // base type is shown separately + } + + // The "id" attribute is always filterable regardless of what the API reports. + if attrName == "id" && k == "filterable" { + continue + } + + // For "ann", compare by enabled/disabled rather than exact representation, + // since the API returns an object but writes accept a boolean. + if k == "ann" { + if annEnabled(oldMap[k]) != annEnabled(newMap[k]) { + oldVal, _ := json.Marshal(oldMap[k]) + newVal, _ := json.Marshal(newMap[k]) + changes = append(changes, PropertyChange{ + Key: k, + OldValue: string(oldVal), + NewValue: string(newVal), + }) + } + continue + } + + oldVal, _ := json.Marshal(oldMap[k]) + newVal, _ := json.Marshal(newMap[k]) + if string(oldVal) != string(newVal) { + changes = append(changes, PropertyChange{ + Key: k, + OldValue: string(oldVal), + NewValue: string(newVal), + }) + } + } + return changes +} + +// toMap normalizes an attribute type to a map for property comparison. +func toMap(attrType any) map[string]any { + switch v := attrType.(type) { + case map[string]any: + return v + case string: + return map[string]any{"type": v} + default: + return map[string]any{"type": fmt.Sprintf("%v", v)} + } +} + // ComputeSchemaDiff computes the difference between current and new schemas. // currentSchema can be nil (namespace doesn't exist yet). func ComputeSchemaDiff(currentSchema, newSchema map[string]any) SchemaDiff { diff := SchemaDiff{ - Unchanged: make(map[string]string), - Additions: make(map[string]string), - Conflicts: make(map[string][2]string), + Unchanged: make(map[string]string), + Additions: make(map[string]string), + Modifications: make(map[string]AttrModification), + Conflicts: make(map[string][2]string), } if currentSchema == nil { @@ -203,7 +380,24 @@ func ComputeSchemaDiff(currentSchema, newSchema map[string]any) SchemaDiff { diff.Unchanged[attr] = newTypeDisplay } else { oldTypeDisplay := SchemaTypeForDisplay(currentSchema[attr]) - diff.Conflicts[attr] = [2]string{oldTypeDisplay, newTypeDisplay} + oldBase := ExtractBaseType(currentSchema[attr]) + newBase := ExtractBaseType(newType) + if oldBase == newBase || oldBase == "unknown" || oldBase == "[]unknown" { + // Same base type, different properties — check for meaningful changes + changes := computePropertyChanges(attr, toMap(currentSchema[attr]), toMap(newType)) + if len(changes) > 0 { + diff.Modifications[attr] = AttrModification{ + BaseType: newBase, + Changes: changes, + } + } else { + // All differences were suppressed (e.g. ann normalization) + diff.Unchanged[attr] = newTypeDisplay + } + } else { + // Different base type — conflict (blocked) + diff.Conflicts[attr] = [2]string{oldTypeDisplay, newTypeDisplay} + } } } diff --git a/internal/tui/documents.go b/internal/tui/documents.go index 3f105be..06f862c 100644 --- a/internal/tui/documents.go +++ b/internal/tui/documents.go @@ -98,6 +98,9 @@ func fetchDocuments(namespace, region string, topK int, schemaDict map[string]an RankBy: turbopuffer.NewRankByVector(vecAttr, zeroVec), TopK: turbopuffer.Int(int64(topK)), ExcludeAttributes: []string{vecAttr}, + Consistency: turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + }, }) if err != nil { return documentsErrMsg{err: err} @@ -121,6 +124,9 @@ func runFTSSearch(namespace, region, field, query string, topK int, schemaDict m params := turbopuffer.NamespaceQueryParams{ TopK: turbopuffer.Int(int64(topK)), RankBy: turbopuffer.NewRankByTextBM25(field, query), + Consistency: turbopuffer.NamespaceQueryParamsConsistency{ + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + }, } if vecAttr, _ := extractVectorInfo(schemaDict); vecAttr != "" { params.ExcludeAttributes = []string{vecAttr} diff --git a/internal/tui/helpers.go b/internal/tui/helpers.go index a83adb3..a337d3f 100644 --- a/internal/tui/helpers.go +++ b/internal/tui/helpers.go @@ -206,14 +206,34 @@ func formatUpdatedAt(t time.Time) string { // ftsFields returns the list of attribute names in the schema that have // full_text_search configured (in the order they appear in the schema map). +// ftsEnabled checks whether a full_text_search value represents an actually +// enabled FTS config (not just a zeroed-out default object from the API). +func ftsEnabled(v any) bool { + switch fts := v.(type) { + case bool: + return fts + case map[string]any: + // The API returns a zeroed-out object for all fields; only consider it + // enabled if tokenizer or language is set (non-empty string). + if t, _ := fts["tokenizer"].(string); t != "" { + return true + } + if l, _ := fts["language"].(string); l != "" { + return true + } + return false + } + return false +} + func ftsFields(schemaData map[string]any) []string { var out []string for name, cfg := range schemaData { has := false switch v := cfg.(type) { case map[string]any: - if _, ok := v["full_text_search"]; ok { - has = true + if ftsVal, ok := v["full_text_search"]; ok { + has = ftsEnabled(ftsVal) } default: if b, err := json.Marshal(v); err == nil { @@ -221,7 +241,7 @@ func ftsFields(schemaData map[string]any) []string { FullTextSearch any `json:"full_text_search"` } if json.Unmarshal(b, &obj) == nil && obj.FullTextSearch != nil { - has = true + has = ftsEnabled(obj.FullTextSearch) } } } diff --git a/internal/tui/namespaces.go b/internal/tui/namespaces.go index 18f8785..d0ba610 100644 --- a/internal/tui/namespaces.go +++ b/internal/tui/namespaces.go @@ -171,15 +171,16 @@ func (m namespacesModel) view(width, height int) string { rowsW := 10 sizeW := 12 statusW := 14 + unindexedW := 12 updatedW := 16 // Adapt name width to terminal if width > 100 { - nameW = width - rowsW - sizeW - statusW - updatedW - 16 + nameW = width - rowsW - sizeW - statusW - unindexedW - updatedW - 18 } - header := fmt.Sprintf(" %-*s %*s %*s %-*s %*s", - nameW, "Name", rowsW, "Rows", sizeW, "Size", statusW, "Index Status", updatedW, "Updated") + header := fmt.Sprintf(" %-*s %*s %*s %-*s %*s %*s", + nameW, "Name", rowsW, "Rows", sizeW, "Size", statusW, "Index Status", unindexedW, "Unindexed", updatedW, "Updated") b.WriteString(columnHeaderStyle.Render(header)) b.WriteString("\n") @@ -204,15 +205,20 @@ func (m namespacesModel) view(width, height int) string { status := "N/A" updated := "N/A" + unindexed := "" + if item.Metadata != nil { rows = formatNumber(item.Metadata.ApproxRowCount) size = formatBytes(item.Metadata.ApproxLogicalBytes) status = metadata.GetIndexStatus(item.Metadata) updated = formatUpdatedAt(item.Metadata.UpdatedAt) + if ub := metadata.GetUnindexedBytes(item.Metadata); ub > 0 { + unindexed = formatBytes(ub) + } } - line := fmt.Sprintf(" %-*s %*s %*s %-*s %*s", - nameW, name, rowsW, rows, sizeW, size, statusW, status, updatedW, updated) + line := fmt.Sprintf(" %-*s %*s %*s %-*s %*s %*s", + nameW, name, rowsW, rows, sizeW, size, statusW, status, unindexedW, unindexed, updatedW, updated) if i == m.cursor { b.WriteString(selectedStyle.Render("▸ " + line[2:])) diff --git a/internal/tui/schema.go b/internal/tui/schema.go index 4d61bf9..6b81fd3 100644 --- a/internal/tui/schema.go +++ b/internal/tui/schema.go @@ -84,7 +84,7 @@ func parseSchemaAttrs(schema map[string]any) []schemaAttr { } else { a.filterable = "-" } - if _, ok := v["full_text_search"]; ok { + if ftsVal, ok := v["full_text_search"]; ok && ftsEnabled(ftsVal) { a.fts = "yes" } else { a.fts = "-" @@ -103,7 +103,7 @@ func parseSchemaAttrs(schema map[string]any) []schemaAttr { } else { a.filterable = "-" } - if obj.FullTextSearch != nil { + if obj.FullTextSearch != nil && ftsEnabled(obj.FullTextSearch) { a.fts = "yes" } else { a.fts = "-" From 876e2bb84c63df7375c85c84597cec5aadd07916 Mon Sep 17 00:00:00 2001 From: Adam Hevenor Date: Wed, 6 May 2026 07:33:27 -0600 Subject: [PATCH 2/3] Use strong consistency for scan queries Scan iterates through all documents in order, so strong consistency is more appropriate than eventual to avoid missing recently written data. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/scan.go b/cmd/scan.go index 1d82e6d..a87daa4 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -60,7 +60,7 @@ func runScan(cmd *cobra.Command, args []string) error { IncludeAttributes: turbopuffer.IncludeAttributesParam{StringArray: []string{field}}, TopK: turbopuffer.Int(int64(pageSize)), Consistency: turbopuffer.NamespaceQueryParamsConsistency{ - Level: turbopuffer.NamespaceQueryParamsConsistencyLevelEventual, + Level: turbopuffer.NamespaceQueryParamsConsistencyLevelStrong, }, } From 9a20dbff4f94c4bc95c70bd88202831057c18e70 Mon Sep 17 00:00:00 2001 From: Adam Hevenor Date: Thu, 7 May 2026 14:06:15 -0600 Subject: [PATCH 3/3] Auto-refresh namespace list every 30 seconds in TUI Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/tui/model.go | 5 ++++ internal/tui/namespaces.go | 47 +++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/internal/tui/model.go b/internal/tui/model.go index fd0b258..4164cfe 100644 --- a/internal/tui/model.go +++ b/internal/tui/model.go @@ -139,6 +139,11 @@ func (m Model) updateEnvs(msg tea.Msg) (tea.Model, tea.Cmd) { func (m Model) updateNamespaces(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { + case namespacesTickMsg: + if !m.namespaces.loading { + return m, m.namespaces.refresh(m.region) + } + return m, namespacesTickCmd() case tea.KeyMsg: switch { case msg.String() == "q" && !m.namespaces.filtering: diff --git a/internal/tui/namespaces.go b/internal/tui/namespaces.go index d0ba610..2aba5fc 100644 --- a/internal/tui/namespaces.go +++ b/internal/tui/namespaces.go @@ -11,15 +11,23 @@ import ( "github.com/hev/tpuff/internal/metadata" ) +const namespacesRefreshInterval = 30 * time.Second + // Messages type namespacesLoadedMsg struct { items []metadata.NamespaceWithMetadata } +type namespacesRefreshedMsg struct { + items []metadata.NamespaceWithMetadata +} + type namespacesErrMsg struct { err error } +type namespacesTickMsg struct{} + type namespacesModel struct { items []metadata.NamespaceWithMetadata filtered []int // indices into items @@ -57,6 +65,34 @@ func (m namespacesModel) init(region string) tea.Cmd { } } +func namespacesTickCmd() tea.Cmd { + return tea.Tick(namespacesRefreshInterval, func(time.Time) tea.Msg { + return namespacesTickMsg{} + }) +} + +func (m namespacesModel) refresh(region string) tea.Cmd { + return func() tea.Msg { + ctx := context.Background() + items := metadata.FetchNamespacesWithMetadata(ctx, false, region, false) + if items == nil { + return nil // silently ignore refresh errors + } + sort.Slice(items, func(i, j int) bool { + ti := time.Time{} + tj := time.Time{} + if items[i].Metadata != nil { + ti = items[i].Metadata.UpdatedAt + } + if items[j].Metadata != nil { + tj = items[j].Metadata.UpdatedAt + } + return ti.After(tj) + }) + return namespacesRefreshedMsg{items: items} + } +} + func (m namespacesModel) selected() *metadata.NamespaceWithMetadata { indices := m.visibleIndices() if m.cursor < 0 || m.cursor >= len(indices) { @@ -98,7 +134,16 @@ func (m namespacesModel) update(msg tea.Msg) (namespacesModel, tea.Cmd) { case namespacesLoadedMsg: m.items = msg.items m.loading = false - return m, nil + m.applyFilter() + return m, namespacesTickCmd() + case namespacesRefreshedMsg: + m.items = msg.items + m.applyFilter() + visible := m.visibleIndices() + if m.cursor >= len(visible) { + m.cursor = max(0, len(visible)-1) + } + return m, namespacesTickCmd() case namespacesErrMsg: m.err = msg.err m.loading = false