From 7ae3fb3d1cdd9ce38e9bebe0fa3deb009910ffd7 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sat, 13 Jun 2026 22:27:23 +0300 Subject: [PATCH 01/13] feat(deps): collapse duplicate dependencies to the resolved node Render each shared dependency once at its resolved (shallowest BFS) occurrence instead of repeating it under every parent. The resolved node is tagged with the count of other parents and each parent that hid a duplicate gets a trailing "(and N other dependencies)" marker. The model and JSON carry the new other_parents/hidden_duplicates counts. --show-duplicates restores the full per-parent repetition with dup:N tags. Stats, --flat node/edge lists, and deps diff stay on the full uncollapsed graph (collapse is forced off internally for diffs). --- cmd/repomap/deps.go | 20 ++++--- deps/collapse.go | 75 ++++++++++++++++++++++++++ deps/collapse_test.go | 114 +++++++++++++++++++++++++++++++++++++++ deps/compare_scan.go | 4 ++ deps/go_graph_test.go | 86 +++++++++++++++++++++++++++++ deps/model.go | 36 +++++++------ deps/pretty.go | 14 ++++- deps/pretty_tree.go | 27 +++++++--- deps/pretty_tree_test.go | 46 ++++++++++++++++ deps/scan.go | 4 ++ 10 files changed, 396 insertions(+), 30 deletions(-) create mode 100644 deps/collapse.go create mode 100644 deps/collapse_test.go diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index 51a4a74..ba3e8fc 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -11,12 +11,13 @@ import ( ) type DepsOptions struct { - Path string `json:"path" args:"true" help:"Path to scan" default:"."` - Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` - Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` - Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` - Flat bool `json:"flat,omitempty" flag:"flat" help:"Export a flat node list with edges instead of the dependency tree"` - IncludeIndirect bool `json:"include_indirect,omitempty" flag:"include-indirect" help:"Include Go indirect requirements in --depth 1 listings (ignored at other depths)"` + Path string `json:"path" args:"true" help:"Path to scan" default:"."` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` + Flat bool `json:"flat,omitempty" flag:"flat" help:"Export a flat node list with edges instead of the dependency tree"` + IncludeIndirect bool `json:"include_indirect,omitempty" flag:"include-indirect" help:"Include Go indirect requirements in --depth 1 listings (ignored at other depths)"` + ShowDuplicates bool `json:"show_duplicates,omitempty" flag:"show-duplicates" help:"Render every occurrence of duplicated dependencies instead of collapsing them to the resolved node"` } type DepsUpdateOptions struct { @@ -51,12 +52,18 @@ JSON to stdout, for example: By default the JSON export contains the dependency tree under "roots". Use --flat to export a flat "nodes" list plus "edges" instead of the tree. +Shared dependencies are collapsed to their resolved (shallowest) occurrence: the +resolved node is tagged with the number of other parents and each parent that +hid a duplicate shows a trailing count. Use --show-duplicates to render every +occurrence instead. + EXAMPLES: repomap deps repomap deps ./service --manager go repomap deps --manager npm,pnpm --depth 0 repomap deps --manager go --depth 0 --flat --json repomap deps --manager go --include-indirect + repomap deps --depth 0 --manager go --show-duplicates repomap deps --manager image,helm ./clusters/prod repomap deps --filter 'github.com/flanksource/*,!*test*'`) } @@ -111,6 +118,7 @@ func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { Filters: splitCommaArgs(opts.Filter), Flat: opts.Flat, IncludeIndirect: opts.IncludeIndirect, + ShowDuplicates: opts.ShowDuplicates, }) } diff --git a/deps/collapse.go b/deps/collapse.go new file mode 100644 index 0000000..a5683ad --- /dev/null +++ b/deps/collapse.go @@ -0,0 +1,75 @@ +package deps + +// collapseDuplicates rewrites package-manager roots in place so each dependency +// renders once at its resolved (shallowest) location instead of repeating under +// every parent. Image/Helm roots are left untouched: repeated images are real +// distinct deployments and the kubernetes display drops the root that would +// carry a collapse marker. +func collapseDuplicates(roots []*Node) { + for _, root := range roots { + if root == nil || !isPackageManager(root.Manager) { + continue + } + collapseRoot(root) + } +} + +func isPackageManager(manager Manager) bool { + switch manager { + case ManagerGo, ManagerMaven, ManagerGradle, ManagerNPM, ManagerPNPM: + return true + } + return false +} + +// collapseRoot keeps the first BFS sighting (shallowest, then sorted) of each +// node ID within a single root, drops later sightings, and records the counts: +// the resolved node's OtherParents and each parent's HiddenDuplicates. +func collapseRoot(root *Node) { + refs := map[string]int{} + countRefs(root, refs) + sortTree(root) + + seen := map[string]bool{} + queue := []*Node{root} + for len(queue) > 0 { + parent := queue[0] + queue = queue[1:] + kept := parent.Children[:0] + hidden := 0 + for _, child := range parent.Children { + if seen[child.ID] { + hidden++ + continue + } + seen[child.ID] = true + if refs[child.ID] > 1 { + child.OtherParents = refs[child.ID] - 1 + } + kept = append(kept, child) + queue = append(queue, child) + } + parent.Children = kept + parent.HiddenDuplicates = hidden + } +} + +func countRefs(node *Node, refs map[string]int) { + if node == nil { + return + } + for _, child := range node.Children { + refs[child.ID]++ + countRefs(child, refs) + } +} + +func sortTree(node *Node) { + if node == nil { + return + } + sortChildren(node) + for _, child := range node.Children { + sortTree(child) + } +} diff --git a/deps/collapse_test.go b/deps/collapse_test.go new file mode 100644 index 0000000..da97284 --- /dev/null +++ b/deps/collapse_test.go @@ -0,0 +1,114 @@ +package deps + +import "testing" + +// diamondRoot builds a go project where a shared dependency is reachable under +// three direct parents, the classic diamond that motivates collapsing. +// +// app +// ├── a ── shared@v1 +// ├── b ── shared@v1 +// └── c ── shared@v1 +func diamondRoot() *Node { + root := NewNode(ManagerGo, "github.com/acme/app", "") + root.Source = "go.mod" + shared := func() *Node { + n := NewNode(ManagerGo, "github.com/acme/shared", "v1.0.0") + n.Depth = 2 + return n + } + for _, name := range []string{"a", "b", "c"} { + parent := NewNode(ManagerGo, "github.com/acme/"+name, "v1.0.0") + parent.Direct = true + parent.Depth = 1 + parent.Children = []*Node{shared()} + root.Children = append(root.Children, parent) + } + return root +} + +func TestCollapseKeepsSharedDependencyOnce(t *testing.T) { + root := diamondRoot() + collapseDuplicates([]*Node{root}) + + var kept int + for _, parent := range root.Children { + if findChild(parent, "github.com/acme/shared") != nil { + kept++ + } + } + if kept != 1 { + t.Fatalf("expected shared dependency to survive under exactly one parent, got %d", kept) + } +} + +func TestCollapseMarksOtherParentsAndHiddenCounts(t *testing.T) { + root := diamondRoot() + collapseDuplicates([]*Node{root}) + + a := findChild(root, "github.com/acme/a") + shared := findChild(a, "github.com/acme/shared") + if shared == nil { + t.Fatalf("shared should be retained under the first sorted parent 'a'") + } + if shared.OtherParents != 2 { + t.Fatalf("resolved shared node should record 2 other parents, got %d", shared.OtherParents) + } + + for _, name := range []string{"github.com/acme/b", "github.com/acme/c"} { + parent := findChild(root, name) + if parent.HiddenDuplicates != 1 { + t.Fatalf("parent %s should hide 1 duplicate, got %d", name, parent.HiddenDuplicates) + } + if findChild(parent, "github.com/acme/shared") != nil { + t.Fatalf("parent %s should no longer carry the shared dependency", name) + } + } +} + +func TestCollapseLeavesImageRootsUntouched(t *testing.T) { + root := NewNode(ManagerImage, "container images", "") + dup := func() *Node { + n := NewNode(ManagerImage, "nginx", "1.25") + n.Depth = 1 + return n + } + root.Children = []*Node{dup(), dup()} + collapseDuplicates([]*Node{root}) + + if len(root.Children) != 2 { + t.Fatalf("image roots must keep every occurrence, got %d children", len(root.Children)) + } + if root.HiddenDuplicates != 0 { + t.Fatalf("image root should not record hidden duplicates, got %d", root.HiddenDuplicates) + } +} + +func TestCollapseIsolatesPerRoot(t *testing.T) { + first := diamondRoot() + second := diamondRoot() + collapseDuplicates([]*Node{first, second}) + + for _, root := range []*Node{first, second} { + a := findChild(root, "github.com/acme/a") + if shared := findChild(a, "github.com/acme/shared"); shared == nil || shared.OtherParents != 2 { + t.Fatalf("each root should dedup independently with OtherParents=2") + } + } +} + +func TestCollapseKeepsConflictingVersions(t *testing.T) { + root := NewNode(ManagerGo, "github.com/acme/app", "") + v1 := NewNode(ManagerGo, "github.com/acme/lib", "v1.0.0") + v1.Direct = true + v1.Depth = 1 + v2 := NewNode(ManagerGo, "github.com/acme/lib", "v2.0.0") + v2.Direct = true + v2.Depth = 1 + root.Children = []*Node{v1, v2} + collapseDuplicates([]*Node{root}) + + if len(root.Children) != 2 { + t.Fatalf("conflicting versions must each survive, got %d", len(root.Children)) + } +} diff --git a/deps/compare_scan.go b/deps/compare_scan.go index 27776f3..4169583 100644 --- a/deps/compare_scan.go +++ b/deps/compare_scan.go @@ -41,6 +41,10 @@ func CompareScan(ctx context.Context, path string, opts CompareOptions) (*Compar return nil, err } + // Diffs always compare the full graph: indexExport walks Roots, so collapsing + // duplicate subtrees would hide real dependency changes. + opts.ShowDuplicates = true + baseSha, err := resolveRef(ctx, repoRoot, opts.BaseRef) if err != nil { return nil, err diff --git a/deps/go_graph_test.go b/deps/go_graph_test.go index 6ec3051..8333318 100644 --- a/deps/go_graph_test.go +++ b/deps/go_graph_test.go @@ -116,6 +116,92 @@ require github.com/acme/lib v1.2.3 } } +// goDiamondScan resolves a go module whose two direct dependencies both require +// the same transitive dep, exercising the collapse pass end-to-end. +func goDiamondScan(t *testing.T, opts Options) *Export { + t.Helper() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require ( + github.com/acme/lib v1.0.0 + github.com/acme/other v1.0.0 +) +`) + opts.Managers = []Manager{ManagerGo} + opts.MaxDepth = 0 + opts.Runner = &fakeRunner{result: CommandResult{Stdout: strings.Join([]string{ + "github.com/acme/app github.com/acme/lib@v1.0.0", + "github.com/acme/app github.com/acme/other@v1.0.0", + "github.com/acme/lib@v1.0.0 github.com/acme/dep@v0.1.0", + "github.com/acme/other@v1.0.0 github.com/acme/dep@v0.1.0", + "", + }, "\n")}} + opts.Now = func() time.Time { return time.Unix(1, 0).UTC() } + got, err := Scan(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + return got +} + +func TestScanCollapsesDuplicatesByDefault(t *testing.T) { + got := goDiamondScan(t, Options{}) + root := got.Roots[0] + lib := findChild(root, "github.com/acme/lib") + other := findChild(root, "github.com/acme/other") + + depUnderLib := findChild(lib, "github.com/acme/dep") + depUnderOther := findChild(other, "github.com/acme/dep") + if (depUnderLib == nil) == (depUnderOther == nil) { + t.Fatalf("dep should render under exactly one parent, lib=%v other=%v", depUnderLib != nil, depUnderOther != nil) + } + resolved, hiddenParent := depUnderLib, other + if depUnderLib == nil { + resolved, hiddenParent = depUnderOther, lib + } + if resolved.OtherParents != 1 { + t.Fatalf("resolved dep should record 1 other parent, got %d", resolved.OtherParents) + } + if hiddenParent.HiddenDuplicates != 1 { + t.Fatalf("the parent that lost dep should hide 1 duplicate, got %d", hiddenParent.HiddenDuplicates) + } +} + +func TestScanShowDuplicatesRendersEveryOccurrence(t *testing.T) { + got := goDiamondScan(t, Options{ShowDuplicates: true}) + root := got.Roots[0] + lib := findChild(root, "github.com/acme/lib") + other := findChild(root, "github.com/acme/other") + if findChild(lib, "github.com/acme/dep") == nil || findChild(other, "github.com/acme/dep") == nil { + t.Fatalf("--show-duplicates should keep dep under both parents") + } + if !got.Metadata.ShowDuplicates { + t.Fatalf("metadata should record show_duplicates") + } +} + +func TestScanFlatStaysFullGraph(t *testing.T) { + collapsed := goDiamondScan(t, Options{}) + flat := goDiamondScan(t, Options{Flat: true}) + // The flat export counts every node once (dedup by id), independent of the + // tree collapse, so dep is present exactly once and edges cover both parents. + var depEdges int + for _, e := range flat.Edges { + if strings.HasSuffix(e.To, "acme/dep@v0.1.0") { + depEdges++ + } + } + if depEdges != 2 { + t.Fatalf("flat edges should retain both parent→dep edges, got %d", depEdges) + } + if len(collapsed.Roots) == 0 || len(flat.Nodes) == 0 { + t.Fatalf("expected collapsed roots and flat nodes to be populated") + } +} + func TestScanGoDepthOneStaysOffline(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app diff --git a/deps/model.go b/deps/model.go index 02f1a20..26e60e3 100644 --- a/deps/model.go +++ b/deps/model.go @@ -27,6 +27,7 @@ type Options struct { Filters []string Flat bool IncludeIndirect bool + ShowDuplicates bool Runner CommandRunner Now func() time.Time } @@ -57,26 +58,29 @@ type Metadata struct { Filter []string `json:"filter,omitempty"` MaxDepth int `json:"max_depth,omitempty"` Flat bool `json:"flat,omitempty"` + ShowDuplicates bool `json:"show_duplicates,omitempty"` ProjectsScanned int `json:"projects_scanned"` } type Node struct { - ID string `json:"id"` - Name string `json:"name"` - Version string `json:"version,omitempty"` - Manager Manager `json:"manager"` - Scope string `json:"scope,omitempty"` - Source string `json:"source,omitempty"` - Path string `json:"path,omitempty"` - Direct bool `json:"direct,omitempty"` - Dev bool `json:"dev,omitempty"` - Optional bool `json:"optional,omitempty"` - Local bool `json:"local,omitempty"` - Depth int `json:"depth"` - Circular bool `json:"circular,omitempty"` - Duplicate *DupRef `json:"duplicate,omitempty"` - Children []*Node `json:"children,omitempty"` - properties map[string]string + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Manager Manager `json:"manager"` + Scope string `json:"scope,omitempty"` + Source string `json:"source,omitempty"` + Path string `json:"path,omitempty"` + Direct bool `json:"direct,omitempty"` + Dev bool `json:"dev,omitempty"` + Optional bool `json:"optional,omitempty"` + Local bool `json:"local,omitempty"` + Depth int `json:"depth"` + Circular bool `json:"circular,omitempty"` + Duplicate *DupRef `json:"duplicate,omitempty"` + OtherParents int `json:"other_parents,omitempty"` + HiddenDuplicates int `json:"hidden_duplicates,omitempty"` + Children []*Node `json:"children,omitempty"` + properties map[string]string } type FlatNode struct { diff --git a/deps/pretty.go b/deps/pretty.go index ca0ab55..d7fb604 100644 --- a/deps/pretty.go +++ b/deps/pretty.go @@ -208,7 +208,12 @@ func statusTags(node *Node) []dependencyTag { if node.Circular { tags = append(tags, dependencyTag{label: "circular", style: "font-bold text-red-600"}) } - if node.Duplicate != nil { + if node.OtherParents > 0 { + tags = append(tags, dependencyTag{label: parentMarker(node.OtherParents), style: "text-cyan-600"}) + if node.Duplicate != nil && node.Duplicate.Conflicts { + tags = append(tags, dependencyTag{label: "conflict", style: "font-bold text-red-600"}) + } + } else if node.Duplicate != nil { tag := fmt.Sprintf("dup:%d", node.Duplicate.Count) style := "text-orange-500" if node.Duplicate.Conflicts { @@ -220,6 +225,13 @@ func statusTags(node *Node) []dependencyTag { return tags } +func parentMarker(other int) string { + if other == 1 { + return "+1 parent" + } + return fmt.Sprintf("+%d parents", other) +} + func sortTags(tags []dependencyTag) []dependencyTag { sort.Slice(tags, func(i, j int) bool { return tags[i].label < tags[j].label diff --git a/deps/pretty_tree.go b/deps/pretty_tree.go index 2a346a9..4b78e2d 100644 --- a/deps/pretty_tree.go +++ b/deps/pretty_tree.go @@ -1,6 +1,7 @@ package deps import ( + "fmt" "path/filepath" "sort" "strings" @@ -57,23 +58,35 @@ func rootDisplayNode(root *Node, scanPath string) *displayNode { if rel := relDisplayPath(scanPath, root.Path); rel != "" { label = label.Space().Append(rel, "font-mono text-muted") } - return &displayNode{text: label, children: packageDisplayNodes(root.Children)} + return &displayNode{text: label, children: packageChildNodes(root)} } -// packageDisplayNodes renders dependency children without repeating the manager -// prefix (the root already carries it). -func packageDisplayNodes(nodes []*Node) []api.TreeNode { - sorted := sortedNodes(nodes) - out := make([]api.TreeNode, 0, len(sorted)) +// packageChildNodes renders dependency children without repeating the manager +// prefix (the root already carries it) and, when duplicate occurrences were +// collapsed away, appends a trailing marker counting them. +func packageChildNodes(parent *Node) []api.TreeNode { + sorted := sortedNodes(parent.Children) + out := make([]api.TreeNode, 0, len(sorted)+1) for _, n := range sorted { out = append(out, &displayNode{ text: nodeText(n, false), - children: packageDisplayNodes(n.Children), + children: packageChildNodes(n), }) } + if parent.HiddenDuplicates > 0 { + out = append(out, &displayNode{text: hiddenDuplicatesText(parent.HiddenDuplicates)}) + } return out } +func hiddenDuplicatesText(hidden int) api.Text { + noun := "dependencies" + if hidden == 1 { + noun = "dependency" + } + return clicky.Text(fmt.Sprintf("(and %d other %s)", hidden, noun), "text-muted italic") +} + // namespaceNodes is the top level of the kubernetes grouping. func namespaceNodes(leaves []*Node) []api.TreeNode { keys, groups := groupLeaves(sortedNodes(leaves), func(n *Node) string { diff --git a/deps/pretty_tree_test.go b/deps/pretty_tree_test.go index f7f2f09..d230074 100644 --- a/deps/pretty_tree_test.go +++ b/deps/pretty_tree_test.go @@ -87,6 +87,52 @@ func TestTreeDropsDefaultScopeAndDirectTags(t *testing.T) { } } +func TestTreeShowsCollapseMarkers(t *testing.T) { + root := diamondRoot() + root.Path = "/abs/proj/go.mod" + collapseDuplicates([]*Node{root}) + out := (&Export{Metadata: Metadata{Path: "/abs/proj"}, Roots: []*Node{root}}).Pretty().String() + + sharedLine := prettyLine(out, "github.com/acme/shared@v1.0.0") + if !strings.Contains(sharedLine, "+2 parents") { + t.Fatalf("resolved shared node should be tagged (+2 parents): %q", sharedLine) + } + + if prettyLine(out, "and 1 other dependency)") == "" { + t.Fatalf("a parent that hid one duplicate should show the singular marker:\n%s", out) + } + if strings.Contains(out, "and 1 other dependencies") { + t.Fatalf("singular hidden count must read 'dependency', not 'dependencies':\n%s", out) + } +} + +func TestTreeShowDuplicatesKeepsEveryOccurrence(t *testing.T) { + root := diamondRoot() + root.Path = "/abs/proj/go.mod" + dups := analyzeDuplicates([]*Node{root}) + applyDuplicateRefs([]*Node{root}, dups) + out := (&Export{Metadata: Metadata{Path: "/abs/proj"}, Roots: []*Node{root}}).Pretty().String() + + var occurrences int + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "github.com/acme/shared@v1.0.0") { + occurrences++ + } + } + if occurrences != 3 { + t.Fatalf("--show-duplicates equivalent should render shared 3 times, got %d:\n%s", occurrences, out) + } + if strings.Contains(out, "+2 parents") { + t.Fatalf("uncollapsed tree must not carry parent-count markers:\n%s", out) + } + if strings.Contains(out, "other dependenc") { + t.Fatalf("uncollapsed tree must not carry hidden-duplicate markers:\n%s", out) + } + if !strings.Contains(out, "dup:3") { + t.Fatalf("uncollapsed tree should keep dup:N tags:\n%s", out) + } +} + func imageTreeExport() *Export { root := NewNode(ManagerImage, "container images", "") root.Depth = 0 diff --git a/deps/scan.go b/deps/scan.go index b8e73d8..4feaad2 100644 --- a/deps/scan.go +++ b/deps/scan.go @@ -100,6 +100,7 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { Filter: opts.Filters, MaxDepth: opts.MaxDepth, Flat: opts.Flat, + ShowDuplicates: opts.ShowDuplicates, ProjectsScanned: projectsScanned, }, Statistics: stats, @@ -110,6 +111,9 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { export.Nodes = nodes export.Edges = edges } else { + if !opts.ShowDuplicates { + collapseDuplicates(filteredRoots) + } export.Roots = filteredRoots } return export, nil From b9d446f5e03f314c8aaf1a425510bf216450395b Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 14 Jun 2026 08:20:25 +0300 Subject: [PATCH 02/13] feat(deps): scan Helm chart directories for subcharts and images Discover local Helm chart directories (Chart.yaml) in addition to Flux HelmRelease manifests. Each chart becomes a dependency root carrying its declared subchart dependencies (helm, version recorded verbatim from the Chart.yaml range) plus the container images referenced in values.yaml and templates/ (image). Templated values (`{{ ... }}`) are skipped since they cannot be resolved offline. Chart discovery is a filesystem walk independent of the git-backed k8s-manifest scanner, so a chart directory outside a git repo still resolves; image-discovery errors now degrade to a warning when other roots resolved. Vendored subcharts under a parent chart's charts/ dir are skipped. Chart roots render package-style (manager-prefixed children), distinct from the namespace/kind grouping used for k8s manifest images. --- deps/pretty_tree.go | 21 ++- deps/pretty_tree_test.go | 1 + deps/scan.go | 38 ++-- deps/scan_chart.go | 383 +++++++++++++++++++++++++++++++++++++++ deps/scan_chart_test.go | 188 +++++++++++++++++++ 5 files changed, 611 insertions(+), 20 deletions(-) create mode 100644 deps/scan_chart.go create mode 100644 deps/scan_chart_test.go diff --git a/deps/pretty_tree.go b/deps/pretty_tree.go index 4b78e2d..ed00443 100644 --- a/deps/pretty_tree.go +++ b/deps/pretty_tree.go @@ -36,8 +36,8 @@ func dependencyDisplayTree(roots []*Node, scanPath string) (api.TextTree, bool) if root == nil { continue } - switch root.Manager { - case ManagerImage, ManagerHelm: + switch { + case (root.Manager == ManagerImage || root.Manager == ManagerHelm) && root.Source == "kubernetes manifests": k8sLeaves = append(k8sLeaves, root.Children...) default: top = append(top, rootDisplayNode(root, scanPath)) @@ -58,19 +58,22 @@ func rootDisplayNode(root *Node, scanPath string) *displayNode { if rel := relDisplayPath(scanPath, root.Path); rel != "" { label = label.Space().Append(rel, "font-mono text-muted") } - return &displayNode{text: label, children: packageChildNodes(root)} + // Chart roots mix helm subcharts with image dependencies, so their children + // keep a manager prefix; single-manager package roots omit it. + showChildManager := root.Source == "Chart.yaml" + return &displayNode{text: label, children: packageChildNodes(root, showChildManager)} } -// packageChildNodes renders dependency children without repeating the manager -// prefix (the root already carries it) and, when duplicate occurrences were -// collapsed away, appends a trailing marker counting them. -func packageChildNodes(parent *Node) []api.TreeNode { +// packageChildNodes renders dependency children and, when duplicate occurrences +// were collapsed away, appends a trailing marker counting them. showManager +// keeps the manager prefix on children of roots that mix managers. +func packageChildNodes(parent *Node, showManager bool) []api.TreeNode { sorted := sortedNodes(parent.Children) out := make([]api.TreeNode, 0, len(sorted)+1) for _, n := range sorted { out = append(out, &displayNode{ - text: nodeText(n, false), - children: packageChildNodes(n), + text: nodeText(n, showManager), + children: packageChildNodes(n, showManager), }) } if parent.HiddenDuplicates > 0 { diff --git a/deps/pretty_tree_test.go b/deps/pretty_tree_test.go index d230074..d968fed 100644 --- a/deps/pretty_tree_test.go +++ b/deps/pretty_tree_test.go @@ -137,6 +137,7 @@ func imageTreeExport() *Export { root := NewNode(ManagerImage, "container images", "") root.Depth = 0 root.Path = "/abs/proj" + root.Source = "kubernetes manifests" nginx := NewNode(ManagerImage, "nginx", "1.25.3") nginx.Depth = 1 diff --git a/deps/scan.go b/deps/scan.go index 4feaad2..2a18206 100644 --- a/deps/scan.go +++ b/deps/scan.go @@ -55,28 +55,44 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { roots = append(roots, projectRoots...) } + var imageErr error if scanImages { - imageRoots, imageWarnings, err := discoverImageDependencyRoots(absPath, imageScanManagers(opts.Managers)) - warnings = append(warnings, imageWarnings...) - if err != nil { - if len(roots) == 0 && packageErr != nil { - return nil, packageErr - } - if len(roots) == 0 { - return nil, err - } - warnings = append(warnings, Warning{Message: err.Error()}) + // Chart-directory scanning is a filesystem walk and runs independently of + // the git-backed k8s-manifest discovery, so a chart dir outside a git repo + // still resolves its own subcharts and images. + chartRoots, chartWarnings, chartErr := discoverChartDependencyRoots(absPath, opts.Managers) + warnings = append(warnings, chartWarnings...) + if chartErr != nil { + return nil, chartErr } + roots = append(roots, chartRoots...) + + var imageRoots []*Node + var imageWarnings []Warning + imageRoots, imageWarnings, imageErr = discoverImageDependencyRoots(absPath, imageScanManagers(opts.Managers)) + warnings = append(warnings, imageWarnings...) roots = append(roots, imageRoots...) } + // Image discovery errors (e.g. not a git repository) only matter when nothing + // else resolved; otherwise they degrade to a warning. + if imageErr != nil && len(roots) == 0 { + if packageErr != nil { + return nil, packageErr + } + return nil, imageErr + } + if imageErr != nil { + warnings = append(warnings, Warning{Message: imageErr.Error()}) + } + if packageErr != nil && len(roots) == 0 { return nil, packageErr } if len(roots) == 0 { return nil, fmt.Errorf("no dependency graphs resolved") } - projectsScanned := len(projects) + imageRootCount(roots) + projectsScanned := len(projects) + imageRootCount(roots) + chartRootCount(roots) filteredRoots := make([]*Node, 0, len(roots)) for _, root := range roots { diff --git a/deps/scan_chart.go b/deps/scan_chart.go new file mode 100644 index 0000000..53fbcfd --- /dev/null +++ b/deps/scan_chart.go @@ -0,0 +1,383 @@ +package deps + +import ( + "fmt" + "io/fs" + "os" + osexec "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/goccy/go-yaml" +) + +// chartFile is the subset of Chart.yaml repomap reads: the chart's own identity +// and its declared subchart dependencies. +type chartFile struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Dependencies []chartDepEntry `yaml:"dependencies"` +} + +type chartDepEntry struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Repository string `yaml:"repository"` +} + +// discoverChartDependencyRoots walks for Helm Chart.yaml files and returns one +// dependency root per chart. Each root carries the subchart dependencies +// declared in Chart.yaml (helm) and the container images referenced in the +// chart's values.yaml and templates/ (image). Versions are recorded verbatim +// from the declarations; nothing is resolved over the network. +func discoverChartDependencyRoots(root string, managers []Manager) ([]*Node, []Warning, error) { + selected := managerSet(managers) + wantHelm := len(selected) == 0 || selected[ManagerHelm] + wantImage := len(selected) == 0 || selected[ManagerImage] + if !wantHelm && !wantImage { + return nil, nil, nil + } + + charts, err := discoverChartFiles(root) + if err != nil { + return nil, nil, err + } + + var roots []*Node + var warnings []Warning + for _, chartPath := range charts { + node, chartWarnings := chartDependencyRoot(root, chartPath, wantHelm, wantImage) + warnings = append(warnings, chartWarnings...) + if node != nil { + roots = append(roots, node) + } + } + sort.SliceStable(roots, func(i, j int) bool { return roots[i].Path < roots[j].Path }) + return roots, warnings, nil +} + +func chartDependencyRoot(scanRoot, chartPath string, wantHelm, wantImage bool) (*Node, []Warning) { + rel := chartRelPath(scanRoot, chartPath) + data, err := os.ReadFile(chartPath) + if err != nil { + return nil, []Warning{{Manager: ManagerHelm, Project: rel, Message: err.Error()}} + } + var chart chartFile + if err := yaml.Unmarshal(data, &chart); err != nil { + return nil, []Warning{{Manager: ManagerHelm, Project: rel, Message: fmt.Sprintf("parse Chart.yaml: %s", err)}} + } + + chartDir := filepath.Dir(chartPath) + name := chart.Name + if name == "" { + name = filepath.Base(chartDir) + } + root := NewNode(ManagerHelm, name, chart.Version) + root.Source = "Chart.yaml" + root.Path = filepath.ToSlash(chartPath) + + var children []*Node + var warnings []Warning + if wantHelm { + children = append(children, chartSubchartNodes(chart, rel)...) + } + if wantImage { + imageNodes, imgWarnings := chartImageNodes(chartDir, scanRoot) + children = append(children, imageNodes...) + warnings = append(warnings, imgWarnings...) + } + if len(children) == 0 { + return nil, warnings + } + root.Children = children + return root, warnings +} + +func chartSubchartNodes(chart chartFile, chartRel string) []*Node { + var nodes []*Node + for _, dep := range chart.Dependencies { + if dep.Name == "" { + continue + } + node := NewNode(ManagerHelm, dep.Name, dep.Version) + node.Direct = true + node.Depth = 1 + node.Scope = "dependencies" + node.Source = dep.Repository + node.Path = chartRel + nodes = append(nodes, node) + } + return nodes +} + +// chartImageNodes collects container images referenced in a chart's values.yaml +// and templates/, deduplicated by image ref. +func chartImageNodes(chartDir, scanRoot string) ([]*Node, []Warning) { + seen := map[string]*Node{} + var order []string + add := func(ref, source string) { + ref = strings.TrimSpace(strings.Trim(ref, `"'`)) + if !looksLikeImage(ref) { + return + } + node := chartImageNode(ref, source) + if _, ok := seen[node.ID]; ok { + return + } + seen[node.ID] = node + order = append(order, node.ID) + } + + var warnings []Warning + valuesPath := filepath.Join(chartDir, "values.yaml") + if data, err := os.ReadFile(valuesPath); err == nil { + var values map[string]interface{} + if err := yaml.Unmarshal(data, &values); err != nil { + warnings = append(warnings, Warning{Manager: ManagerImage, Project: chartRelPath(scanRoot, valuesPath), Message: fmt.Sprintf("parse values.yaml: %s", err)}) + } else { + source := chartRelPath(scanRoot, valuesPath) + collectValuesImages(values, func(ref string) { add(ref, source) }) + } + } + collectTemplateImages(chartDir, scanRoot, add) + + nodes := make([]*Node, 0, len(order)) + for _, id := range order { + nodes = append(nodes, seen[id]) + } + return nodes, warnings +} + +func chartImageNode(ref, source string) *Node { + name, version := splitImageRef(ref) + node := NewNode(ManagerImage, name, version) + node.Direct = true + node.Depth = 1 + node.Source = ref + node.Path = source + return node +} + +// collectValuesImages walks a parsed values.yaml tree and reports every image +// reference it finds, covering both the `image: "repo:tag"` string form and the +// structured `image: {registry, repository, tag}` map form at any nesting depth. +func collectValuesImages(node interface{}, report func(string)) { + switch v := node.(type) { + case map[string]interface{}: + if s, ok := v["image"].(string); ok { + report(s) + } + if img := imageFromMap(v); img != "" { + report(img) + } + for _, val := range v { + collectValuesImages(val, report) + } + case []interface{}: + for _, item := range v { + collectValuesImages(item, report) + } + } +} + +func imageFromMap(m map[string]interface{}) string { + if spec, ok := m["image"].(map[string]interface{}); ok { + if img := repoTagImage(spec); img != "" { + return img + } + } + return repoTagImage(m) +} + +func repoTagImage(m map[string]interface{}) string { + repo := scalarString(m["repository"]) + if repo == "" { + // `name` is a less common alias for `repository`; only trust it when a tag + // sits alongside, so non-image maps that merely have a name are ignored. + if _, hasTag := m["tag"]; hasTag { + repo = scalarString(m["name"]) + } + } + if repo == "" { + return "" + } + if registry := scalarString(m["registry"]); registry != "" { + repo = registry + "/" + repo + } + if tag := scalarString(m["tag"]); tag != "" { + return repo + ":" + tag + } + return repo +} + +func scalarString(v interface{}) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + default: + return fmt.Sprintf("%v", t) + } +} + +// collectTemplateImages scans a chart's templates/ for literal `image:` values, +// skipping Go-templated values that cannot be resolved offline. +func collectTemplateImages(chartDir, scanRoot string, add func(ref, source string)) { + templatesDir := filepath.Join(chartDir, "templates") + _ = filepath.WalkDir(templatesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !isTemplateFile(d.Name()) { + return nil + } + data, readErr := os.ReadFile(path) + if readErr != nil { + return nil + } + source := chartRelPath(scanRoot, path) + for _, ref := range imageValuesInTemplate(string(data)) { + add(ref, source) + } + return nil + }) +} + +func imageValuesInTemplate(content string) []string { + var refs []string + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + // Accept both `image:` and the list-item form `- image:`. + trimmed = strings.TrimSpace(strings.TrimPrefix(trimmed, "-")) + const key = "image:" + if !strings.HasPrefix(trimmed, key) { + continue + } + value := strings.TrimSpace(trimmed[len(key):]) + if value == "" || strings.Contains(value, "{{") { + continue + } + refs = append(refs, value) + } + return refs +} + +func isTemplateFile(name string) bool { + switch strings.ToLower(filepath.Ext(name)) { + case ".yaml", ".yml", ".tpl": + return true + } + return false +} + +// looksLikeImage rejects values that are clearly not container image references: +// empty, whitespace, Go-templated, or not starting with an image-ref character. +// Bare official images (e.g. busybox) are accepted since the value came from an +// explicit image field. +func looksLikeImage(ref string) bool { + if ref == "" || strings.ContainsAny(ref, " \t{}|<>\"'`") || strings.Contains(ref, "{{") { + return false + } + c := ref[0] + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + +func splitImageRef(ref string) (name, version string) { + name = stripImageVersion(ref) + if i := imageTagSeparator(ref); i >= 0 { + version = ref[i+1:] + if at := strings.Index(version, "@"); at >= 0 { + version = version[:at] + } + } + return name, version +} + +func discoverChartFiles(root string) ([]string, error) { + files, ok := gitChartFiles(root) + if !ok { + var err error + files, err = walkChartFiles(root) + if err != nil { + return nil, err + } + } + return filterVendoredCharts(files), nil +} + +func gitChartFiles(root string) ([]string, bool) { + cmd := osexec.Command("git", "-C", root, "ls-files", "--cached", "--others", "--exclude-standard", "-z", "--", ".") + out, err := cmd.Output() + if err != nil { + return nil, false + } + var files []string + for _, rel := range strings.Split(string(out), "\x00") { + if rel == "" || !strings.EqualFold(filepath.Base(filepath.FromSlash(rel)), "chart.yaml") { + continue + } + files = append(files, filepath.Join(root, filepath.FromSlash(rel))) + } + sort.Strings(files) + return files, true +} + +func walkChartFiles(root string) ([]string, error) { + var files []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != root && ignoredDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + if strings.EqualFold(d.Name(), "Chart.yaml") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(files) + return files, nil +} + +// filterVendoredCharts drops Chart.yaml files that live in a parent chart's +// charts/ directory (downloaded dependencies), keeping standalone charts and +// monorepo charts/ layouts. +func filterVendoredCharts(files []string) []string { + present := make(map[string]bool, len(files)) + for _, f := range files { + present[filepath.Dir(f)] = true + } + out := files[:0] + for _, f := range files { + dir := filepath.Dir(f) + parent := filepath.Dir(dir) + if filepath.Base(parent) == "charts" && present[filepath.Dir(parent)] { + continue + } + out = append(out, f) + } + return out +} + +func chartRelPath(scanRoot, path string) string { + if rel, err := filepath.Rel(scanRoot, path); err == nil { + return filepath.ToSlash(rel) + } + return filepath.ToSlash(path) +} + +func chartRootCount(roots []*Node) int { + count := 0 + for _, root := range roots { + if root != nil && root.Manager == ManagerHelm && root.Source == "Chart.yaml" { + count++ + } + } + return count +} diff --git a/deps/scan_chart_test.go b/deps/scan_chart_test.go new file mode 100644 index 0000000..ce8bd49 --- /dev/null +++ b/deps/scan_chart_test.go @@ -0,0 +1,188 @@ +package deps + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +const chartYAMLFixture = `apiVersion: v2 +name: web +version: 1.4.0 +dependencies: + - name: postgresql + version: "12.x.x" + repository: https://charts.bitnami.com/bitnami + - name: redis + version: 18.1.2 + repository: oci://registry-1.docker.io/bitnamicharts +` + +const chartValuesFixture = `image: + registry: docker.io + repository: bitnami/nginx + tag: "1.25.3" +sidecar: + image: busybox:1.36 +postgresql: + image: + repository: postgres + tag: 15.5 +ui: + image: + name: ghcr.io/acme/ui + tag: "3.1.0" +` + +const chartTemplateFixture = `apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: app + image: ghcr.io/acme/app:2.0.1 + - name: tools + image: alpine + - image: quay.io/prometheus/node-exporter:1.7.0 + name: dash-first + - name: templated + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +` + +func writeChartFixture(t *testing.T, dir string) { + t.Helper() + writeFile(t, filepath.Join(dir, "charts", "web", "Chart.yaml"), chartYAMLFixture) + writeFile(t, filepath.Join(dir, "charts", "web", "values.yaml"), chartValuesFixture) + writeFile(t, filepath.Join(dir, "charts", "web", "templates", "deployment.yaml"), chartTemplateFixture) +} + +func TestScanChartSubchartDependencies(t *testing.T) { + dir := t.TempDir() + writeChartFixture(t, dir) + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerHelm}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Roots) != 1 { + t.Fatalf("expected one chart root, got %d: %#v", len(got.Roots), got.Roots) + } + root := got.Roots[0] + if root.Manager != ManagerHelm || root.Name != "web" || root.Version != "1.4.0" { + t.Fatalf("chart root metadata wrong: %#v", root) + } + if root.Source != "Chart.yaml" { + t.Fatalf("chart root should be sourced from Chart.yaml, got %q", root.Source) + } + + pg := findChild(root, "postgresql") + if pg == nil || pg.Version != "12.x.x" || pg.Manager != ManagerHelm { + t.Fatalf("postgresql subchart not resolved with declared range: %#v", pg) + } + if pg.Source != "https://charts.bitnami.com/bitnami" { + t.Fatalf("subchart should carry its repository as source: %#v", pg) + } + redis := findChild(root, "redis") + if redis == nil || redis.Version != "18.1.2" { + t.Fatalf("redis subchart not resolved: %#v", redis) + } +} + +func TestScanChartHelmOnlyExcludesImages(t *testing.T) { + dir := t.TempDir() + writeChartFixture(t, dir) + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerHelm}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + for _, child := range got.Roots[0].Children { + if child.Manager == ManagerImage { + t.Fatalf("helm-only scan should not include image children: %#v", child) + } + } +} + +func TestScanChartExtractsImagesFromValuesAndTemplates(t *testing.T) { + dir := t.TempDir() + writeChartFixture(t, dir) + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerImage}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Roots) != 1 { + t.Fatalf("expected one chart root, got %d", len(got.Roots)) + } + root := got.Roots[0] + + want := map[string]string{ + "docker.io/bitnami/nginx": "1.25.3", + "busybox": "1.36", + "postgres": "15.5", + "ghcr.io/acme/ui": "3.1.0", + "ghcr.io/acme/app": "2.0.1", + "alpine": "", // bare image from a template, no tag + "quay.io/prometheus/node-exporter": "1.7.0", // list-item `- image:` form + } + for name, version := range want { + img := findChild(root, name) + if img == nil { + t.Fatalf("expected image %q in chart, got children %#v", name, childNames(root)) + } + if img.Version != version { + t.Fatalf("image %q version = %q, want %q", name, img.Version, version) + } + if img.Manager != ManagerImage { + t.Fatalf("image %q should use the image manager: %#v", name, img) + } + } + if templated := findChild(root, "{{ .Values.image.repository }}"); templated != nil { + t.Fatalf("templated image refs must be skipped, got %#v", templated) + } +} + +func TestScanChartSkipsVendoredSubcharts(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "Chart.yaml"), "apiVersion: v2\nname: parent\nversion: 1.0.0\ndependencies:\n - name: child\n version: 1.0.0\n") + // A vendored copy of the dependency under charts/ must not become its own root. + writeFile(t, filepath.Join(dir, "charts", "child", "Chart.yaml"), "apiVersion: v2\nname: child\nversion: 1.0.0\n") + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerHelm}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Roots) != 1 || got.Roots[0].Name != "parent" { + t.Fatalf("vendored subchart should not be a separate root, got %#v", childRootNames(got.Roots)) + } +} + +func childNames(node *Node) []string { + var out []string + for _, c := range node.Children { + out = append(out, c.Name) + } + return out +} + +func childRootNames(roots []*Node) []string { + var out []string + for _, r := range roots { + out = append(out, r.Name) + } + return out +} From f3ea4546b93ddbdb58b099b7f5baa9366d5ce133 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 14 Jun 2026 10:19:25 +0300 Subject: [PATCH 03/13] feat(deps): recursively resolve chart subcharts and image base images At --depth other than 1 (mirroring go/maven/gradle), dependency scanning now recurses into remote dependencies, backed by a persistent cache under the user cache dir: - Helm charts: each subchart is fetched from its Helm repository (index resolved via SemVer range, chart .tgz loaded with the Helm SDK chart loader), and its own subcharts and values/template images are recursed. - Container images: the base image is resolved from the OCI base-image label and from the Dockerfile FROM directives in the image's source repository (located via the OCI source label or a ghcr/quay heuristic, cloned into the cache), then recursed. The cache (remote_cache.go) keeps immutable artifacts (git clones, chart archives, image config by digest) long and revalidates indexes/tags daily, with negative caching and singleflight. Remote failures degrade to per-node warnings; only cache initialization is fatal. --depth 1 stays fully offline. Helm is pinned to v3.17.x and only pkg/chart/loader is used (index.yaml parsed directly): v3.18+ bumps distribution/v3 past what registry-scanner needs, and pkg/repo transitively requires a k8s API removed in the pinned k8s release. --- cmd/repomap/deps.go | 6 + deps/chart_remote.go | 43 ++++++ deps/dockerfile.go | 149 ++++++++++++++++++ deps/dockerfile_test.go | 121 +++++++++++++++ deps/image_base.go | 167 ++++++++++++++++++++ deps/model.go | 3 + deps/remote_cache.go | 264 ++++++++++++++++++++++++++++++++ deps/remote_cache_test.go | 151 ++++++++++++++++++ deps/remote_helm_client.go | 208 +++++++++++++++++++++++++ deps/remote_helm_client_test.go | 42 +++++ deps/resolve_remote.go | 168 ++++++++++++++++++++ deps/resolve_remote_test.go | 196 ++++++++++++++++++++++++ deps/scan.go | 11 ++ deps/scan_chart_test.go | 4 + deps/scan_test.go | 1 + imageupdate/labels.go | 46 ++++++ 16 files changed, 1580 insertions(+) create mode 100644 deps/chart_remote.go create mode 100644 deps/dockerfile.go create mode 100644 deps/dockerfile_test.go create mode 100644 deps/image_base.go create mode 100644 deps/remote_cache.go create mode 100644 deps/remote_cache_test.go create mode 100644 deps/remote_helm_client.go create mode 100644 deps/remote_helm_client_test.go create mode 100644 deps/resolve_remote.go create mode 100644 deps/resolve_remote_test.go create mode 100644 imageupdate/labels.go diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index ba3e8fc..2baac85 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -44,6 +44,12 @@ by shelling out to the package manager (go mod graph, mvn dependency:tree, gradle dependencies). The tool must be installed; rerun with --depth 1 for offline direct-only output. +For Helm charts and container images, --depth other than 1 additionally recurses +into remote dependencies: subcharts are fetched from their Helm repositories and +image base images are resolved from registry labels and Dockerfile FROM +directives (cloning source repos). Fetches are cached under the user cache dir; +failures degrade to warnings. Use --depth 1 to stay fully offline. + The command uses the normal Clicky output flow. Use --json to write structured JSON to stdout, for example: diff --git a/deps/chart_remote.go b/deps/chart_remote.go new file mode 100644 index 0000000..cfbafb0 --- /dev/null +++ b/deps/chart_remote.go @@ -0,0 +1,43 @@ +package deps + +import ( + "context" + "fmt" +) + +// chartResolver fetches a Helm chart dependency and produces its direct children: +// nested subchart dependency nodes and image nodes harvested from the fetched +// chart's values.yaml and templates. +type chartResolver struct { + helm *helmClient +} + +func (c *chartResolver) expand(ctx context.Context, dep chartDepEntry, parentDepth int) (children []*Node, resolvedVersion string, warnings []string) { + fc, cleanup, err := c.helm.fetchChart(ctx, dep) + if err != nil { + return nil, "", []string{fmt.Sprintf("chart %s@%s: %s", dep.Name, dep.Version, err)} + } + defer cleanup() + + for _, sub := range fc.Dependencies { + if sub.Name == "" { + continue + } + n := NewNode(ManagerHelm, sub.Name, sub.Version) + n.Depth = parentDepth + 1 + n.Scope = "dependencies" + n.Source = sub.Repository + children = append(children, n) + } + + imgNodes, imgWarns := chartImageNodes(fc.Dir, fc.Dir) + for _, img := range imgNodes { + img.Depth = parentDepth + 1 + img.Direct = false + children = append(children, img) + } + for _, w := range imgWarns { + warnings = append(warnings, w.Message) + } + return children, fc.Version, warnings +} diff --git a/deps/dockerfile.go b/deps/dockerfile.go new file mode 100644 index 0000000..74fab73 --- /dev/null +++ b/deps/dockerfile.go @@ -0,0 +1,149 @@ +package deps + +import ( + "fmt" + "regexp" + "strings" +) + +// imageRef is a parsed container image reference split into name, tag, and digest. +type imageRef struct { + Name string + Version string + Digest string +} + +var argRefPattern = regexp.MustCompile(`\$\{(\w+)\}|\$(\w+)`) + +// parseDockerfileFrom extracts the external base images referenced by FROM +// directives. It substitutes ARG/ENV defaults declared earlier in the file, +// excludes references to internal build stages (FROM ... AS ) and the +// scratch terminal, and de-duplicates. Unresolved ARG references are warned. +func parseDockerfileFrom(content string) (bases []imageRef, warnings []string) { + args := map[string]string{} + stages := map[string]bool{} + seen := map[string]bool{} + + for _, line := range dockerfileLines(content) { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + switch strings.ToUpper(fields[0]) { + case "ARG", "ENV": + if name, val, ok := parseArgAssign(fields[1:]); ok { + args[name] = val + } + case "FROM": + ref, stage, ok, warn := parseFromDirective(fields[1:], args, stages) + if warn != "" { + warnings = append(warnings, warn) + } + if stage != "" { + stages[strings.ToLower(stage)] = true + } + if !ok { + continue + } + key := ref.Name + ":" + ref.Version + "@" + ref.Digest + if !seen[key] { + seen[key] = true + bases = append(bases, ref) + } + } + } + return bases, warnings +} + +// parseFromDirective interprets the tokens after FROM: it strips flags +// (--platform=...), resolves the image token, and records an AS stage name. +// ok is false for internal stage references, scratch, and unresolved ARGs. +func parseFromDirective(tokens []string, args map[string]string, stages map[string]bool) (ref imageRef, stage string, ok bool, warn string) { + var image string + for i := 0; i < len(tokens); i++ { + tok := tokens[i] + switch { + case strings.HasPrefix(tok, "--"): + continue + case image == "": + image = tok + case strings.EqualFold(tok, "AS") && i+1 < len(tokens): + stage = tokens[i+1] + i++ + } + } + if image == "" { + return imageRef{}, stage, false, "" + } + resolved, complete := substituteArgs(image, args) + if !complete { + return imageRef{}, stage, false, fmt.Sprintf("unresolved build arg in FROM %q", image) + } + if strings.EqualFold(resolved, "scratch") || stages[strings.ToLower(resolved)] { + return imageRef{}, stage, false, "" + } + return parseImageRef(resolved), stage, true, "" +} + +// substituteArgs replaces ${VAR}/$VAR with known ARG/ENV defaults; complete is +// false when any reference cannot be resolved. +func substituteArgs(s string, args map[string]string) (out string, complete bool) { + complete = true + out = argRefPattern.ReplaceAllStringFunc(s, func(m string) string { + name := strings.Trim(m, "${}") + if v, ok := args[name]; ok { + return v + } + complete = false + return m + }) + return out, complete +} + +// parseImageRef splits an image reference into name, tag, and digest, leaving a +// registry host:port intact. +func parseImageRef(s string) imageRef { + var digest string + if at := strings.Index(s, "@"); at >= 0 { + digest = s[at+1:] + s = s[:at] + } + name, version := s, "" + if i := imageTagSeparator(s); i >= 0 { + name, version = s[:i], s[i+1:] + } + return imageRef{Name: name, Version: version, Digest: digest} +} + +func parseArgAssign(tokens []string) (name, value string, ok bool) { + if len(tokens) == 0 { + return "", "", false + } + eq := strings.SplitN(tokens[0], "=", 2) + if len(eq) != 2 { + return "", "", false + } + return eq[0], strings.Trim(eq[1], `"'`), true +} + +// dockerfileLines joins backslash line continuations and trims carriage returns. +func dockerfileLines(content string) []string { + var lines []string + var buf strings.Builder + for _, ln := range strings.Split(content, "\n") { + ln = strings.TrimRight(ln, "\r") + trimmedRight := strings.TrimRight(ln, " \t") + if strings.HasSuffix(trimmedRight, "\\") { + buf.WriteString(strings.TrimSuffix(trimmedRight, "\\")) + buf.WriteString(" ") + continue + } + buf.WriteString(ln) + lines = append(lines, strings.TrimSpace(buf.String())) + buf.Reset() + } + if buf.Len() > 0 { + lines = append(lines, strings.TrimSpace(buf.String())) + } + return lines +} diff --git a/deps/dockerfile_test.go b/deps/dockerfile_test.go new file mode 100644 index 0000000..de6b770 --- /dev/null +++ b/deps/dockerfile_test.go @@ -0,0 +1,121 @@ +package deps + +import "testing" + +func refsToStrings(refs []imageRef) []string { + out := make([]string, 0, len(refs)) + for _, r := range refs { + s := r.Name + if r.Version != "" { + s += ":" + r.Version + } + if r.Digest != "" { + s += "@" + r.Digest + } + out = append(out, s) + } + return out +} + +func TestParseDockerfileFrom(t *testing.T) { + cases := []struct { + name string + content string + want []string + }{ + { + name: "single FROM with tag", + content: "FROM nginx:1.25.3\nRUN echo hi\n", + want: []string{"nginx:1.25.3"}, + }, + { + name: "bare image no tag", + content: "FROM alpine\n", + want: []string{"alpine"}, + }, + { + name: "multi-stage excludes internal stage reference", + content: "FROM golang:1.22 AS build\n" + + "RUN go build\n" + + "FROM gcr.io/distroless/static:nonroot\n" + + "COPY --from=build /app /app\n", + want: []string{"golang:1.22", "gcr.io/distroless/static:nonroot"}, + }, + { + name: "final FROM referencing a prior stage is internal", + content: "FROM golang:1.22 AS build\n" + + "FROM build\n", + want: []string{"golang:1.22"}, + }, + { + name: "platform flag is stripped", + content: "FROM --platform=linux/amd64 ubuntu:22.04\n", + want: []string{"ubuntu:22.04"}, + }, + { + name: "ARG substitution in tag", + content: "ARG GO_VERSION=1.22\nFROM golang:${GO_VERSION}\n", + want: []string{"golang:1.22"}, + }, + { + name: "ARG substitution in registry and unbraced var", + content: "ARG REG=docker.io\nFROM $REG/library/busybox:1.36\n", + want: []string{"docker.io/library/busybox:1.36"}, + }, + { + name: "scratch is terminal and excluded", + content: "FROM scratch\nCOPY x /\n", + want: nil, + }, + { + name: "digest pinned", + content: "FROM nginx@sha256:abc123\n", + want: []string{"nginx@sha256:abc123"}, + }, + { + name: "tag and digest", + content: "FROM nginx:1.25@sha256:abc123\n", + want: []string{"nginx:1.25@sha256:abc123"}, + }, + { + name: "case-insensitive directives and comments", + content: "# base\nfrom Ubuntu:22.04 as Base\n", + want: []string{"Ubuntu:22.04"}, + }, + { + name: "registry with port keeps host colon", + content: "FROM localhost:5000/team/app:2.0\n", + want: []string{"localhost:5000/team/app:2.0"}, + }, + { + name: "duplicate external bases collapse", + content: "FROM alpine:3.20 AS a\nFROM alpine:3.20 AS b\n", + want: []string{"alpine:3.20"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, _ := parseDockerfileFrom(tc.content) + gotStrings := refsToStrings(got) + if len(gotStrings) != len(tc.want) { + t.Fatalf("bases = %v, want %v", gotStrings, tc.want) + } + for i := range tc.want { + if gotStrings[i] != tc.want[i] { + t.Fatalf("base[%d] = %q, want %q (all: %v)", i, gotStrings[i], tc.want[i], gotStrings) + } + } + }) + } +} + +func TestParseDockerfileUnresolvedArgWarns(t *testing.T) { + bases, warnings := parseDockerfileFrom("FROM $UNDEFINED_BASE\n") + if len(bases) != 0 { + t.Fatalf("unresolved ARG FROM should yield no base, got %v", refsToStrings(bases)) + } + if len(warnings) == 0 { + t.Fatalf("expected a warning for the unresolved ARG reference") + } +} diff --git a/deps/image_base.go b/deps/image_base.go new file mode 100644 index 0000000..fba8d0e --- /dev/null +++ b/deps/image_base.go @@ -0,0 +1,167 @@ +package deps + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const ( + labelBaseName = "org.opencontainers.image.base.name" + labelBaseDigest = "org.opencontainers.image.base.digest" + labelSource = "org.opencontainers.image.source" +) + +// imageResolver resolves the base image(s) of a container image by combining the +// OCI base-image label with the Dockerfile FROM directives in the image's source +// repository. +type imageResolver struct { + cache RemoteCache + parser func(content string) ([]imageRef, []string) +} + +func newImageResolver(cache RemoteCache) *imageResolver { + return &imageResolver{cache: cache, parser: parseDockerfileFrom} +} + +// baseImages returns the distinct base images of ref. It never returns a hard +// error: remote failures are reported as warnings so recursion degrades per +// branch. +func (r *imageResolver) baseImages(ctx context.Context, ref string) (bases []imageRef, warnings []string) { + cfg, err := r.cache.ImageConfig(ctx, ref) + if err != nil { + return nil, []string{fmt.Sprintf("image %s: %s", ref, err)} + } + + seen := map[string]bool{} + add := func(b imageRef) { + key := b.Name + ":" + b.Version + "@" + b.Digest + if b.Name == "" || seen[key] { + return + } + seen[key] = true + bases = append(bases, b) + } + + // Signal A: explicit OCI base-image label. + if name := cfg.Labels[labelBaseName]; name != "" { + b := parseImageRef(name) + if b.Digest == "" { + b.Digest = cfg.Labels[labelBaseDigest] + } + add(b) + } + + // Signal B: source repo Dockerfile FROM. + source := cfg.Labels[labelSource] + if source == "" { + source = sourceRepoHeuristic(ref) + } + if source != "" { + dbases, dwarn := r.dockerfileBases(ctx, ref, source) + warnings = append(warnings, dwarn...) + for _, b := range dbases { + add(b) + } + } + + if len(bases) == 0 && len(warnings) == 0 { + warnings = append(warnings, fmt.Sprintf("image %s: no base image resolvable (no base/source label, no known registry heuristic)", ref)) + } + return bases, warnings +} + +func (r *imageResolver) dockerfileBases(ctx context.Context, ref, source string) ([]imageRef, []string) { + url := normalizeGitURL(source) + if url == "" { + return nil, []string{fmt.Sprintf("image %s: unrecognized source %q", ref, source)} + } + dir, err := r.cache.GitRepo(ctx, url, parseImageRef(ref).Version) + if err != nil { + return nil, []string{fmt.Sprintf("image %s: clone %s: %s", ref, url, err)} + } + path, warnings := findDockerfile(dir) + if path == "" { + return nil, []string{fmt.Sprintf("image %s: no Dockerfile in %s", ref, source)} + } + data, err := os.ReadFile(path) + if err != nil { + return nil, append(warnings, fmt.Sprintf("image %s: reading Dockerfile: %s", ref, err)) + } + bases, pwarn := r.parser(string(data)) + return bases, append(warnings, pwarn...) +} + +// sourceRepoHeuristic maps a registry path to a probable git source URL for +// registries that mirror their org/repo layout from GitHub. +func sourceRepoHeuristic(ref string) string { + name := stripImageVersion(ref) + for _, host := range []string{"ghcr.io/", "quay.io/"} { + if strings.HasPrefix(name, host) { + parts := strings.Split(strings.TrimPrefix(name, host), "/") + if len(parts) >= 2 { + return "https://github.com/" + parts[0] + "/" + parts[1] + } + } + } + return "" +} + +// normalizeGitURL turns an OCI source label into a cloneable URL. +func normalizeGitURL(source string) string { + source = strings.TrimSpace(source) + source = strings.TrimPrefix(source, "git+") + switch { + case source == "": + return "" + case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"), strings.HasPrefix(source, "git@"): + return strings.TrimSuffix(source, ".git") + case strings.HasPrefix(source, "github.com/"), strings.HasPrefix(source, "gitlab.com/"): + return "https://" + strings.TrimSuffix(source, ".git") + default: + return "" + } +} + +// findDockerfile returns the repo-root Dockerfile if present, else the first +// Dockerfile found in a shallow walk, warning when several exist. +func findDockerfile(root string) (string, []string) { + if p := filepath.Join(root, "Dockerfile"); fileExists(p) { + return p, nil + } + var found []string + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() { + if d.Name() == ".git" || ignoredDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + if d.Name() == "Dockerfile" || strings.HasPrefix(d.Name(), "Dockerfile.") { + found = append(found, path) + } + return nil + }) + if len(found) == 0 { + return "", nil + } + if len(found) > 1 { + rels := make([]string, len(found)) + for i, f := range found { + rels[i], _ = filepath.Rel(root, f) + } + return found[0], []string{fmt.Sprintf("multiple Dockerfiles found, using %s (others: %s)", rels[0], strings.Join(rels[1:], ", "))} + } + return found[0], nil +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} diff --git a/deps/model.go b/deps/model.go index 26e60e3..0a30ace 100644 --- a/deps/model.go +++ b/deps/model.go @@ -30,6 +30,9 @@ type Options struct { ShowDuplicates bool Runner CommandRunner Now func() time.Time + // remote injects a pre-built remote resolver (cache + chart/image resolvers) + // for tests; nil in production, where Scan builds a disk-backed one. + remote *remoteDeps } type Project struct { diff --git a/deps/remote_cache.go b/deps/remote_cache.go new file mode 100644 index 0000000..bfbf1a1 --- /dev/null +++ b/deps/remote_cache.go @@ -0,0 +1,264 @@ +package deps + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sync/singleflight" +) + +// Differentiated TTLs (decision 3). Immutable-per-ref artifacts (git clones at a +// ref, chart .tgz, image config by digest) are kept long; mutable lookups +// (helm index.yaml, image tag→config) refresh daily; negatives retry soon. +const ( + ttlImmutable = 365 * 24 * time.Hour + ttlIndex = 24 * time.Hour + ttlNegative = 1 * time.Hour +) + +// ImageConfig is the resolved OCI config metadata for an image reference. +type ImageConfig struct { + Digest string + Labels map[string]string +} + +// notFoundError marks a definitively-absent remote resource (404, missing repo) +// so it can be negatively cached and degraded to a warning, distinct from a +// transient network failure which is never cached. +type notFoundError struct{ what string } + +func (e notFoundError) Error() string { return e.what + ": not found" } + +func isNotFound(err error) bool { + var nf notFoundError + return errors.As(err, &nf) +} + +// labelResolver reads an image's config digest and OCI labels from a registry. +// Production is backed by imageupdate; tests inject a fake. +type labelResolver interface { + Labels(ctx context.Context, ref string) (digest string, labels map[string]string, err error) +} + +// RemoteCache provides cached access to the remote artifacts recursion needs: +// arbitrary HTTP blobs (helm index/chart), git checkouts, and image configs. +type RemoteCache interface { + Fetch(ctx context.Context, url string, ttl time.Duration) ([]byte, error) + GitRepo(ctx context.Context, url, ref string) (dir string, err error) + ImageConfig(ctx context.Context, ref string) (ImageConfig, error) +} + +type diskCache struct { + root string + now func() time.Time + get func(ctx context.Context, url string) ([]byte, error) + runner CommandRunner + labels labelResolver + group singleflight.Group +} + +// newDiskCache builds the production cache rooted at os.UserCacheDir()/repomap. +func newDiskCache(now func() time.Time, runner CommandRunner, labels labelResolver) (*diskCache, error) { + base, err := os.UserCacheDir() + if err != nil { + return nil, fmt.Errorf("locating user cache dir: %w", err) + } + root := filepath.Join(base, "repomap", "deps") + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("creating cache dir %s: %w", root, err) + } + c := &diskCache{root: root, now: now, runner: runner, labels: labels} + c.get = c.httpGet + return c, nil +} + +type cacheEntry[T any] struct { + FetchedAt time.Time + NotFound bool + Value T +} + +func (c *diskCache) Fetch(ctx context.Context, url string, ttl time.Duration) ([]byte, error) { + v, err, _ := c.group.Do("blob:"+url, func() (any, error) { + path := c.entryPath("blobs", url) + if e, ok := readEntry[[]byte](path); ok && !c.expiredAt(e.FetchedAt, e.NotFound, ttl) { + if e.NotFound { + return nil, notFoundError{url} + } + return e.Value, nil + } + data, err := c.get(ctx, url) + if isNotFound(err) { + writeEntry(path, cacheEntry[[]byte]{FetchedAt: c.now(), NotFound: true}) + return nil, notFoundError{url} + } + if err != nil { + return nil, err // transient: do not cache + } + writeEntry(path, cacheEntry[[]byte]{FetchedAt: c.now(), Value: data}) + return data, nil + }) + if err != nil { + return nil, err + } + return v.([]byte), nil +} + +func (c *diskCache) ImageConfig(ctx context.Context, ref string) (ImageConfig, error) { + v, err, _ := c.group.Do("img:"+ref, func() (any, error) { + path := c.entryPath("imageconfig", ref) + if e, ok := readEntry[ImageConfig](path); ok && !c.expiredAt(e.FetchedAt, e.NotFound, ttlIndex) { + if e.NotFound { + return ImageConfig{}, notFoundError{ref} + } + return e.Value, nil + } + digest, labels, err := c.labels.Labels(ctx, ref) + if isNotFound(err) { + writeEntry(path, cacheEntry[ImageConfig]{FetchedAt: c.now(), NotFound: true}) + return ImageConfig{}, notFoundError{ref} + } + if err != nil { + return ImageConfig{}, err + } + cfg := ImageConfig{Digest: digest, Labels: labels} + writeEntry(path, cacheEntry[ImageConfig]{FetchedAt: c.now(), Value: cfg}) + return cfg, nil + }) + if err != nil { + return ImageConfig{}, err + } + return v.(ImageConfig), nil +} + +// GitRepo clones url at ref into the cache (kept for ttlImmutable) and returns +// the checkout directory. A ref that is not a branch/tag falls back to the +// default branch. +func (c *diskCache) GitRepo(ctx context.Context, url, ref string) (string, error) { + v, err, _ := c.group.Do("git:"+url+"@"+ref, func() (any, error) { + dir := filepath.Join(c.root, "git", hashKey(url), sanitizeRef(ref)) + marker := filepath.Join(dir, ".repomap-fetched") + if data, err := os.ReadFile(marker); err == nil { + if at, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(data))); perr == nil && + c.now().Sub(at) < ttlImmutable { + return dir, nil + } + } + if err := c.cloneInto(ctx, url, ref, dir); err != nil { + return "", err + } + _ = os.WriteFile(marker, []byte(c.now().Format(time.RFC3339)), 0o644) + return dir, nil + }) + if err != nil { + return "", err + } + return v.(string), nil +} + +func (c *diskCache) cloneInto(ctx context.Context, url, ref, dir string) error { + tmp, err := os.MkdirTemp(c.root, "clone-*") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmp) }() + + args := []string{"clone", "--depth", "1", "--single-branch", url, tmp} + if ref != "" { + args = []string{"clone", "--depth", "1", "--single-branch", "--branch", ref, url, tmp} + } + if _, err := c.runner.Run(ctx, Command{Name: "git", Args: args}); err != nil { + // Ref may not be a branch/tag; fall back to the default branch. + if ref == "" { + return notFoundError{url} + } + if _, err2 := c.runner.Run(ctx, Command{Name: "git", Args: []string{"clone", "--depth", "1", url, tmp}}); err2 != nil { + return notFoundError{url} + } + } + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return err + } + _ = os.RemoveAll(dir) + return os.Rename(tmp, dir) +} + +func (c *diskCache) httpGet(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound { + return nil, notFoundError{url} + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("GET %s: status %d", url, resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +func (c *diskCache) expiredAt(at time.Time, notFound bool, ttl time.Duration) bool { + if notFound { + ttl = ttlNegative + } + return c.now().Sub(at) >= ttl +} + +func (c *diskCache) entryPath(kind, key string) string { + return filepath.Join(c.root, kind, hashKey(key)+".gob") +} + +func hashKey(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func sanitizeRef(ref string) string { + if ref == "" { + return "_default" + } + return strings.NewReplacer("/", "_", ":", "_", " ", "_").Replace(ref) +} + +func writeEntry[T any](path string, e cacheEntry[T]) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return + } + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(e); err != nil { + return + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, buf.Bytes(), 0o644); err != nil { + return + } + _ = os.Rename(tmp, path) +} + +func readEntry[T any](path string) (cacheEntry[T], bool) { + data, err := os.ReadFile(path) + if err != nil { + return cacheEntry[T]{}, false + } + var e cacheEntry[T] + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&e); err != nil { + return cacheEntry[T]{}, false + } + return e, true +} diff --git a/deps/remote_cache_test.go b/deps/remote_cache_test.go new file mode 100644 index 0000000..e468d07 --- /dev/null +++ b/deps/remote_cache_test.go @@ -0,0 +1,151 @@ +package deps + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +func testClock(t time.Time) (func() time.Time, *time.Time) { + cur := t + return func() time.Time { return cur }, &cur +} + +func newTestCache(t *testing.T, now func() time.Time) *diskCache { + t.Helper() + return &diskCache{root: t.TempDir(), now: now} +} + +func TestCacheFetchHitAndTTL(t *testing.T) { + now, clock := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + return []byte("payload"), nil + } + + for i := 0; i < 3; i++ { + if _, err := c.Fetch(context.Background(), "http://x/index.yaml", ttlIndex); err != nil { + t.Fatal(err) + } + } + if calls != 1 { + t.Fatalf("expected 1 remote fetch within TTL, got %d", calls) + } + + *clock = clock.Add(ttlIndex + time.Minute) // expire + if _, err := c.Fetch(context.Background(), "http://x/index.yaml", ttlIndex); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("expected re-fetch after TTL expiry, got %d calls", calls) + } +} + +func TestCacheNegativeCaching(t *testing.T) { + now, clock := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + return nil, notFoundError{url} + } + + if _, err := c.Fetch(context.Background(), "http://x/missing", ttlImmutable); !isNotFound(err) { + t.Fatalf("want notFound, got %v", err) + } + if _, err := c.Fetch(context.Background(), "http://x/missing", ttlImmutable); !isNotFound(err) { + t.Fatalf("want cached notFound, got %v", err) + } + if calls != 1 { + t.Fatalf("negative result should be cached (1 call), got %d", calls) + } + + *clock = clock.Add(ttlNegative + time.Minute) // negatives expire on the short TTL + if _, err := c.Fetch(context.Background(), "http://x/missing", ttlImmutable); !isNotFound(err) { + t.Fatalf("want notFound after negative expiry, got %v", err) + } + if calls != 2 { + t.Fatalf("negative entry should retry after ttlNegative, got %d calls", calls) + } +} + +func TestCacheTransientErrorNotCached(t *testing.T) { + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + return nil, context.DeadlineExceeded + } + for i := 0; i < 2; i++ { + if _, err := c.Fetch(context.Background(), "http://x/flaky", ttlIndex); err == nil { + t.Fatal("expected transient error") + } + } + if calls != 2 { + t.Fatalf("transient errors must not be cached, want 2 calls got %d", calls) + } +} + +func TestCacheSingleflightCollapsesConcurrentFetches(t *testing.T) { + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + release := make(chan struct{}) + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + <-release // hold so all goroutines pile onto the same in-flight call + return []byte("v"), nil + } + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = c.Fetch(context.Background(), "http://x/same", ttlIndex) + }() + } + time.Sleep(20 * time.Millisecond) + close(release) + wg.Wait() + if calls != 1 { + t.Fatalf("singleflight should collapse concurrent fetches to 1, got %d", calls) + } +} + +func TestCacheImageConfig(t *testing.T) { + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.labels = fakeLabels{fn: func(ref string) (string, map[string]string, error) { + atomic.AddInt64(&calls, 1) + return "sha256:deadbeef", map[string]string{"org.opencontainers.image.source": "https://github.com/acme/app"}, nil + }} + + cfg, err := c.ImageConfig(context.Background(), "ghcr.io/acme/app:1.0") + if err != nil { + t.Fatal(err) + } + if cfg.Digest != "sha256:deadbeef" || cfg.Labels["org.opencontainers.image.source"] != "https://github.com/acme/app" { + t.Fatalf("unexpected config %#v", cfg) + } + if _, err := c.ImageConfig(context.Background(), "ghcr.io/acme/app:1.0"); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("image config should be cached, got %d calls", calls) + } +} + +type fakeLabels struct { + fn func(ref string) (string, map[string]string, error) +} + +func (f fakeLabels) Labels(_ context.Context, ref string) (string, map[string]string, error) { + return f.fn(ref) +} diff --git a/deps/remote_helm_client.go b/deps/remote_helm_client.go new file mode 100644 index 0000000..c2f26bf --- /dev/null +++ b/deps/remote_helm_client.go @@ -0,0 +1,208 @@ +package deps + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/goccy/go-yaml" + "helm.sh/helm/v3/pkg/chart/loader" +) + +// helmClient resolves Helm chart dependencies remotely. It uses the Helm SDK's +// chart loader (helm.sh/helm/v3/pkg/chart/loader) to parse fetched charts, and +// parses the repository index.yaml directly over HTTP — the SDK's pkg/repo is +// avoided because it transitively requires a k8s API version removed in the +// k8s release this module pins, and pulls oras/containerd. +// +// Pin note: helm.sh/helm/v3 is held at v3.17.x; v3.18+ bumps +// distribution/distribution/v3 past the version registry-scanner requires. +type helmClient struct { + cache RemoteCache +} + +type helmIndex struct { + Entries map[string][]helmIndexEntry `yaml:"entries"` +} + +type helmIndexEntry struct { + Version string `yaml:"version"` + URLs []string `yaml:"urls"` +} + +type fetchedChart struct { + Dir string + Dependencies []chartDepEntry + Version string +} + +// fetchChart resolves a Chart.yaml dependency to an extracted chart directory +// plus its own declared dependencies. The returned cleanup removes the +// extraction dir and must be called by the caller. +func (h *helmClient) fetchChart(ctx context.Context, dep chartDepEntry) (*fetchedChart, func(), error) { + repoURL := strings.TrimSuffix(dep.Repository, "/") + if repoURL == "" { + return nil, nil, notFoundError{dep.Name + " (no repository)"} + } + if strings.HasPrefix(repoURL, "oci://") { + return nil, nil, fmt.Errorf("oci helm repositories are not yet supported") + } + + version, tgzURL, err := h.resolveChart(ctx, repoURL, dep.Name, dep.Version) + if err != nil { + return nil, nil, err + } + tgz, err := h.cache.Fetch(ctx, tgzURL, ttlImmutable) + if err != nil { + return nil, nil, err + } + + dir, err := os.MkdirTemp("", "repomap-chart-*") + if err != nil { + return nil, nil, err + } + cleanup := func() { _ = os.RemoveAll(dir) } + if err := extractTarGz(tgz, dir); err != nil { + cleanup() + return nil, nil, err + } + chartDir, err := findChartDir(dir) + if err != nil { + cleanup() + return nil, nil, err + } + ch, err := loader.Load(chartDir) + if err != nil { + cleanup() + return nil, nil, fmt.Errorf("loading chart %s: %w", dep.Name, err) + } + + deps := make([]chartDepEntry, 0, len(ch.Metadata.Dependencies)) + for _, d := range ch.Metadata.Dependencies { + deps = append(deps, chartDepEntry{Name: d.Name, Version: d.Version, Repository: d.Repository}) + } + return &fetchedChart{Dir: chartDir, Dependencies: deps, Version: version}, cleanup, nil +} + +// resolveChart picks the newest index version satisfying the dependency's +// version constraint and returns its absolute .tgz URL. +func (h *helmClient) resolveChart(ctx context.Context, repoURL, name, constraint string) (version, tgzURL string, err error) { + data, err := h.cache.Fetch(ctx, repoURL+"/index.yaml", ttlIndex) + if err != nil { + return "", "", err + } + var idx helmIndex + if err := yaml.Unmarshal(data, &idx); err != nil { + return "", "", fmt.Errorf("parsing index.yaml for %s: %w", repoURL, err) + } + entries := idx.Entries[name] + if len(entries) == 0 { + return "", "", notFoundError{name + " in " + repoURL} + } + version, url := pickChartVersion(entries, constraint) + if version == "" { + return "", "", notFoundError{name + "@" + constraint} + } + if strings.HasPrefix(url, "oci://") { + return "", "", fmt.Errorf("chart %s@%s is published as an OCI artifact (%s), which is not yet supported", name, version, url) + } + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + url = repoURL + "/" + strings.TrimPrefix(url, "/") + } + return version, url, nil +} + +// pickChartVersion selects the highest entry satisfying constraint (a SemVer +// range). If constraint is not a valid range it falls back to an exact match. +func pickChartVersion(entries []helmIndexEntry, constraint string) (version, url string) { + cons, consErr := semver.NewConstraint(constraint) + var best *semver.Version + for _, e := range entries { + if consErr != nil { + if e.Version == constraint && len(e.URLs) > 0 { + return e.Version, e.URLs[0] + } + continue + } + v, err := semver.NewVersion(e.Version) + if err != nil || !cons.Check(v) || len(e.URLs) == 0 { + continue + } + if best == nil || v.GreaterThan(best) { + best, version, url = v, e.Version, e.URLs[0] + } + } + return version, url +} + +// extractTarGz unpacks a gzipped tar archive into dest, guarding against path +// traversal. +func extractTarGz(data []byte, dest string) error { + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + defer func() { _ = gz.Close() }() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + target := filepath.Join(dest, filepath.Clean(hdr.Name)) + if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("tar entry escapes destination: %s", hdr.Name) + } + if hdr.FileInfo().IsDir() { + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.Create(target) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { //nolint:gosec // chart archives are size-bounded + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + } +} + +// findChartDir returns the directory containing Chart.yaml within an extracted +// chart archive (charts unpack to a single top-level dir named for the chart). +func findChartDir(root string) (string, error) { + if _, err := os.Stat(filepath.Join(root, "Chart.yaml")); err == nil { + return root, nil + } + entries, err := os.ReadDir(root) + if err != nil { + return "", err + } + for _, e := range entries { + if e.IsDir() { + candidate := filepath.Join(root, e.Name()) + if _, err := os.Stat(filepath.Join(candidate, "Chart.yaml")); err == nil { + return candidate, nil + } + } + } + return "", fmt.Errorf("no Chart.yaml found under %s", root) +} diff --git a/deps/remote_helm_client_test.go b/deps/remote_helm_client_test.go new file mode 100644 index 0000000..009e689 --- /dev/null +++ b/deps/remote_helm_client_test.go @@ -0,0 +1,42 @@ +package deps + +import ( + "context" + "strings" + "testing" +) + +func TestPickChartVersion(t *testing.T) { + entries := []helmIndexEntry{ + {Version: "16.1.0", URLs: []string{"u-16.1.0"}}, + {Version: "16.7.27", URLs: []string{"u-16.7.27"}}, + {Version: "17.0.0", URLs: []string{"u-17.0.0"}}, + } + cases := []struct{ constraint, wantVer, wantURL string }{ + {"16.x.x", "16.7.27", "u-16.7.27"}, // newest satisfying the range + {"16.7.27", "16.7.27", "u-16.7.27"}, // exact + {">=17.0.0", "17.0.0", "u-17.0.0"}, + {"99.x", "", ""}, // nothing satisfies + } + for _, tc := range cases { + ver, url := pickChartVersion(entries, tc.constraint) + if ver != tc.wantVer || url != tc.wantURL { + t.Fatalf("pickChartVersion(%q) = (%q,%q), want (%q,%q)", tc.constraint, ver, url, tc.wantVer, tc.wantURL) + } + } +} + +func TestFetchChartOCIDegrades(t *testing.T) { + const repo = "https://charts.bitnami.com/bitnami" + cache := &memCache{blobs: map[string][]byte{ + repo + "/index.yaml": []byte("entries:\n postgresql:\n - version: 16.7.27\n urls:\n - oci://registry-1.docker.io/bitnamicharts/postgresql:16.7.27\n"), + }} + h := &helmClient{cache: cache} + _, _, err := h.fetchChart(context.Background(), chartDepEntry{Name: "postgresql", Version: "16.x.x", Repository: repo}) + if err == nil { + t.Fatal("expected OCI chart to degrade with an error") + } + if got := err.Error(); !strings.Contains(got, "OCI artifact") { + t.Fatalf("error should explain the OCI limitation, got %q", got) + } +} diff --git a/deps/resolve_remote.go b/deps/resolve_remote.go new file mode 100644 index 0000000..4808272 --- /dev/null +++ b/deps/resolve_remote.go @@ -0,0 +1,168 @@ +package deps + +import ( + "context" + "time" + + "github.com/flanksource/repomap/imageupdate" +) + +// defaultRemoteDepth bounds recursive fetching when MaxDepth is 0 (unlimited), +// so the network walk always terminates. +const defaultRemoteDepth = 10 + +// remoteDeps bundles the remote cache-backed resolvers used to recurse into +// chart subcharts and image base images. +type remoteDeps struct { + charts *chartResolver + images *imageResolver +} + +func remoteDepsFromCache(cache RemoteCache) *remoteDeps { + return &remoteDeps{ + charts: &chartResolver{helm: &helmClient{cache: cache}}, + images: newImageResolver(cache), + } +} + +func newRemoteDeps(opts Options) (*remoteDeps, error) { + if opts.remote != nil { + return opts.remote, nil + } + now := time.Now + if opts.Now != nil { + now = opts.Now + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + } + cache, err := newDiskCache(now, runner, imageupdate.NewLabelResolver()) + if err != nil { + return nil, err + } + return remoteDepsFromCache(cache), nil +} + +// resolveRemote recursively expands chart-subchart and image-base children in +// place. It is gated by the caller to MaxDepth != 1. Remote failures degrade to +// warnings; only cache initialization can fail hard. +func resolveRemote(ctx context.Context, roots []*Node, opts Options) ([]Warning, error) { + rd, err := newRemoteDeps(opts) + if err != nil { + return nil, err + } + limit := opts.MaxDepth + if limit == 0 { + limit = defaultRemoteDepth + } + visited := map[string]bool{} + var warnings []Warning + for _, root := range roots { + warnings = append(warnings, rd.expandNode(ctx, root, limit, visited)...) + } + return warnings, nil +} + +// expandNode fetches a node's remote children (if it is a fetchable chart or +// image and within depth/visited bounds), then recurses into all children +// (existing offline ones plus the newly fetched). +func (rd *remoteDeps) expandNode(ctx context.Context, node *Node, limit int, visited map[string]bool) []Warning { + if node == nil { + return nil + } + var warnings []Warning + if node.Depth < limit { + if key := remoteKey(node); key != "" && !visited[key] { + visited[key] = true + warnings = append(warnings, rd.fetchChildren(ctx, node)...) + } + } + for _, child := range node.Children { + warnings = append(warnings, rd.expandNode(ctx, child, limit, visited)...) + } + return warnings +} + +func (rd *remoteDeps) fetchChildren(ctx context.Context, node *Node) []Warning { + switch { + case isFetchableChart(node): + children, version, warns := rd.charts.expand(ctx, chartDepEntry{ + Name: node.Name, + Version: node.Version, + Repository: node.Source, + }, node.Depth) + if version != "" { + node.Version = version + node.ID = NodeID(ManagerHelm, node.Name, version) + } + node.Children = append(node.Children, children...) + return nodeWarnings(node, warns) + case isFetchableImage(node): + bases, warns := rd.images.baseImages(ctx, imageRefOf(node)) + for _, b := range bases { + node.Children = append(node.Children, baseImageNode(b, node.Depth+1)) + } + return nodeWarnings(node, warns) + } + return nil +} + +// isFetchableChart matches a Helm dependency that can be resolved from a remote +// repository — excludes the local Chart.yaml root and the synthetic k8s wrapper. +func isFetchableChart(n *Node) bool { + return n.Manager == ManagerHelm && n.Source != "" && + n.Source != "Chart.yaml" && n.Source != "kubernetes manifests" +} + +// isFetchableImage matches a real image node — excludes the synthetic +// "container images" k8s wrapper root. +func isFetchableImage(n *Node) bool { + return n.Manager == ManagerImage && n.Source != "" && n.Source != "kubernetes manifests" +} + +func remoteKey(n *Node) string { + switch { + case isFetchableChart(n): + return "helm:" + n.Name + "@" + n.Version + case isFetchableImage(n): + return "image:" + imageRefOf(n) + } + return "" +} + +// imageRefOf returns the original image reference to resolve; Source holds the +// full ref captured at discovery (e.g. "ghcr.io/acme/app:1.2.3"). +func imageRefOf(n *Node) string { + if n.Source != "" && n.Source != "kubernetes manifests" { + return n.Source + } + if n.Version != "" { + return n.Name + ":" + n.Version + } + return n.Name +} + +func baseImageNode(b imageRef, depth int) *Node { + ref := b.Name + if b.Version != "" { + ref += ":" + b.Version + } + if b.Digest != "" { + ref += "@" + b.Digest + } + n := NewNode(ManagerImage, b.Name, b.Version) + n.Depth = depth + n.Direct = false + n.Scope = "base" + n.Source = ref // makes the base itself fetchable (base-of-base recursion) + return n +} + +func nodeWarnings(node *Node, msgs []string) []Warning { + out := make([]Warning, 0, len(msgs)) + for _, m := range msgs { + out = append(out, Warning{Manager: node.Manager, Project: node.Name, Message: m}) + } + return out +} diff --git a/deps/resolve_remote_test.go b/deps/resolve_remote_test.go new file mode 100644 index 0000000..9eb47e8 --- /dev/null +++ b/deps/resolve_remote_test.go @@ -0,0 +1,196 @@ +package deps + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + "time" +) + +// memCache is an in-memory RemoteCache for recursion tests: no network, no disk +// reads beyond the git fixture dirs the test writes. +type memCache struct { + blobs map[string][]byte + images map[string]ImageConfig + gitDirs map[string]string // keyed by url, any ref +} + +func (m *memCache) Fetch(_ context.Context, url string, _ time.Duration) ([]byte, error) { + if b, ok := m.blobs[url]; ok { + return b, nil + } + return nil, notFoundError{url} +} + +func (m *memCache) GitRepo(_ context.Context, url, _ string) (string, error) { + if d, ok := m.gitDirs[url]; ok { + return d, nil + } + return "", notFoundError{url} +} + +func (m *memCache) ImageConfig(_ context.Context, ref string) (ImageConfig, error) { + if c, ok := m.images[ref]; ok { + return c, nil + } + return ImageConfig{}, notFoundError{ref} +} + +func buildChartTgz(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, content := range files { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// chartRootWithSubchart builds an offline chart root carrying a single subchart +// leaf, as the offline scan would produce it. +func chartRootWithSubchart(name, version, repo string) *Node { + root := NewNode(ManagerHelm, "parent", "1.0.0") + root.Source = "Chart.yaml" + root.Depth = 0 + sub := NewNode(ManagerHelm, name, version) + sub.Source = repo + sub.Scope = "dependencies" + sub.Direct = true + sub.Depth = 1 + root.Children = []*Node{sub} + return root +} + +func TestResolveRemoteChartRecursion(t *testing.T) { + const repo = "https://charts.example.com" + tgz := buildChartTgz(t, map[string]string{ + "flanksource-ui/Chart.yaml": "apiVersion: v2\nname: flanksource-ui\nversion: 1.4.212\n" + + "dependencies:\n - name: common\n version: 2.0.0\n repository: https://other.example.com\n", + "flanksource-ui/values.yaml": "image:\n repository: flanksource/ui\n tag: \"3.2.1\"\n", + }) + cache := &memCache{ + blobs: map[string][]byte{ + repo + "/index.yaml": []byte("entries:\n flanksource-ui:\n - version: 1.4.212\n urls:\n - " + repo + "/flanksource-ui-1.4.212.tgz\n"), + repo + "/flanksource-ui-1.4.212.tgz": tgz, + }, + } + + root := chartRootWithSubchart("flanksource-ui", "1.4.212", repo) + opts := Options{MaxDepth: 0, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + + ui := findChild(root, "flanksource-ui") + if ui == nil { + t.Fatalf("subchart node missing") + } + if common := findChild(ui, "common"); common == nil || common.Manager != ManagerHelm || common.Depth != 2 { + t.Fatalf("nested subchart 'common' not resolved: %#v (children: %v)", common, childNames(ui)) + } + img := findChild(ui, "flanksource/ui") + if img == nil || img.Manager != ManagerImage || img.Version != "3.2.1" || img.Depth != 2 { + t.Fatalf("subchart values image not harvested: %#v (children: %v)", img, childNames(ui)) + } +} + +func TestResolveRemoteImageBaseViaLabel(t *testing.T) { + cache := &memCache{images: map[string]ImageConfig{ + "app:1.0": {Labels: map[string]string{labelBaseName: "debian:12"}}, + "debian:12": {Labels: map[string]string{}}, + }} + root := imageRootWithImage("app:1.0") + opts := Options{MaxDepth: 0, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + app := findChild(root, "app") + base := findChild(app, "debian") + if base == nil || base.Version != "12" || base.Depth != 2 { + t.Fatalf("base image from OCI label not attached: %#v (children: %v)", base, childNames(app)) + } +} + +func TestResolveRemoteImageBaseViaDockerfile(t *testing.T) { + repoDir := t.TempDir() + if err := os.WriteFile(filepath.Join(repoDir, "Dockerfile"), + []byte("FROM golang:1.22 AS build\nFROM gcr.io/distroless/static:nonroot\n"), 0o644); err != nil { + t.Fatal(err) + } + cache := &memCache{ + images: map[string]ImageConfig{"ghcr.io/acme/app:1.0": {Labels: map[string]string{labelSource: "https://github.com/acme/app"}}}, + gitDirs: map[string]string{"https://github.com/acme/app": repoDir}, + } + root := imageRootWithImage("ghcr.io/acme/app:1.0") + opts := Options{MaxDepth: 2, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + app := findChild(root, "ghcr.io/acme/app") + if app == nil { + t.Fatalf("app node missing (children: %v)", childNames(root)) + } + // Both FROM lines reference external images (golang is the builder's base); + // only a bare `FROM ` reference would be excluded as internal. + if base := findChild(app, "gcr.io/distroless/static"); base == nil || base.Version != "nonroot" { + t.Fatalf("runtime FROM base not attached: %#v (children: %v)", base, childNames(app)) + } + if builder := findChild(app, "golang"); builder == nil || builder.Version != "1.22" { + t.Fatalf("builder-stage FROM base not attached: %#v (children: %v)", builder, childNames(app)) + } +} + +func TestResolveRemoteDepthLimit(t *testing.T) { + cache := &memCache{images: map[string]ImageConfig{ + "a:1": {Labels: map[string]string{labelBaseName: "b:1"}}, + "b:1": {Labels: map[string]string{labelBaseName: "c:1"}}, + "c:1": {Labels: map[string]string{labelBaseName: "d:1"}}, + }} + root := imageRootWithImage("a:1") // a is Depth 1 + opts := Options{MaxDepth: 2, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + a := findChild(root, "a") + b := findChild(a, "b") + if b == nil || b.Depth != 2 { + t.Fatalf("expected b at depth 2, got %#v", b) + } + if c := findChild(b, "c"); c != nil { + t.Fatalf("depth 2 limit should stop fetching past b, but found c: %#v", c) + } +} + +func imageRootWithImage(ref string) *Node { + root := NewNode(ManagerImage, "container images", "") + root.Source = "kubernetes manifests" + root.Depth = 0 + img := imageNodeFromRef(ref) + img.Depth = 1 + img.Direct = true + root.Children = []*Node{img} + return root +} + +func imageNodeFromRef(ref string) *Node { + r := parseImageRef(ref) + n := NewNode(ManagerImage, r.Name, r.Version) + n.Source = ref + return n +} diff --git a/deps/scan.go b/deps/scan.go index 2a18206..c3c61b1 100644 --- a/deps/scan.go +++ b/deps/scan.go @@ -94,6 +94,17 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { } projectsScanned := len(projects) + imageRootCount(roots) + chartRootCount(roots) + // Remote recursion mirrors the transitive gate used by go/maven/gradle: at + // any depth other than 1, expand chart subcharts and image base images by + // fetching remote content. Failures degrade to warnings inside resolveRemote. + if opts.MaxDepth != 1 { + remoteWarnings, err := resolveRemote(ctx, roots, opts) + if err != nil { + return nil, err + } + warnings = append(warnings, remoteWarnings...) + } + filteredRoots := make([]*Node, 0, len(roots)) for _, root := range roots { if filtered := filterAndPrune(root, opts.Filters, opts.MaxDepth); filtered != nil { diff --git a/deps/scan_chart_test.go b/deps/scan_chart_test.go index ce8bd49..a66a7eb 100644 --- a/deps/scan_chart_test.go +++ b/deps/scan_chart_test.go @@ -64,6 +64,7 @@ func TestScanChartSubchartDependencies(t *testing.T) { got, err := Scan(context.Background(), dir, Options{ Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion Now: func() time.Time { return time.Unix(1, 0).UTC() }, }) if err != nil { @@ -99,6 +100,7 @@ func TestScanChartHelmOnlyExcludesImages(t *testing.T) { got, err := Scan(context.Background(), dir, Options{ Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion Now: func() time.Time { return time.Unix(1, 0).UTC() }, }) if err != nil { @@ -117,6 +119,7 @@ func TestScanChartExtractsImagesFromValuesAndTemplates(t *testing.T) { got, err := Scan(context.Background(), dir, Options{ Managers: []Manager{ManagerImage}, + MaxDepth: 1, // offline: do not trigger remote recursion Now: func() time.Time { return time.Unix(1, 0).UTC() }, }) if err != nil { @@ -161,6 +164,7 @@ func TestScanChartSkipsVendoredSubcharts(t *testing.T) { got, err := Scan(context.Background(), dir, Options{ Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion Now: func() time.Time { return time.Unix(1, 0).UTC() }, }) if err != nil { diff --git a/deps/scan_test.go b/deps/scan_test.go index 5efbfac..6589e39 100644 --- a/deps/scan_test.go +++ b/deps/scan_test.go @@ -160,6 +160,7 @@ func TestScanImageAndHelmManifestTargets(t *testing.T) { got, err := Scan(context.Background(), ".", Options{ Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion Now: func() time.Time { return time.Unix(1, 0).UTC() }, }) if err != nil { diff --git a/imageupdate/labels.go b/imageupdate/labels.go new file mode 100644 index 0000000..04e8218 --- /dev/null +++ b/imageupdate/labels.go @@ -0,0 +1,46 @@ +package imageupdate + +import ( + "context" + "fmt" + + "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/image" + "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/options" +) + +// LabelResolver reads an image's config digest and OCI labels from its registry, +// authenticating via the local Docker credential store. It is used by the deps +// recursion to discover an image's source repository +// (org.opencontainers.image.source) and declared base image +// (org.opencontainers.image.base.name/base.digest). +type LabelResolver struct { + newClient RegistryClientFactory +} + +// NewLabelResolver returns a LabelResolver wired to the live registry client. +func NewLabelResolver() *LabelResolver { + return &LabelResolver{newClient: liveRegistryClientFactory(NewKeychainResolver())} +} + +// Labels resolves the config digest and OCI labels for an image reference of the +// form registry/repo:tag[@digest]. +func (l *LabelResolver) Labels(ctx context.Context, ref string) (digest string, labels map[string]string, err error) { + img := image.NewFromIdentifier(ref) + client, err := l.newClient(ctx, img) + if err != nil { + return "", nil, err + } + tagName := "latest" + if img.ImageTag != nil && img.ImageTag.TagName != "" { + tagName = img.ImageTag.TagName + } + manifest, err := client.ManifestForTag(ctx, tagName) + if err != nil { + return "", nil, fmt.Errorf("manifest for %s:%s: %w", img.GetFullNameWithoutTag(), tagName, err) + } + info, err := client.TagMetadata(ctx, manifest, options.NewManifestOptions()) + if err != nil { + return "", nil, fmt.Errorf("metadata for %s:%s: %w", img.GetFullNameWithoutTag(), tagName, err) + } + return info.EncodedDigest(), info.Labels, nil +} From 51cc379666e54bdc228f148a9276e4805076fd0c Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 23 Jun 2026 19:04:11 +0300 Subject: [PATCH 04/13] refactor(deps,imageupdate): unify image and package dependency updates under deps command Consolidate image/helm and package manager (go/npm/pnpm) update logic into a single `deps update` command with unified filtering, version resolution, and interactive selection. The legacy `images update` command now delegates to `deps update --manager image,helm`. Key changes: - Add --latest and --version flags to resolve dependencies non-interactively - Add resource filters (--kind, --namespace, --name, --selector) and name patterns (--image, --chart) for image/helm targets - Make expression argument optional; empty expression matches all dependencies - Support Flux v2 HelmRelease spec.chartRef (OCIRepository/HelmChart) in addition to inline spec.chart - Extract version math, matching, plan rendering, resolution, and prompts into separate files - Add helm credentials support (basic auth, TLS) for private chart repositories - Refactor imageupdate discovery into DiscoverRepoTargets and Filter functions - Add comprehensive tests for --latest, --version, resource filters, and chartRef resolution BREAKING CHANGE: `images update` is deprecated; use `deps update --manager image,helm` instead. The `deps update` expression argument is now optional (was required). --- .grite/export.json | 46 ++ cmd/repomap/deps.go | 88 ++- cmd/repomap/images.go | 96 +-- cmd/repomap/images_test_helpers.go | 62 ++ cmd/repomap/images_update.go | 297 +------ cmd/repomap/images_update_test.go | 146 ---- deps/helm_credentials.go | 112 +++ deps/helm_credentials_test.go | 78 ++ deps/remote_cache.go | 19 +- deps/scan_image.go | 12 +- deps/update.go | 746 ++---------------- deps/update_apply.go | 86 ++ deps/update_image.go | 94 +-- deps/update_match.go | 178 +++++ deps/update_modes.go | 68 ++ deps/update_modes_test.go | 195 +++++ deps/update_plan.go | 131 +++ deps/update_prompt.go | 78 ++ deps/update_resolve.go | 148 ++++ deps/update_test.go | 6 +- deps/update_version.go | 94 +++ imageupdate/chartref.go | 215 +++++ imageupdate/chartref_test.go | 108 +++ imageupdate/discover.go | 140 ++++ imageupdate/extract.go | 60 +- imageupdate/sourceref.go | 150 ++-- imageupdate/sourceref_test.go | 14 +- imageupdate/target.go | 9 + .../helmrelease-chartref-helmchart.yaml | 29 + .../manifests/helmrelease-chartref-oci.yaml | 19 + tracked_yaml.go | 33 + 31 files changed, 2177 insertions(+), 1380 deletions(-) create mode 100644 .grite/export.json create mode 100644 cmd/repomap/images_test_helpers.go delete mode 100644 cmd/repomap/images_update_test.go create mode 100644 deps/helm_credentials.go create mode 100644 deps/helm_credentials_test.go create mode 100644 deps/update_apply.go create mode 100644 deps/update_match.go create mode 100644 deps/update_modes.go create mode 100644 deps/update_modes_test.go create mode 100644 deps/update_plan.go create mode 100644 deps/update_prompt.go create mode 100644 deps/update_resolve.go create mode 100644 deps/update_version.go create mode 100644 imageupdate/chartref.go create mode 100644 imageupdate/chartref_test.go create mode 100644 imageupdate/discover.go create mode 100644 imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml create mode 100644 imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml create mode 100644 tracked_yaml.go diff --git a/.grite/export.json b/.grite/export.json new file mode 100644 index 0000000..f500e0b --- /dev/null +++ b/.grite/export.json @@ -0,0 +1,46 @@ +{ + "meta": { + "schema_version": 1, + "generated_ts": 1782230638451, + "event_count": 2 + }, + "issues": [ + { + "issue_id": "a701ab0f379f53c67a4006a29274b673", + "title": "Fix repomap update", + "state": "closed", + "labels": [ + "priority:medium", + "session:0eee6b92-6966-4968-93a5-486b87b18e25" + ], + "assignees": [], + "created_ts": 1782228570737, + "updated_ts": 1782230638424, + "comment_count": 1 + } + ], + "events": [ + { + "event_id": "eebcf55cb0fa3528523dd496c0982c02289e77beb9a4fbf2be52ebee84108d22", + "issue_id": "a701ab0f379f53c67a4006a29274b673", + "actor": "b1f0880938c36f0a054030e48c354418", + "ts_unix_ms": 1782230638397, + "kind": { + "StateChanged": { + "state": "closed" + } + } + }, + { + "event_id": "0654b3d43459dedc11799a46e7ba51943d4ea07f3ae0379f8e6d9d5a9c11aa90", + "issue_id": "a701ab0f379f53c67a4006a29274b673", + "actor": "b1f0880938c36f0a054030e48c354418", + "ts_unix_ms": 1782230638424, + "kind": { + "LabelRemoved": { + "label": "status:in_progress" + } + } + } + ] +} \ No newline at end of file diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index 2baac85..594e893 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "strings" "github.com/flanksource/clicky" @@ -21,15 +22,23 @@ type DepsOptions struct { } type DepsUpdateOptions struct { - Args []string `json:"args" args:"true" required:"true" help:"Dependency MatchItem expression followed by optional path"` - Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to update: go, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` - Check bool `json:"check" flag:"check" help:"Resolve and list available updates without prompting or writing"` - DryRun bool `json:"dry_run" flag:"dry-run" help:"Show planned dependency updates without running package-manager commands"` + Args []string `json:"args" args:"true" help:"Optional dependency MatchItem expression followed by optional path"` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to update: go, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Kind []string `json:"kind,omitempty" flag:"kind,k" help:"Filter image/helm targets by kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` + Namespace []string `json:"namespace,omitempty" flag:"namespace,n" help:"Filter image/helm targets by namespace (MatchItem syntax)"` + Name []string `json:"name,omitempty" flag:"name" help:"Filter image/helm targets by resource name (MatchItem syntax)"` + Selector []string `json:"selector,omitempty" flag:"selector,l" help:"Filter image/helm targets by label selector, e.g. app=nginx"` + Image []string `json:"image,omitempty" flag:"image" help:"Only container images matching this repo pattern (MatchItem syntax)"` + Chart []string `json:"chart,omitempty" flag:"chart" help:"Only Helm charts matching this name (MatchItem syntax)"` + Latest bool `json:"latest,omitempty" flag:"latest" help:"Resolve each matched dependency to its highest stable version"` + Version string `json:"version,omitempty" flag:"version" help:"Apply this concrete version to all matched dependencies"` + Check bool `json:"check" flag:"check" help:"Resolve and list available updates without prompting or writing"` + DryRun bool `json:"dry_run" flag:"dry-run" help:"Show planned dependency updates without running package-manager commands"` } func (opts DepsOptions) GetName() string { return "deps" } -func (opts DepsUpdateOptions) GetName() string { return "update [path]" } +func (opts DepsUpdateOptions) GetName() string { return "update [expr] [path]" } func (opts DepsOptions) Help() api.Text { return clicky.Text(`Generate dependency graphs for Go, Maven, Gradle, npm, pnpm, image, and Helm dependencies. @@ -77,11 +86,17 @@ EXAMPLES: func (opts DepsUpdateOptions) Help() api.Text { return clicky.Text(`Update direct package, image, and Helm chart dependencies. -The required expr argument uses commons MatchItem syntax and is matched against +The optional expr argument uses commons MatchItem syntax and is matched against dependency names, manager-qualified names, versions, and scopes. Manifest path -matching is explicit with path: or file:. Matched direct -dependencies are resolved to published versions, then repomap prompts for which -dependencies and versions to apply. Applied updates are staged with git add +matching is explicit with path: or file:. With no expr, every +matched dependency is considered. Image and Helm targets (from git-tracked +Kubernetes/Flux manifests, including HelmRelease spec.chartRef OCIRepository and +HelmChart sources) can be further narrowed with --kind/--namespace/--name/ +--selector and the --image/--chart name patterns. + +By default repomap prompts for which dependencies and versions to apply. Use +--latest to resolve each to its highest stable version, or --version to apply a +concrete version, both non-interactively. Applied updates are staged with git add (manifests plus lockfiles); --dry-run and --check never stage. Use --check to list updateable dependencies without prompting or writing. @@ -89,8 +104,9 @@ Use --check to list updateable dependencies without prompting or writing. EXAMPLES: repomap deps update 'github.com/flanksource/*' repomap deps update '*' --check + repomap deps update --manager helm -k HelmRelease --latest + repomap deps update --manager image -n default --version 1.27.0 repomap deps update 'path:apps/*/package.json' - repomap deps update 'image:ghcr.io/flanksource/*' --manager image repomap deps update 'helm:mission-control' --manager helm repomap deps update 'npm:@flanksource/*' ./web --manager npm repomap deps update 'left-pad,!*beta*' --dry-run`) @@ -129,17 +145,11 @@ func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { } func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { - if len(opts.Args) == 0 { - return nil, fmt.Errorf("dependency update expression is required") - } - if len(opts.Args) > 2 { - return nil, fmt.Errorf("expected [path], got %d arguments", len(opts.Args)) - } - path := "." - if len(opts.Args) == 2 { - path = opts.Args[1] + expr, rawPath, err := parseDepsUpdateArgs(opts.Args) + if err != nil { + return nil, err } - path, err := resolvePath(path) + path, err := resolvePath(rawPath) if err != nil { return nil, err } @@ -147,9 +157,21 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { if err != nil { return nil, err } + var expression []string + if expr != "" { + expression = []string{expr} + } plans, err := depgraph.Update(ctx, path, depgraph.UpdateOptions{ Managers: managers, - Expression: []string{opts.Args[0]}, + Expression: expression, + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, + Image: opts.Image, + Chart: opts.Chart, + Latest: opts.Latest, + Version: opts.Version, Check: opts.Check, DryRun: opts.DryRun, }) @@ -159,6 +181,30 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { return api.NewTableFrom(plans), nil } +// parseDepsUpdateArgs interprets the optional positional [expr] [path]. With one +// argument, an existing directory is treated as the path and anything else as the +// expression, so `deps update ./clusters` and `deps update 'left-pad'` both work. +func parseDepsUpdateArgs(args []string) (expr, path string, err error) { + switch len(args) { + case 0: + return "", ".", nil + case 1: + if isExistingDir(args[0]) { + return "", args[0], nil + } + return args[0], ".", nil + case 2: + return args[0], args[1], nil + default: + return "", "", fmt.Errorf("expected [expr] [path], got %d arguments", len(args)) + } +} + +func isExistingDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + func parseManagers(values []string) ([]depgraph.Manager, error) { parts := splitCommaArgs(values) if len(parts) == 0 { diff --git a/cmd/repomap/images.go b/cmd/repomap/images.go index 57b23b6..ec78f96 100644 --- a/cmd/repomap/images.go +++ b/cmd/repomap/images.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "path/filepath" "strings" "github.com/flanksource/commons/collections" @@ -10,9 +9,21 @@ import ( "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" - "github.com/flanksource/repomap/kubernetes" ) +// versionOnly strips an image/chart current value down to its tag/version +// (dropping any registry/repo prefix and digest suffix). +func versionOnly(currentValue string) string { + if i := strings.LastIndex(currentValue, ":"); i >= 0 { + v := currentValue[i+1:] + if at := strings.Index(v, "@"); at >= 0 { + v = v[:at] + } + return v + } + return currentValue +} + // resolveConcurrency bounds how many registry/Helm version lookups run at once. const resolveConcurrency = 8 @@ -41,8 +52,9 @@ type imageFilterOptions struct { } // discoverAndFilter resolves the scan path, discovers every image/chart target -// in the repo, and applies the shared resource + image/chart filters. It returns -// the matching targets and the HelmRepository source index for chart resolution. +// in the repo (via the shared imageupdate discovery, which also resolves chart +// sources), and applies the shared resource + image/chart filters. It returns the +// matching targets and the source index for chart resolution. func discoverAndFilter(opts imageFilterOptions) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, *repomap.ArchConf, error) { path, err := resolvePath(opts.Path) if err != nil { @@ -53,82 +65,12 @@ func discoverAndFilter(opts imageFilterOptions) ([]imageupdate.UpdateTarget, *im return nil, nil, nil, fmt.Errorf("failed to load config: %w", err) } - targets, sourceIndex, err := discoverTargets(conf, path) + res, err := imageupdate.DiscoverRepoTargets(conf, path) if err != nil { return nil, nil, nil, err } matcher := repomap.NewResourceMatcher(opts.Kind, opts.Namespace, opts.Name, opts.Selector) - targets = filterTargets(targets, matcher, opts) - return targets, sourceIndex, conf, nil -} - -// discoverTargets reads every tracked YAML file, builds the kustomize/Flux tree -// so HelmRepository sources can be resolved through Kustomization namespace -// transformers, indexes the HelmRepositories, and extracts update targets from -// files under the scan prefix. -func discoverTargets(conf *repomap.ArchConf, scanPath string) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, error) { - files, err := gitListFiles(conf.RepoPath()) - if err != nil { - return nil, nil, fmt.Errorf("failed to list files: %w", err) - } - - var prefix string - if rel, err := filepath.Rel(conf.RepoPath(), scanPath); err == nil && rel != "." { - prefix = rel + string(filepath.Separator) - } - - // Pass 0: read all tracked YAML (paths are repo-relative POSIX from git). - contents := map[string]string{} - for _, f := range files { - if !kubernetes.IsYaml(f) { - continue - } - content, err := conf.ReadFileWithFallback(f, "") - if err != nil { - continue - } - contents[f] = content - } - - // Pass 1: build the kustomize/Flux tree and the HelmRepository index. - tree := imageupdate.BuildKustomizeTree(contents) - sourceIndex := imageupdate.NewSourceIndex(tree) - - // Pass 2: index sources repo-wide; extract targets only under the scan prefix. - var targets []imageupdate.UpdateTarget - for f, content := range contents { - _ = sourceIndex.IndexHelmRepositories(f, content) - - if prefix != "" && !strings.HasPrefix(f, prefix) { - continue - } - fileTargets, err := imageupdate.ExtractTargets(f, content) - if err != nil { - continue - } - targets = append(targets, fileTargets...) - } - return targets, sourceIndex, nil -} - -func filterTargets(targets []imageupdate.UpdateTarget, matcher repomap.ResourceMatcher, opts imageFilterOptions) []imageupdate.UpdateTarget { - var out []imageupdate.UpdateTarget - for _, t := range targets { - if !matcher.MatchesRef(t.Ref) { - continue - } - if t.Kind == imageupdate.TargetImage && len(opts.Image) > 0 { - if matched, _ := collections.MatchItem(t.Image.GetFullNameWithoutTag(), opts.Image...); !matched { - continue - } - } - if t.Kind == imageupdate.TargetChart && len(opts.Chart) > 0 { - if matched, _ := collections.MatchItem(t.ChartName, opts.Chart...); !matched { - continue - } - } - out = append(out, t) - } - return out + targets := imageupdate.Filter(res.Targets, matcher, opts.Image, opts.Chart) + return targets, res.Index, conf, nil } diff --git a/cmd/repomap/images_test_helpers.go b/cmd/repomap/images_test_helpers.go new file mode 100644 index 0000000..700b081 --- /dev/null +++ b/cmd/repomap/images_test_helpers.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/image" + "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/registry" + "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/registry/mocks" + "github.com/stretchr/testify/mock" + + "github.com/flanksource/repomap" + "github.com/flanksource/repomap/imageupdate" +) + +const deploymentManifest = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: default +spec: + template: + spec: + containers: + - name: web + image: nginx:1.25.3 # keep me +` + +func fakeImageResolver(tags []string) *imageupdate.Resolver { + return &imageupdate.Resolver{ + NewRegistryClient: func(ctx context.Context, img *image.ContainerImage) (registry.RegistryClient, error) { + m := &mocks.RegistryClient{} + m.On("Tags", mock.Anything).Return(tags, nil) + return m, nil + }, + } +} + +// writeRepo creates a temp git repo with one manifest and a repomap conf rooted +// there. The manifest is committed so git ls-files discovers it. +func writeRepo(t *testing.T) (*repomap.ArchConf, string) { + t.Helper() + dir := t.TempDir() + if out, err := exec.Command("git", "-C", dir, "init").CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, out) + } + rel := "deploy.yaml" + if err := os.WriteFile(filepath.Join(dir, rel), []byte(deploymentManifest), 0o644); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("git", "-C", dir, "add", rel).CombinedOutput(); err != nil { + t.Fatalf("git add: %v: %s", err, out) + } + conf, err := repomap.GetConf(dir) + if err != nil { + t.Fatal(err) + } + return conf, rel +} diff --git a/cmd/repomap/images_update.go b/cmd/repomap/images_update.go index 96ab8a8..a31b432 100644 --- a/cmd/repomap/images_update.go +++ b/cmd/repomap/images_update.go @@ -3,19 +3,15 @@ package main import ( "context" "fmt" - "path/filepath" - "strings" + "os" "github.com/flanksource/clicky" "github.com/flanksource/clicky/api" - "github.com/flanksource/clicky/task" - flanksourceContext "github.com/flanksource/commons/context" - - "github.com/flanksource/repomap" - "github.com/flanksource/repomap/imageupdate" - "github.com/flanksource/repomap/kubernetes" + depgraph "github.com/flanksource/repomap/deps" ) +// UpdateImageOptions are the flags for the deprecated `images update` command, +// which now delegates to `deps update --manager image,helm`. type UpdateImageOptions struct { imageFilterOptions Latest bool `json:"latest" flag:"latest" help:"Resolve each target to the highest stable semver"` @@ -26,276 +22,45 @@ type UpdateImageOptions struct { func (opts UpdateImageOptions) GetName() string { return "update" } func (opts UpdateImageOptions) Help() api.Text { - return clicky.Text(`Update container image tags and Helm chart versions in tracked manifests. - -Discovers apps/v1 workload images and Flux HelmRelease chart versions in -git-tracked YAML, resolves the target version from the container registry or -Helm repository, and edits the manifest in place (preserving comments). + return clicky.Text(`DEPRECATED: use 'repomap deps update --manager image,helm' instead. -With --latest the highest stable version is chosen. With --version a specific -version is applied. With neither, an interactive picker lists available versions. +Update container image tags and Helm chart versions in tracked manifests. This +command now delegates to 'deps update', which additionally stages applied edits +with git and resolves HelmRelease spec.chartRef (OCIRepository/HelmChart) charts. EXAMPLES: - repomap images update -n default -k HelmRelease --latest - repomap images update -k Deployment --image nginx --version 1.27.0 - repomap images update -k HelmRelease --dry-run`) + repomap deps update --manager helm -n default -k HelmRelease --latest + repomap deps update --manager image -k Deployment --image nginx --version 1.27.0`) } func init() { - cmd := clicky.AddCommand(imagesCmd, UpdateImageOptions{}, runUpdateImage) - cmd.Short = "Update image tags and Helm chart versions in tracked manifests" + cmd := clicky.AddNamedCommandWithContext("update", imagesCmd, UpdateImageOptions{}, runUpdateImage) + cmd.Short = "(deprecated) Update image tags and Helm chart versions; use 'deps update'" } -// UpdatePlan is one resolved (and possibly applied) version change. -type UpdatePlan struct { - Ref kubernetes.KubernetesRef `json:"ref"` - Kind imageupdate.TargetKind `json:"kind"` - File string `json:"file"` - Field string `json:"field"` - OldValue string `json:"old_value"` - NewValue string `json:"new_value"` - Written bool `json:"written"` - DryRun bool `json:"dry_run"` - Skipped string `json:"skipped,omitempty"` -} +// runUpdateImage maps the legacy image-update flags onto deps.Update with the +// image and helm managers, so the two commands share one resolution/apply path. +func runUpdateImage(ctx context.Context, opts UpdateImageOptions) (any, error) { + fmt.Fprintln(os.Stderr, "warning: 'images update' is deprecated; use 'repomap deps update --manager image,helm'") -func (p UpdatePlan) Pretty() api.Text { - t := p.Ref.Pretty().Space() - if p.Skipped != "" { - return t.Append("skipped: "+p.Skipped, "text-muted") - } - t = t.Append(kubernetes.VersionChange{OldVersion: p.OldValue, NewVersion: p.NewValue}.Pretty()) - switch { - case p.DryRun: - t = t.Space().Append("(dry-run)", "text-yellow-600") - case p.Written: - t = t.Space().Append("written", "text-green-600") - } - return t -} - -func (UpdatePlan) Columns() []api.ColumnDef { - return []api.ColumnDef{ - api.Column("resource").Label("Resource").Build(), - api.Column("file").Label("File").Build(), - api.Column("change").Label("Change").Build(), - api.Column("status").Label("Status").Build(), - } -} - -func (p UpdatePlan) Row() map[string]any { - row := map[string]any{ - "resource": p.Ref.Pretty(), - "file": clicky.Text(p.File, "font-mono"), - "change": kubernetes.VersionChange{OldVersion: p.OldValue, NewVersion: p.NewValue}.Pretty(), - } - switch { - case p.Skipped != "": - row["status"] = clicky.Text(p.Skipped, "text-muted") - case p.DryRun: - row["status"] = clicky.Text("dry-run", "text-yellow-600") - case p.Written: - row["status"] = clicky.Text("written", "text-green-600") - } - return row -} - -func runUpdateImage(opts UpdateImageOptions) (any, error) { - if opts.Latest && opts.Version != "" { - return nil, fmt.Errorf("--latest and --version are mutually exclusive") - } - - targets, sourceIndex, conf, err := discoverAndFilter(opts.imageFilterOptions) + path, err := resolvePath(opts.Path) if err != nil { return nil, err } - if len(targets) == 0 { - return nil, fmt.Errorf("no matching image or chart targets found") - } - - resolver := imageupdate.NewResolver() - ctx := context.Background() - - // Resolve chart sources up front (cheap, no network) so the concurrent - // version lookups have a repo URL to query. - for i := range targets { - if targets[i].Kind == imageupdate.TargetChart { - if err := sourceIndex.Resolve(&targets[i]); err != nil { - return nil, err - } - } - } - - var plans []UpdatePlan - if opts.Latest || opts.Version != "" { - plans = resolveConcurrently(ctx, resolver, conf, targets, opts) - } else { - // Interactive picker must run serially (it prompts the user per target). - for _, t := range targets { - plans = append(plans, planTarget(ctx, resolver, conf, t, opts, nil)) - } - } - return api.NewTableFrom(plans), nil -} - -// resolveConcurrently runs each target's version lookup as its own clicky task, -// then applies the resulting edits serially (concurrent writes to the same file -// would race; resolution is the slow, parallelisable part). -func resolveConcurrently(ctx context.Context, resolver *imageupdate.Resolver, conf *repomap.ArchConf, targets []imageupdate.UpdateTarget, opts UpdateImageOptions) []UpdatePlan { - type resolved struct { - newValue string - skipped string - err error - } - results := make([]resolved, len(targets)) - group := task.StartGroup[int]("Resolving image versions", task.WithConcurrency(resolveConcurrency)) - for i, t := range targets { - idx, target := i, t - displayFile := displayPathForRepoFile(conf, target.File) - group.Add(taskName(target, displayFile), func(ctx flanksourceContext.Context, tk *task.Task) (int, error) { - newValue, skipped, err := resolveNewValue(ctx, resolver, target, opts, tk) - results[idx] = resolved{newValue, skipped, err} - return idx, nil - }) - } - _, _ = group.GetResults() - - plans := make([]UpdatePlan, len(targets)) - for i, t := range targets { - plans[i] = applyResolved(conf, t, results[i].newValue, results[i].skipped, results[i].err, opts) - } - return plans -} - -// planTarget resolves the new version for a target and applies the edit. tk may -// be nil when not running inside a task. -func planTarget(ctx context.Context, resolver *imageupdate.Resolver, conf *repomap.ArchConf, t imageupdate.UpdateTarget, opts UpdateImageOptions, tk *task.Task) UpdatePlan { - newValue, skipped, err := resolveNewValue(ctx, resolver, t, opts, tk) - return applyResolved(conf, t, newValue, skipped, err, opts) -} - -// resolveNewValue determines the replacement value for a target without writing. -// It returns a skip reason instead of a value when no update applies. -func resolveNewValue(ctx context.Context, resolver *imageupdate.Resolver, t imageupdate.UpdateTarget, opts UpdateImageOptions, tk *task.Task) (newValue, skipped string, err error) { - logf(tk, "resolving version") - newVersion, err := chooseVersion(ctx, resolver, t, opts) - if err != nil { - return "", "", err - } - if newVersion == "" { - return "", "no version selected", nil - } - newValue = newVersion - if t.Kind == imageupdate.TargetImage { - newValue, err = resolver.NewImageValue(ctx, t, newVersion) - if err != nil { - return "", "", err - } - } - if newValue == t.CurrentValue { - return "", "already up to date", nil - } - return newValue, "", nil -} - -// applyResolved builds the plan and applies the edit (unless dry-run, skipped, -// or errored). -func applyResolved(conf *repomap.ArchConf, t imageupdate.UpdateTarget, newValue, skipped string, resErr error, opts UpdateImageOptions) UpdatePlan { - plan := UpdatePlan{ - Ref: t.Ref, - Kind: t.Kind, - File: displayPathForRepoFile(conf, t.File), - Field: t.FieldJSONPath, - OldValue: t.CurrentValue, - DryRun: opts.DryRun, - } - if resErr != nil { - plan.Skipped = resErr.Error() - return plan - } - if skipped != "" { - plan.Skipped = skipped - return plan - } - plan.NewValue = newValue - - absFile := filepath.Join(conf.RepoPath(), t.File) - if _, err := imageupdate.ApplyEdit(absFile, t, newValue, opts.DryRun); err != nil { - plan.Skipped = err.Error() - return plan - } - plan.Written = !opts.DryRun - return plan -} - -func logf(tk *task.Task, format string, args ...any) { - if tk != nil { - tk.Infof(format, args...) - } -} - -// chooseVersion returns the target version per the CLI mode: explicit --version, -// resolved --latest, or an interactive pick from available candidates. -func chooseVersion(ctx context.Context, resolver *imageupdate.Resolver, t imageupdate.UpdateTarget, opts UpdateImageOptions) (string, error) { - if opts.Version != "" { - available, err := resolver.Available(ctx, t) - if err != nil { - return "", err - } - if !contains(available, opts.Version) { - return "", fmt.Errorf("%s: version %q is not available (have: %s)", - t.CurrentValue, opts.Version, strings.Join(available, ", ")) - } - return opts.Version, nil - } - if opts.Latest { - return resolver.ResolveLatest(ctx, t) - } - - available, err := resolver.Available(ctx, t) - if err != nil { - return "", err - } - if len(available) == 0 { - return "", fmt.Errorf("no available versions for %s", t.CurrentValue) - } - return pickVersion(t, available), nil -} - -func pickVersion(t imageupdate.UpdateTarget, available []string) string { - title := fmt.Sprintf("Select version for %s/%s (current %s)", t.Ref.Kind, t.Ref.Name, t.CurrentValue) - choice, ok := clicky.PromptSelect(available, clicky.PromptSelectOptions[string]{ - Title: title, - Render: func(v string) api.Textable { - text := clicky.Text(v) - if v == versionOnly(t.CurrentValue) { - text = text.Space().Append("(current)", "text-muted") - } - return text - }, + plans, err := depgraph.Update(ctx, path, depgraph.UpdateOptions{ + Managers: []depgraph.Manager{depgraph.ManagerImage, depgraph.ManagerHelm}, + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, + Image: opts.Image, + Chart: opts.Chart, + Latest: opts.Latest, + Version: opts.Version, + DryRun: opts.DryRun, }) - if !ok { - return "" - } - return choice -} - -func versionOnly(currentValue string) string { - if i := strings.LastIndex(currentValue, ":"); i >= 0 { - v := currentValue[i+1:] - if at := strings.Index(v, "@"); at >= 0 { - v = v[:at] - } - return v - } - return currentValue -} - -func contains(list []string, v string) bool { - for _, item := range list { - if item == v { - return true - } + if err != nil { + return nil, err } - return false + return api.NewTableFrom(plans), nil } diff --git a/cmd/repomap/images_update_test.go b/cmd/repomap/images_update_test.go deleted file mode 100644 index 18184b2..0000000 --- a/cmd/repomap/images_update_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package main - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/flanksource/repomap" - "github.com/flanksource/repomap/imageupdate" -) - -const deploymentManifest = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: web - namespace: default -spec: - template: - spec: - containers: - - name: web - image: nginx:1.25.3 # keep me -` - -type fakeRegistryClient struct{ tags []string } - -func (f fakeRegistryClient) Tags(context.Context) ([]string, error) { - return f.tags, nil -} - -func (f fakeRegistryClient) Digest(context.Context, string) (string, error) { - return "", nil -} - -func fakeImageResolver(tags []string) *imageupdate.Resolver { - return &imageupdate.Resolver{ - NewRegistryClient: func(context.Context, *imageupdate.ContainerImage) (imageupdate.RegistryClient, error) { - return fakeRegistryClient{tags: tags}, nil - }, - } -} - -// writeRepo creates a temp dir with one manifest and a repomap conf rooted there. -func writeRepo(t *testing.T) (*repomap.ArchConf, string) { - t.Helper() - dir := t.TempDir() - if out, err := exec.Command("git", "-C", dir, "init").CombinedOutput(); err != nil { - t.Fatalf("git init: %v: %s", err, out) - } - rel := "deploy.yaml" - if err := os.WriteFile(filepath.Join(dir, rel), []byte(deploymentManifest), 0o644); err != nil { - t.Fatal(err) - } - conf, err := repomap.GetConf(dir) - if err != nil { - t.Fatal(err) - } - return conf, rel -} - -func TestPlanTarget_LatestDryRun(t *testing.T) { - conf, rel := writeRepo(t) - content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - targets, err := imageupdate.ExtractTargets(rel, string(content)) - if err != nil || len(targets) != 1 { - t.Fatalf("extract: %v (%d targets)", err, len(targets)) - } - - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0", "1.28.0-rc.1"}) - plan := planTarget(context.Background(), resolver, conf, targets[0], - UpdateImageOptions{Latest: true, DryRun: true}, nil) - if plan.Skipped != "" { - t.Fatalf("unexpected skip: %s", plan.Skipped) - } - if plan.NewValue != "nginx:1.27.0" { - t.Errorf("new value = %q, want nginx:1.27.0", plan.NewValue) - } - if plan.Written { - t.Error("dry-run must not write") - } - // file must be untouched - after, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - if string(after) != deploymentManifest { - t.Error("dry-run modified the manifest") - } -} - -func TestPlanTarget_VersionWritesAndPreservesComment(t *testing.T) { - conf, rel := writeRepo(t) - content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - targets, _ := imageupdate.ExtractTargets(rel, string(content)) - - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0"}) - plan := planTarget(context.Background(), resolver, conf, targets[0], - UpdateImageOptions{Version: "1.27.0"}, nil) - if !plan.Written { - t.Fatalf("expected written, skipped=%q", plan.Skipped) - } - after, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - got := splitLine(string(after), 11) - want := " image: nginx:1.27.0 # keep me" - if got != want { - t.Errorf("line 11 = %q, want %q", got, want) - } -} - -func TestPlanTarget_RejectsUnavailableVersion(t *testing.T) { - conf, rel := writeRepo(t) - content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - targets, _ := imageupdate.ExtractTargets(rel, string(content)) - - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0"}) - plan := planTarget(context.Background(), resolver, conf, targets[0], - UpdateImageOptions{Version: "9.9.9"}, nil) - if plan.Skipped == "" || !strings.Contains(plan.Skipped, "not available") { - t.Fatalf("expected skip for unavailable version, got skipped=%q written=%v", plan.Skipped, plan.Written) - } -} - -func splitLine(content string, n int) string { - lines := splitLines(content) - if n < 1 || n > len(lines) { - return "" - } - return lines[n-1] -} - -func splitLines(s string) []string { - var out []string - cur := "" - for _, r := range s { - if r == '\n' { - out = append(out, cur) - cur = "" - continue - } - cur += string(r) - } - if cur != "" { - out = append(out, cur) - } - return out -} diff --git a/deps/helm_credentials.go b/deps/helm_credentials.go new file mode 100644 index 0000000..c8a89a6 --- /dev/null +++ b/deps/helm_credentials.go @@ -0,0 +1,112 @@ +package deps + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "os" + "strings" + + "github.com/goccy/go-yaml" + "helm.sh/helm/v3/pkg/helmpath" +) + +// helmRepoCreds mirrors one entry of Helm's repositories.yaml, carrying the +// authentication a private chart repository needs. +type helmRepoCreds struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Username string `yaml:"username"` + Password string `yaml:"password"` + CertFile string `yaml:"certFile"` + KeyFile string `yaml:"keyFile"` + CAFile string `yaml:"caFile"` + InsecureSkipTLSVerify bool `yaml:"insecure_skip_tls_verify"` +} + +type helmRepoFile struct { + Repositories []helmRepoCreds `yaml:"repositories"` +} + +// helmCredentials reuses the user's `helm repo add` logins so chart discovery +// authenticates to private repositories exactly like the helm CLI. +type helmCredentials struct { + repos []helmRepoCreds +} + +// loadHelmCredentials reads repositories.yaml from $HELM_REPOSITORY_CONFIG or +// Helm's default config path. A missing or unreadable file yields no +// credentials (public repositories keep working). +func loadHelmCredentials() *helmCredentials { + path := os.Getenv("HELM_REPOSITORY_CONFIG") + if path == "" { + path = helmpath.ConfigPath("repositories.yaml") + } + return newHelmCredentials(path) +} + +func newHelmCredentials(path string) *helmCredentials { + data, err := os.ReadFile(path) + if err != nil { + return &helmCredentials{} + } + var f helmRepoFile + if err := yaml.Unmarshal(data, &f); err != nil { + return &helmCredentials{} + } + return &helmCredentials{repos: f.Repositories} +} + +// match returns the credentials for the repository whose URL is the longest +// prefix of url (so index.yaml/.tgz fetches under a repo inherit its login). +func (h *helmCredentials) match(url string) (helmRepoCreds, bool) { + best := -1 + var found helmRepoCreds + for _, r := range h.repos { + base := strings.TrimSuffix(r.URL, "/") + if base == "" || (url != base && !strings.HasPrefix(url, base+"/")) { + continue + } + if len(base) > best { + best, found = len(base), r + } + } + return found, best >= 0 +} + +// authorize applies the matched repository's basic-auth and TLS settings to req, +// returning the http client to use (a custom one only when TLS options apply). +func (h *helmCredentials) authorize(req *http.Request) *http.Client { + if h == nil { + return http.DefaultClient + } + cred, ok := h.match(req.URL.String()) + if !ok { + return http.DefaultClient + } + if cred.Username != "" { + req.SetBasicAuth(cred.Username, cred.Password) + } + return httpClientForCreds(cred) +} + +func httpClientForCreds(cred helmRepoCreds) *http.Client { + if cred.CAFile == "" && cred.CertFile == "" && !cred.InsecureSkipTLSVerify { + return http.DefaultClient + } + tlsCfg := &tls.Config{InsecureSkipVerify: cred.InsecureSkipTLSVerify} //nolint:gosec // honors the user's helm repo setting + if cred.CAFile != "" { + if pem, err := os.ReadFile(cred.CAFile); err == nil { + pool := x509.NewCertPool() + if pool.AppendCertsFromPEM(pem) { + tlsCfg.RootCAs = pool + } + } + } + if cred.CertFile != "" && cred.KeyFile != "" { + if cert, err := tls.LoadX509KeyPair(cred.CertFile, cred.KeyFile); err == nil { + tlsCfg.Certificates = []tls.Certificate{cert} + } + } + return &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} +} diff --git a/deps/helm_credentials_test.go b/deps/helm_credentials_test.go new file mode 100644 index 0000000..af00f45 --- /dev/null +++ b/deps/helm_credentials_test.go @@ -0,0 +1,78 @@ +package deps + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHelmCredentialsMatchLongestPrefix(t *testing.T) { + creds := &helmCredentials{repos: []helmRepoCreds{ + {Name: "broad", URL: "https://charts.example.com"}, + {Name: "narrow", URL: "https://charts.example.com/team/", Username: "u"}, + }} + got, ok := creds.match("https://charts.example.com/team/index.yaml") + if !ok || got.Name != "narrow" { + t.Fatalf("expected longest-prefix match 'narrow', got %q (ok=%v)", got.Name, ok) + } + if other, ok := creds.match("https://unrelated.example.com/index.yaml"); ok { + t.Fatalf("unrelated URL should not match, got %q", other.Name) + } +} + +func TestCacheFetchSendsHelmBasicAuth(t *testing.T) { + var gotUser, gotPass string + var hadAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUser, gotPass, hadAuth = r.BasicAuth() + if !hadAuth { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte("index-body")) + })) + defer srv.Close() + + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + c.get = c.httpGet // exercise the real HTTP path + c.helmAuth = &helmCredentials{repos: []helmRepoCreds{ + {Name: "private", URL: srv.URL, Username: "alice", Password: "s3cret"}, + }} + + data, err := c.Fetch(context.Background(), srv.URL+"/index.yaml", ttlIndex) + if err != nil { + t.Fatalf("fetch with credentials failed: %v", err) + } + if string(data) != "index-body" { + t.Fatalf("body = %q, want index-body", data) + } + if !hadAuth || gotUser != "alice" || gotPass != "s3cret" { + t.Fatalf("server did not receive expected basic auth (user=%q pass set=%v hadAuth=%v)", gotUser, gotPass != "", hadAuth) + } +} + +func TestCacheFetchNoAuthForUnmatchedRepo(t *testing.T) { + var hadAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _, hadAuth = r.BasicAuth() + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + c.get = c.httpGet + c.helmAuth = &helmCredentials{repos: []helmRepoCreds{ + {Name: "other", URL: "https://charts.other.com", Username: "x", Password: "y"}, + }} + + if _, err := c.Fetch(context.Background(), srv.URL+"/index.yaml", ttlIndex); err != nil { + t.Fatal(err) + } + if hadAuth { + t.Fatalf("credentials must not be sent to a non-matching repository") + } +} diff --git a/deps/remote_cache.go b/deps/remote_cache.go index bfbf1a1..d097394 100644 --- a/deps/remote_cache.go +++ b/deps/remote_cache.go @@ -60,12 +60,13 @@ type RemoteCache interface { } type diskCache struct { - root string - now func() time.Time - get func(ctx context.Context, url string) ([]byte, error) - runner CommandRunner - labels labelResolver - group singleflight.Group + root string + now func() time.Time + get func(ctx context.Context, url string) ([]byte, error) + runner CommandRunner + labels labelResolver + helmAuth *helmCredentials + group singleflight.Group } // newDiskCache builds the production cache rooted at os.UserCacheDir()/repomap. @@ -78,7 +79,7 @@ func newDiskCache(now func() time.Time, runner CommandRunner, labels labelResolv if err := os.MkdirAll(root, 0o755); err != nil { return nil, fmt.Errorf("creating cache dir %s: %w", root, err) } - c := &diskCache{root: root, now: now, runner: runner, labels: labels} + c := &diskCache{root: root, now: now, runner: runner, labels: labels, helmAuth: loadHelmCredentials()} c.get = c.httpGet return c, nil } @@ -199,7 +200,9 @@ func (c *diskCache) httpGet(ctx context.Context, url string) ([]byte, error) { if err != nil { return nil, err } - resp, err := http.DefaultClient.Do(req) + // Reuse the user's `helm repo add` login (basic auth / TLS) for matching repos. + client := c.helmAuth.authorize(req) + resp, err := client.Do(req) if err != nil { return nil, err } diff --git a/deps/scan_image.go b/deps/scan_image.go index 19c7f58..139093a 100644 --- a/deps/scan_image.go +++ b/deps/scan_image.go @@ -14,7 +14,7 @@ func discoverImageDependencyRoots(root string, managers []Manager) ([]*Node, []W if err != nil { return nil, nil, err } - targets, sourceIndex, err := discoverImageTargets(conf, root) + res, err := imageupdate.DiscoverRepoTargets(conf, root) if err != nil { return nil, nil, err } @@ -22,16 +22,14 @@ func discoverImageDependencyRoots(root string, managers []Manager) ([]*Node, []W var imageChildren []*Node var helmChildren []*Node var warnings []Warning - for _, target := range targets { + for _, w := range res.Warnings { + warnings = append(warnings, Warning{Manager: ManagerHelm, Project: w.File, Message: w.Message}) + } + for _, target := range res.Targets { manager := managerForUpdateTarget(target) if manager == "" || (len(selected) > 0 && !selected[manager]) { continue } - if target.Kind == imageupdate.TargetChart { - if err := sourceIndex.Resolve(&target); err != nil { - warnings = append(warnings, Warning{Manager: ManagerHelm, Project: target.File, Message: err.Error()}) - } - } node := nodeForImageTarget(target, manager) switch manager { case ManagerImage: diff --git a/deps/update.go b/deps/update.go index 04dd058..de17a78 100644 --- a/deps/update.go +++ b/deps/update.go @@ -9,12 +9,7 @@ import ( "sort" "strings" - "github.com/Masterminds/semver/v3" - "github.com/flanksource/clicky" - "github.com/flanksource/clicky/api" - "github.com/flanksource/clicky/task" - "github.com/flanksource/commons/collections" - flanksourceContext "github.com/flanksource/commons/context" + "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" "golang.org/x/mod/modfile" ) @@ -39,8 +34,23 @@ type ImageVersionResolver interface { } type UpdateOptions struct { - Managers []Manager - Expression []string + Managers []Manager + Expression []string + + // Resource filters for image/helm targets (ignored by package managers). + Kind []string + Namespace []string + Name []string + Selector []string + Image []string + Chart []string + + // Latest resolves each matched dependency to its highest stable version; + // Version applies an explicit version to all matched dependencies. They are + // mutually exclusive and both bypass the interactive pickers. + Latest bool + Version string + Check bool DryRun bool Runner CommandRunner @@ -49,6 +59,15 @@ type UpdateOptions struct { SelectVersion VersionSelector } +// DiscoverFilter narrows image/helm targets during discovery by Kubernetes +// resource metadata and image/chart name patterns. It has no effect on package +// managers, whose candidates come from manifest/lockfile parsing. +type DiscoverFilter struct { + Matcher repomap.ResourceMatcher + ImagePatterns []string + ChartPatterns []string +} + type UpdateCandidate struct { Manager Manager `json:"manager"` Name string `json:"name"` @@ -86,38 +105,63 @@ func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, if path == "" { path = "." } + if opts.Latest && opts.Version != "" { + return nil, fmt.Errorf("--latest and --version are mutually exclusive") + } managers, err := updateManagers(opts.Managers) if err != nil { return nil, err } patterns := splitUpdatePatterns(opts.Expression) - if len(patterns) == 0 { - return nil, fmt.Errorf("dependency update expression is required") - } if opts.Runner == nil { opts.Runner = ExecRunner{} } - candidates, err := DiscoverUpdateCandidates(path, managers) + filter := DiscoverFilter{ + Matcher: repomap.NewResourceMatcher(opts.Kind, opts.Namespace, opts.Name, opts.Selector), + ImagePatterns: splitUpdatePatterns(opts.Image), + ChartPatterns: splitUpdatePatterns(opts.Chart), + } + candidates, err := DiscoverUpdateCandidates(path, managers, filter) if err != nil { return nil, err } candidates = filterUpdateCandidates(candidates, patterns) if len(candidates) == 0 { - return nil, fmt.Errorf("no direct dependencies matched %q", strings.Join(patterns, ",")) + if len(patterns) > 0 { + return nil, fmt.Errorf("no direct dependencies matched %q", strings.Join(patterns, ",")) + } + return nil, fmt.Errorf("no updatable dependencies found") } - choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) - if len(choices) == 0 { - return orderedUpdatePlans(candidates, plansByKey), nil - } + // --check lists available updates without writing; it takes precedence over + // the write modes so `--check --version`/`--check --latest` never mutate files. if opts.Check { + choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) for _, choice := range choices { plansByKey[choice.Candidate.key()] = checkUpdatePlan(choice) } return orderedUpdatePlans(candidates, plansByKey), nil } + // --version applies an explicit version to every matched dependency without + // listing available versions (it only validates image/helm availability). + if opts.Version != "" { + return applyExplicitVersionUpdates(ctx, candidates, opts), nil + } + + choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) + + // --latest resolves each candidate to its highest stable version. + if opts.Latest { + applyLatestUpdates(ctx, candidates, choices, plansByKey, opts) + return orderedUpdatePlans(candidates, plansByKey), nil + } + + if len(choices) == 0 { + return orderedUpdatePlans(candidates, plansByKey), nil + } + selectCandidates := opts.SelectCandidates if selectCandidates == nil { selectCandidates = promptUpdateCandidates @@ -154,7 +198,7 @@ func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, return orderedUpdatePlans(candidates, plansByKey), nil } -func DiscoverUpdateCandidates(path string, managers []Manager) ([]UpdateCandidate, error) { +func DiscoverUpdateCandidates(path string, managers []Manager, filter DiscoverFilter) ([]UpdateCandidate, error) { defaultManagers := len(managers) == 0 managers, err := updateManagers(managers) if err != nil { @@ -190,7 +234,7 @@ func DiscoverUpdateCandidates(path string, managers []Manager) ([]UpdateCandidat } } if imageManagers := imageUpdateManagers(managers); len(imageManagers) > 0 { - candidates, err := discoverImageUpdateCandidates(absPath, imageManagers) + candidates, err := discoverImageUpdateCandidates(absPath, imageManagers, filter) if err != nil { if !defaultManagers || len(out) == 0 { if packageErr != nil && defaultManagers { @@ -326,664 +370,6 @@ func filterUpdateCandidates(candidates []UpdateCandidate, patterns []string) []U return out } -func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) ([]UpdateChoice, map[string]UpdatePlan) { - type result struct { - versions []string - latestStable string - latestPrerelease string - err error - } - results := make([]result, len(candidates)) - group := task.StartGroup[int]("Resolving dependency versions", task.WithConcurrency(updateResolveConcurrency)) - for i, candidate := range candidates { - idx, c := i, candidate - group.Add(updateTaskName(c), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { - tk.Infof("looking up published versions") - versions, latestStable, latestPrerelease, err := resolveCandidateVersions(ctx, opts, c) - results[idx] = result{versions: versions, latestStable: latestStable, latestPrerelease: latestPrerelease, err: err} - if err != nil { - tk.Warnf("%s", err.Error()) - tk.Warning() - } else if len(versions) == 0 { - tk.Infof("already up to date") - tk.Success() - } else { - tk.Success() - } - return idx, nil - }) - } - _, _ = group.GetResults() - - plansByKey := map[string]UpdatePlan{} - choices := make([]UpdateChoice, 0, len(candidates)) - for i, candidate := range candidates { - result := results[i] - if result.err != nil { - plansByKey[candidate.key()] = skippedUpdatePlan(candidate, result.err.Error()) - continue - } - if len(result.versions) == 0 { - continue - } - choices = append(choices, UpdateChoice{ - Candidate: candidate, - Versions: result.versions, - LatestStable: result.latestStable, - LatestPrerelease: result.latestPrerelease, - }) - } - return choices, plansByKey -} - -func resolveCandidateVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, string, string, error) { - var ( - versions []string - err error - ) - switch candidate.Manager { - case ManagerImage, ManagerHelm: - versions, _, _, err = availableImageTargetVersions(ctx, opts.ImageResolver, candidate) - default: - versions, err = AvailableDependencyVersions(ctx, opts.Runner, candidate) - } - if err != nil { - return nil, "", "", err - } - versions = updateableVersions(candidate.Current, versions) - return versions, latestStableVersion(versions), latestPrereleaseVersion(versions), nil -} - -func AvailableDependencyVersions(ctx context.Context, runner CommandRunner, candidate UpdateCandidate) ([]string, error) { - if runner == nil { - runner = ExecRunner{} - } - var ( - result CommandResult - err error - ) - switch candidate.Manager { - case ManagerGo: - result, err = runner.Run(ctx, Command{ - Dir: candidate.Dir, - Name: "go", - Args: []string{"list", "-m", "-versions", "-json", candidate.Name}, - Env: []string{"GOFLAGS=-mod=readonly"}, - }) - case ManagerNPM: - result, err = runner.Run(ctx, Command{ - Dir: candidate.Dir, - Name: "npm", - Args: []string{"view", candidate.Name, "versions", "--json"}, - }) - case ManagerPNPM: - result, err = runner.Run(ctx, Command{ - Dir: candidate.Dir, - Name: "pnpm", - Args: []string{"view", candidate.Name, "versions", "--json"}, - }) - default: - return nil, fmt.Errorf("dependency version lookup does not support manager %q", candidate.Manager) - } - if err != nil { - if result.Stderr != "" { - return nil, fmt.Errorf("%s: %w", strings.TrimSpace(result.Stderr), err) - } - return nil, err - } - versions, err := parseAvailableVersions(candidate.Manager, result.Stdout) - if err != nil { - return nil, err - } - return sortDependencyVersions(versions), nil -} - -func parseAvailableVersions(manager Manager, stdout string) ([]string, error) { - stdout = strings.TrimSpace(stdout) - if stdout == "" { - return nil, nil - } - if manager == ManagerGo { - var payload struct { - Versions []string `json:"Versions"` - } - if err := json.Unmarshal([]byte(stdout), &payload); err != nil { - return nil, err - } - return payload.Versions, nil - } - var list []string - if err := json.Unmarshal([]byte(stdout), &list); err == nil { - return list, nil - } - var single string - if err := json.Unmarshal([]byte(stdout), &single); err == nil && single != "" { - return []string{single}, nil - } - return nil, fmt.Errorf("expected JSON version array") -} - -func applyDependencyUpdate(ctx context.Context, candidate UpdateCandidate, version string, opts UpdateOptions) UpdatePlan { - if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { - return applyImageTargetUpdate(ctx, candidate, version, opts) - } - plan := planFromCandidate(candidate) - plan.NewVersion = version - if selectedVersionIsCurrent(candidate.Current, version) { - plan.Skipped = "already at selected version" - return plan - } - cmd, err := updateCommand(candidate, version) - if err != nil { - plan.Skipped = err.Error() - return plan - } - plan.Command = append([]string{cmd.Name}, cmd.Args...) - if opts.DryRun { - plan.DryRun = true - return plan - } - runner := opts.Runner - if runner == nil { - runner = ExecRunner{} - } - if _, err := runner.Run(ctx, cmd); err != nil { - plan.Skipped = err.Error() - return plan - } - plan.Written = true - stageUpdatePlan(ctx, &plan, candidate, runner) - return plan -} - -func stageUpdatePlan(ctx context.Context, plan *UpdatePlan, candidate UpdateCandidate, runner CommandRunner) { - staged, err := stageUpdatedFiles(ctx, runner, candidate) - plan.Staged = staged - if err != nil { - plan.StageError = err.Error() - } -} - -func updateCommand(candidate UpdateCandidate, version string) (Command, error) { - target := candidate.Name + "@" + version - switch candidate.Manager { - case ManagerGo: - return Command{Dir: candidate.Dir, Name: "go", Args: []string{"get", target}}, nil - case ManagerNPM: - args := []string{"install", "--package-lock-only", "--ignore-scripts"} - if flag := packageSaveFlag(candidate.Scope); flag != "" { - args = append(args, flag) - } - args = append(args, target) - return Command{Dir: candidate.Dir, Name: "npm", Args: args}, nil - case ManagerPNPM: - args := []string{"add", "--lockfile-only", "--ignore-scripts"} - if flag := packageSaveFlag(candidate.Scope); flag != "" { - args = append(args, flag) - } - args = append(args, target) - return Command{Dir: candidate.Dir, Name: "pnpm", Args: args}, nil - default: - return Command{}, fmt.Errorf("package-manager updates do not support manager %q", candidate.Manager) - } -} - -func packageSaveFlag(scope string) string { - switch scope { - case "dependencies": - return "--save-prod" - case "devDependencies": - return "--save-dev" - case "optionalDependencies": - return "--save-optional" - case "peerDependencies": - return "--save-peer" - default: - return "" - } -} - -func promptUpdateCandidates(choices []UpdateChoice) ([]UpdateChoice, bool) { - return runUpdateChoiceTreePicker(choices) -} - -func promptUpdateVersion(choice UpdateChoice) (string, bool) { - candidate := choice.Candidate - title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, candidate.File, candidate.Current) - return clicky.PromptSelect(choice.Versions, clicky.PromptSelectOptions[string]{ - Title: title, - PageSize: 12, - Render: func(version string) api.Textable { - text := clicky.Text(version, "font-mono") - var tags []string - if selectedVersionIsCurrent(candidate.Current, version) { - tags = append(tags, "current") - } - if version == choice.LatestStable { - tags = append(tags, "latest stable") - } - if version == choice.LatestPrerelease { - tags = append(tags, "latest pre-release") - } - if isPrerelease(version) { - tags = append(tags, "pre-release") - } - if len(tags) > 0 { - text = text.Space().Append("("+strings.Join(tags, ", ")+")", "text-muted") - } - return text - }, - }) -} - -type updateChoiceFileGroup struct { - File string - Choices []UpdateChoice -} - -func groupUpdateChoicesByFile(choices []UpdateChoice) []updateChoiceFileGroup { - byFile := map[string][]UpdateChoice{} - for _, choice := range choices { - file := choice.Candidate.File - byFile[file] = append(byFile[file], choice) - } - files := make([]string, 0, len(byFile)) - for file := range byFile { - files = append(files, file) - } - sort.Strings(files) - groups := make([]updateChoiceFileGroup, 0, len(files)) - for _, file := range files { - groupChoices := append([]UpdateChoice(nil), byFile[file]...) - sort.SliceStable(groupChoices, func(i, j int) bool { - return groupChoices[i].Candidate.less(groupChoices[j].Candidate) - }) - groups = append(groups, updateChoiceFileGroup{File: file, Choices: groupChoices}) - } - return groups -} - -func sortSelectedUpdateChoicesByFile(choices []UpdateChoice) []UpdateChoice { - var out []UpdateChoice - for _, group := range groupUpdateChoicesByFile(choices) { - out = append(out, group.Choices...) - } - return out -} - -func (p UpdatePlan) Pretty() api.Text { - t := clicky.Text(fmt.Sprintf("[%s] %s", p.Manager, p.Name), managerStyle(p.Manager)) - if p.OldVersion != "" || p.NewVersion != "" { - t = t.Space().Append(p.OldVersion, "font-mono text-muted") - if p.NewVersion != "" { - t = t.Append(" -> ", "text-muted").Append(p.NewVersion, "font-mono text-green-600") - } - } - switch { - case p.Skipped != "": - t = t.Space().Append("skipped: "+p.Skipped, "text-muted") - case p.Checked: - t = t.Space().Append("update available", "text-green-600") - case p.DryRun: - t = t.Space().Append("(dry-run)", "text-yellow-600") - case p.Written: - t = t.Space().Append("written", "text-green-600") - } - if len(p.Staged) > 0 { - t = t.Space().Append("staged "+strings.Join(p.Staged, ", "), "text-muted") - } - if p.StageError != "" { - t = t.Space().Append("stage failed: "+p.StageError, "text-yellow-600") - } - return t -} - -func (UpdatePlan) Columns() []api.ColumnDef { - return []api.ColumnDef{ - api.Column("manager").Label("Manager").Build(), - api.Column("dependency").Label("Dependency").Build(), - api.Column("file").Label("File").Build(), - api.Column("scope").Label("Scope").Build(), - api.Column("change").Label("Change").Build(), - api.Column("status").Label("Status").Build(), - } -} - -func (p UpdatePlan) Row() map[string]any { - row := map[string]any{ - "manager": clicky.Text(string(p.Manager), managerStyle(p.Manager)), - "dependency": clicky.Text(p.Name, "font-bold text-cyan-600"), - "file": clicky.Text(p.File, "font-mono"), - "scope": clicky.Text(p.Scope, "text-muted"), - "change": updateChangeText(p.OldVersion, p.NewVersion), - } - switch { - case p.Skipped != "": - row["status"] = clicky.Text(p.Skipped, "text-muted") - case p.Checked: - row["status"] = clicky.Text("update available", "text-green-600") - case p.DryRun: - row["status"] = clicky.Text("dry-run", "text-yellow-600") - case p.Written: - status := clicky.Text("written", "text-green-600") - if len(p.Staged) > 0 { - status = status.Append(" + staged "+strings.Join(p.Staged, ", "), "text-muted") - } - if p.StageError != "" { - status = status.Append(" (stage failed: "+p.StageError+")", "text-yellow-600") - } - row["status"] = status - default: - row["status"] = clicky.Text("") - } - return row -} - -func updateChangeText(oldVersion, newVersion string) api.Text { - text := clicky.Text(oldVersion, "font-mono text-muted") - if newVersion != "" { - text = text.Append(" -> ", "text-muted").Append(newVersion, "font-mono text-green-600") - } - return text -} - -func planFromCandidate(candidate UpdateCandidate) UpdatePlan { - return UpdatePlan{ - Manager: candidate.Manager, - Name: candidate.Name, - File: candidate.File, - Scope: candidate.Scope, - OldVersion: candidate.Current, - } -} - -func skippedUpdatePlan(candidate UpdateCandidate, reason string) UpdatePlan { - plan := planFromCandidate(candidate) - plan.Skipped = reason - return plan -} - -func checkUpdatePlan(choice UpdateChoice) UpdatePlan { - plan := planFromCandidate(choice.Candidate) - plan.NewVersion = checkUpdateVersion(choice) - plan.Checked = true - return plan -} - -func checkUpdateVersion(choice UpdateChoice) string { - if choice.LatestStable != "" { - return choice.LatestStable - } - if choice.LatestPrerelease != "" { - return choice.LatestPrerelease - } - if len(choice.Versions) > 0 { - return choice.Versions[0] - } - return "" -} - -func orderedUpdatePlans(candidates []UpdateCandidate, plansByKey map[string]UpdatePlan) []UpdatePlan { - plans := make([]UpdatePlan, 0, len(plansByKey)) - for _, candidate := range candidates { - if plan, ok := plansByKey[candidate.key()]; ok { - plans = append(plans, plan) - } - } - return plans -} - -func updateManagers(managers []Manager) ([]Manager, error) { - if len(managers) == 0 { - return []Manager{ManagerGo, ManagerNPM, ManagerPNPM, ManagerImage, ManagerHelm}, nil - } - out := make([]Manager, 0, len(managers)) - var unsupported []string - for _, manager := range managers { - if !supportedUpdateManagers[manager] { - unsupported = append(unsupported, string(manager)) - continue - } - out = append(out, manager) - } - if len(unsupported) > 0 { - sort.Strings(unsupported) - return nil, fmt.Errorf("dependency updates currently support go, npm, pnpm, image, and helm; unsupported manager(s): %s", strings.Join(unsupported, ", ")) - } - return out, nil -} - -func packageUpdateManagers(managers []Manager) []Manager { - var out []Manager - for _, manager := range managers { - switch manager { - case ManagerGo, ManagerNPM, ManagerPNPM: - out = append(out, manager) - } - } - return out -} - -func imageUpdateManagers(managers []Manager) []Manager { - var out []Manager - for _, manager := range managers { - switch manager { - case ManagerImage, ManagerHelm: - out = append(out, manager) - } - } - return out -} - -func splitUpdatePatterns(values []string) []string { - var out []string - for _, value := range values { - for _, part := range strings.Split(value, ",") { - part = strings.TrimSpace(part) - if part != "" { - out = append(out, part) - } - } - } - return out -} - -func (c UpdateCandidate) matches(patterns []string) bool { - identityPatterns, pathPatterns := splitExplicitPathPatterns(patterns) - hasPositivePattern := identityPatterns.hasPositive || pathPatterns.hasPositive - matchedPositive := false - - for _, value := range c.matchValues() { - if value == "" { - continue - } - ok, negated := collections.MatchItem(value, identityPatterns.values...) - if negated { - return false - } - if ok && identityPatterns.hasPositive { - matchedPositive = true - } - } - if len(pathPatterns.values) > 0 { - ok, negated := collections.MatchItem(c.File, pathPatterns.values...) - if negated { - return false - } - if ok && pathPatterns.hasPositive { - matchedPositive = true - } - } - if hasPositivePattern { - return matchedPositive - } - return true -} - -func (c UpdateCandidate) matchValues() []string { - return []string{ - c.Name, - string(c.Manager), - fmt.Sprintf("%s:%s", c.Manager, c.Name), - fmt.Sprintf("%s:%s@%s", c.Manager, c.Name, c.Current), - c.Scope, - c.Current, - } -} - -type updatePatternSet struct { - values []string - hasPositive bool -} - -func splitExplicitPathPatterns(patterns []string) (identity updatePatternSet, path updatePatternSet) { - for _, pattern := range patterns { - pattern = strings.TrimSpace(pattern) - if pattern == "" { - continue - } - negated := strings.HasPrefix(pattern, "!") - body := strings.TrimPrefix(pattern, "!") - field, value, explicit := strings.Cut(body, ":") - if explicit && (field == "path" || field == "file") { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if strings.HasPrefix(value, "!") { - negated = true - value = strings.TrimPrefix(value, "!") - } - if negated { - value = "!" + value - } else { - path.hasPositive = true - } - path.values = append(path.values, value) - continue - } - if !negated { - identity.hasPositive = true - } - identity.values = append(identity.values, pattern) - } - return identity, path -} - -func (c UpdateCandidate) key() string { - return strings.Join([]string{string(c.Manager), c.Dir, c.File, c.Scope, c.Name}, "\x00") -} - -func (c UpdateCandidate) less(other UpdateCandidate) bool { - if c.Manager != other.Manager { - return c.Manager < other.Manager - } - if c.File != other.File { - return c.File < other.File - } - if c.Scope != other.Scope { - return c.Scope < other.Scope - } - return c.Name < other.Name -} - -func updateTaskName(candidate UpdateCandidate) string { - return fmt.Sprintf("%s %s", candidate.Manager, candidate.Name) -} - -func isLocalUpdateSpec(spec string) bool { - spec = strings.TrimSpace(spec) - if spec == "" { - return true - } - if strings.HasPrefix(spec, "workspace:") { - return true - } - return isLocalRef(spec) -} - -func sortDependencyVersions(versions []string) []string { - type parsed struct { - orig string - ver *semver.Version - } - seen := map[string]bool{} - var parsedVersions []parsed - for _, version := range versions { - version = strings.TrimSpace(version) - if version == "" || seen[version] { - continue - } - seen[version] = true - sv, err := semver.NewVersion(version) - if err != nil { - continue - } - parsedVersions = append(parsedVersions, parsed{orig: version, ver: sv}) - } - sort.Slice(parsedVersions, func(i, j int) bool { - return parsedVersions[i].ver.GreaterThan(parsedVersions[j].ver) - }) - out := make([]string, len(parsedVersions)) - for i, item := range parsedVersions { - out[i] = item.orig - } - return out -} - -func latestStableVersion(versions []string) string { - for _, version := range versions { - if !isPrerelease(version) { - return version - } - } - return "" -} - -func latestPrereleaseVersion(versions []string) string { - for _, version := range versions { - if isPrerelease(version) { - return version - } - } - return "" -} - -func isPrerelease(version string) bool { - sv, err := semver.NewVersion(version) - return err == nil && strings.TrimSpace(sv.Prerelease()) != "" -} - -func updateableVersions(current string, versions []string) []string { - current = normalizeCurrentVersion(current) - currentSemver, currentErr := semver.NewVersion(current) - out := make([]string, 0, len(versions)) - for _, version := range versions { - if version == "" || version == current { - continue - } - versionSemver, versionErr := semver.NewVersion(version) - if currentErr != nil || versionErr != nil { - out = append(out, version) - continue - } - if versionSemver.GreaterThan(currentSemver) { - out = append(out, version) - } - } - return out -} - -func selectedVersionIsCurrent(current, selected string) bool { - return normalizeCurrentVersion(current) == selected -} - -func normalizeCurrentVersion(current string) string { - current = strings.TrimSpace(current) - current = strings.TrimPrefix(current, "npm:") - current = strings.TrimLeft(current, "^~<>= ") - if idx := strings.IndexAny(current, " |,"); idx >= 0 { - current = current[:idx] - } - return strings.TrimSpace(current) -} +// version math, candidate matching, manager helpers, plan rendering, version +// resolution, apply, and prompts live in update_version.go, update_match.go, +// update_plan.go, update_resolve.go, update_apply.go, and update_prompt.go. diff --git a/deps/update_apply.go b/deps/update_apply.go new file mode 100644 index 0000000..ed674f3 --- /dev/null +++ b/deps/update_apply.go @@ -0,0 +1,86 @@ +package deps + +import ( + "context" + "fmt" +) + +func applyDependencyUpdate(ctx context.Context, candidate UpdateCandidate, version string, opts UpdateOptions) UpdatePlan { + if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { + return applyImageTargetUpdate(ctx, candidate, version, opts) + } + plan := planFromCandidate(candidate) + plan.NewVersion = version + if selectedVersionIsCurrent(candidate.Current, version) { + plan.Skipped = "already at selected version" + return plan + } + cmd, err := updateCommand(candidate, version) + if err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Command = append([]string{cmd.Name}, cmd.Args...) + if opts.DryRun { + plan.DryRun = true + return plan + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + } + if _, err := runner.Run(ctx, cmd); err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Written = true + stageUpdatePlan(ctx, &plan, candidate, runner) + return plan +} + +func stageUpdatePlan(ctx context.Context, plan *UpdatePlan, candidate UpdateCandidate, runner CommandRunner) { + staged, err := stageUpdatedFiles(ctx, runner, candidate) + plan.Staged = staged + if err != nil { + plan.StageError = err.Error() + } +} + +func updateCommand(candidate UpdateCandidate, version string) (Command, error) { + target := candidate.Name + "@" + version + switch candidate.Manager { + case ManagerGo: + return Command{Dir: candidate.Dir, Name: "go", Args: []string{"get", target}}, nil + case ManagerNPM: + args := []string{"install", "--package-lock-only", "--ignore-scripts"} + if flag := packageSaveFlag(candidate.Scope); flag != "" { + args = append(args, flag) + } + args = append(args, target) + return Command{Dir: candidate.Dir, Name: "npm", Args: args}, nil + case ManagerPNPM: + args := []string{"add", "--lockfile-only", "--ignore-scripts"} + if flag := packageSaveFlag(candidate.Scope); flag != "" { + args = append(args, flag) + } + args = append(args, target) + return Command{Dir: candidate.Dir, Name: "pnpm", Args: args}, nil + default: + return Command{}, fmt.Errorf("package-manager updates do not support manager %q", candidate.Manager) + } +} + +func packageSaveFlag(scope string) string { + switch scope { + case "dependencies": + return "--save-prod" + case "devDependencies": + return "--save-dev" + case "optionalDependencies": + return "--save-optional" + case "peerDependencies": + return "--save-peer" + default: + return "" + } +} diff --git a/deps/update_image.go b/deps/update_image.go index 4e8bf26..8f2d1a7 100644 --- a/deps/update_image.go +++ b/deps/update_image.go @@ -3,26 +3,24 @@ package deps import ( "context" "fmt" - "os/exec" "path/filepath" - "sort" "strings" "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" - "github.com/flanksource/repomap/kubernetes" ) -func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCandidate, error) { +func discoverImageUpdateCandidates(root string, managers []Manager, filter DiscoverFilter) ([]UpdateCandidate, error) { selected := managerSet(managers) conf, err := repomap.GetConf(root) if err != nil { return nil, err } - targets, sourceIndex, err := discoverImageTargets(conf, root) + res, err := imageupdate.DiscoverRepoTargets(conf, root) if err != nil { return nil, err } + targets := imageupdate.Filter(res.Targets, filter.Matcher, filter.ImagePatterns, filter.ChartPatterns) var out []UpdateCandidate for _, target := range targets { @@ -30,11 +28,6 @@ func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCan if manager == "" || !selected[manager] { continue } - if target.Kind == imageupdate.TargetChart { - if err := sourceIndex.Resolve(&target); err != nil { - return nil, err - } - } targetCopy := target out = append(out, UpdateCandidate{ Manager: manager, @@ -49,80 +42,6 @@ func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCan return out, nil } -func discoverImageTargets(conf *repomap.ArchConf, scanPath string) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, error) { - files, err := gitTrackedFiles(conf.RepoPath()) - if err != nil { - return nil, nil, fmt.Errorf("failed to list files: %w", err) - } - - var prefix string - if rel, err := filepath.Rel(conf.RepoPath(), scanPath); err == nil && rel != "." { - prefix = filepath.ToSlash(rel) + "/" - } - - contents := map[string]string{} - for _, file := range files { - if !kubernetes.IsYaml(file) { - continue - } - content, err := conf.ReadFileWithFallback(file, "") - if err != nil || content == "" { - continue - } - contents[file] = content - } - - tree := imageupdate.BuildKustomizeTree(contents) - sourceIndex := imageupdate.NewSourceIndex(tree) - - keys := make([]string, 0, len(contents)) - for file := range contents { - keys = append(keys, file) - } - sort.Strings(keys) - - var targets []imageupdate.UpdateTarget - for _, file := range keys { - content := contents[file] - _ = sourceIndex.IndexHelmRepositories(file, content) - if prefix != "" && !strings.HasPrefix(file, prefix) { - continue - } - fileTargets, err := imageupdate.ExtractTargets(file, content) - if err != nil { - continue - } - targets = append(targets, fileTargets...) - } - sort.SliceStable(targets, func(i, j int) bool { - if targets[i].File != targets[j].File { - return targets[i].File < targets[j].File - } - if targets[i].FieldLine != targets[j].FieldLine { - return targets[i].FieldLine < targets[j].FieldLine - } - return updateTargetName(targets[i]) < updateTargetName(targets[j]) - }) - return targets, sourceIndex, nil -} - -func gitTrackedFiles(repoPath string) ([]string, error) { - cmd := exec.Command("git", "-C", repoPath, "ls-files", "-z") - out, err := cmd.Output() - if err != nil { - return nil, err - } - var files []string - for _, file := range strings.Split(string(out), "\x00") { - file = strings.TrimSpace(file) - if file != "" { - files = append(files, filepath.ToSlash(file)) - } - } - sort.Strings(files) - return files, nil -} - func managerForUpdateTarget(target imageupdate.UpdateTarget) Manager { switch target.Kind { case imageupdate.TargetImage: @@ -142,7 +61,12 @@ func updateTargetName(target imageupdate.UpdateTarget) string { } return stripImageVersion(target.CurrentValue) case imageupdate.TargetChart: - return target.ChartName + if target.ChartName != "" { + return target.ChartName + } + // Unresolved chartRef: fall back to the referenced source name so the + // candidate still has an identity and its lookup fails loudly by name. + return target.ChartRefName default: return target.CurrentValue } diff --git a/deps/update_match.go b/deps/update_match.go new file mode 100644 index 0000000..0f9c7ae --- /dev/null +++ b/deps/update_match.go @@ -0,0 +1,178 @@ +package deps + +import ( + "fmt" + "sort" + "strings" + + "github.com/flanksource/commons/collections" +) + +func updateManagers(managers []Manager) ([]Manager, error) { + if len(managers) == 0 { + return []Manager{ManagerGo, ManagerNPM, ManagerPNPM, ManagerImage, ManagerHelm}, nil + } + out := make([]Manager, 0, len(managers)) + var unsupported []string + for _, manager := range managers { + if !supportedUpdateManagers[manager] { + unsupported = append(unsupported, string(manager)) + continue + } + out = append(out, manager) + } + if len(unsupported) > 0 { + sort.Strings(unsupported) + return nil, fmt.Errorf("dependency updates currently support go, npm, pnpm, image, and helm; unsupported manager(s): %s", strings.Join(unsupported, ", ")) + } + return out, nil +} + +func packageUpdateManagers(managers []Manager) []Manager { + var out []Manager + for _, manager := range managers { + switch manager { + case ManagerGo, ManagerNPM, ManagerPNPM: + out = append(out, manager) + } + } + return out +} + +func imageUpdateManagers(managers []Manager) []Manager { + var out []Manager + for _, manager := range managers { + switch manager { + case ManagerImage, ManagerHelm: + out = append(out, manager) + } + } + return out +} + +func splitUpdatePatterns(values []string) []string { + var out []string + for _, value := range values { + for _, part := range strings.Split(value, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + } + return out +} + +func (c UpdateCandidate) matches(patterns []string) bool { + identityPatterns, pathPatterns := splitExplicitPathPatterns(patterns) + hasPositivePattern := identityPatterns.hasPositive || pathPatterns.hasPositive + matchedPositive := false + + for _, value := range c.matchValues() { + if value == "" { + continue + } + ok, negated := collections.MatchItem(value, identityPatterns.values...) + if negated { + return false + } + if ok && identityPatterns.hasPositive { + matchedPositive = true + } + } + if len(pathPatterns.values) > 0 { + ok, negated := collections.MatchItem(c.File, pathPatterns.values...) + if negated { + return false + } + if ok && pathPatterns.hasPositive { + matchedPositive = true + } + } + if hasPositivePattern { + return matchedPositive + } + return true +} + +func (c UpdateCandidate) matchValues() []string { + return []string{ + c.Name, + string(c.Manager), + fmt.Sprintf("%s:%s", c.Manager, c.Name), + fmt.Sprintf("%s:%s@%s", c.Manager, c.Name, c.Current), + c.Scope, + c.Current, + } +} + +type updatePatternSet struct { + values []string + hasPositive bool +} + +func splitExplicitPathPatterns(patterns []string) (identity updatePatternSet, path updatePatternSet) { + for _, pattern := range patterns { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + negated := strings.HasPrefix(pattern, "!") + body := strings.TrimPrefix(pattern, "!") + field, value, explicit := strings.Cut(body, ":") + if explicit && (field == "path" || field == "file") { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if strings.HasPrefix(value, "!") { + negated = true + value = strings.TrimPrefix(value, "!") + } + if negated { + value = "!" + value + } else { + path.hasPositive = true + } + path.values = append(path.values, value) + continue + } + if !negated { + identity.hasPositive = true + } + identity.values = append(identity.values, pattern) + } + return identity, path +} + +func (c UpdateCandidate) key() string { + return strings.Join([]string{string(c.Manager), c.Dir, c.File, c.Scope, c.Name}, "\x00") +} + +func (c UpdateCandidate) less(other UpdateCandidate) bool { + if c.Manager != other.Manager { + return c.Manager < other.Manager + } + if c.File != other.File { + return c.File < other.File + } + if c.Scope != other.Scope { + return c.Scope < other.Scope + } + return c.Name < other.Name +} + +func updateTaskName(candidate UpdateCandidate) string { + return fmt.Sprintf("%s %s", candidate.Manager, candidate.Name) +} + +func isLocalUpdateSpec(spec string) bool { + spec = strings.TrimSpace(spec) + if spec == "" { + return true + } + if strings.HasPrefix(spec, "workspace:") { + return true + } + return isLocalRef(spec) +} diff --git a/deps/update_modes.go b/deps/update_modes.go new file mode 100644 index 0000000..8b3b437 --- /dev/null +++ b/deps/update_modes.go @@ -0,0 +1,68 @@ +package deps + +import ( + "context" + "fmt" +) + +// applyExplicitVersionUpdates applies opts.Version to every candidate. For +// image/helm targets it first validates the version is published; package-manager +// updates pass the version straight to the manager, which validates it. +func applyExplicitVersionUpdates(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) []UpdatePlan { + plansByKey := make(map[string]UpdatePlan, len(candidates)) + for _, candidate := range candidates { + plansByKey[candidate.key()] = applyExplicitVersion(ctx, candidate, opts) + } + return orderedUpdatePlans(candidates, plansByKey) +} + +func applyExplicitVersion(ctx context.Context, candidate UpdateCandidate, opts UpdateOptions) UpdatePlan { + if selectedVersionIsCurrent(candidate.Current, opts.Version) { + return skippedUpdatePlan(candidate, "already at selected version") + } + if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { + available, _, _, err := availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + if err != nil { + return skippedUpdatePlan(candidate, err.Error()) + } + if !containsString(available, opts.Version) { + return skippedUpdatePlan(candidate, fmt.Sprintf("version %q is not available", opts.Version)) + } + } + return applyDependencyUpdate(ctx, candidate, opts.Version, opts) +} + +// applyLatestUpdates fills plansByKey with the highest-stable update for each +// candidate that has one, recording "already up to date" for the rest. Candidates +// whose version lookup already failed (a skipped plan exists) are left untouched. +func applyLatestUpdates(ctx context.Context, candidates []UpdateCandidate, choices []UpdateChoice, plansByKey map[string]UpdatePlan, opts UpdateOptions) { + choiceByKey := make(map[string]UpdateChoice, len(choices)) + for _, choice := range choices { + choiceByKey[choice.Candidate.key()] = choice + } + for _, candidate := range candidates { + key := candidate.key() + if _, done := plansByKey[key]; done { + continue + } + choice, ok := choiceByKey[key] + if !ok { + plansByKey[key] = skippedUpdatePlan(candidate, "already up to date") + continue + } + if choice.LatestStable == "" { + plansByKey[key] = skippedUpdatePlan(candidate, "no stable version available") + continue + } + plansByKey[key] = applyDependencyUpdate(ctx, candidate, choice.LatestStable, opts) + } +} + +func containsString(values []string, target string) bool { + for _, v := range values { + if v == target { + return true + } + } + return false +} diff --git a/deps/update_modes_test.go b/deps/update_modes_test.go new file mode 100644 index 0000000..fca4ed1 --- /dev/null +++ b/deps/update_modes_test.go @@ -0,0 +1,195 @@ +package deps + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const chartRefOCIFixture = `apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: podinfo + namespace: apps +spec: + chartRef: + kind: OCIRepository + name: podinfo +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: podinfo + namespace: apps +spec: + url: oci://ghcr.io/stefanprodan/charts/podinfo + ref: + tag: 6.5.0 +` + +func setupImageRepo(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + for rel, content := range files { + writeFile(t, filepath.Join(dir, rel), content) + } + runGit(t, dir, "add", ".") + return dir +} + +func TestUpdate_LatestSelectsHighestStable(t *testing.T) { + setupImageRepo(t, map[string]string{"apps/workloads.yaml": deploymentUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Latest: true, + DryRun: true, + ImageResolver: fakeImageVersionResolver{ + "nginx": {"1.28.0-rc.1", "1.27.0", "1.25.3"}, + "ghcr.io/flanksource/proxy": {"v0.4.1"}, + }, + }) + if err != nil { + t.Fatal(err) + } + + nginx := findPlan(plans, "nginx") + if nginx == nil || nginx.NewVersion != "1.27.0" || !nginx.DryRun || nginx.Written { + t.Fatalf("nginx plan = %#v, want 1.27.0 dry-run (skipping the rc prerelease)", nginx) + } + proxy := findPlan(plans, "ghcr.io/flanksource/proxy") + if proxy == nil || proxy.Skipped != "already up to date" { + t.Fatalf("proxy plan = %#v, want skipped already up to date", proxy) + } +} + +func TestUpdate_ExplicitVersionWritesEdit(t *testing.T) { + dir := setupImageRepo(t, map[string]string{"apps/workloads.yaml": deploymentUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Image: []string{"nginx"}, + Version: "1.27.0", + ImageResolver: fakeImageVersionResolver{"nginx": {"1.27.0", "1.25.3"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 { + t.Fatalf("plans = %#v, want only the nginx candidate (image filter)", plans) + } + if !plans[0].Written || plans[0].NewVersion != "1.27.0" { + t.Fatalf("plan = %#v, want written 1.27.0", plans[0]) + } + got := readFileString(t, filepath.Join(dir, "apps", "workloads.yaml")) + if !strings.Contains(got, "image: nginx:1.27.0") { + t.Fatalf("manifest not updated:\n%s", got) + } + if !strings.Contains(got, "image: ghcr.io/flanksource/proxy:v0.4.1") { + t.Fatalf("image filter must leave the sidecar untouched:\n%s", got) + } +} + +func TestUpdate_ExplicitVersionRejectsUnavailable(t *testing.T) { + setupImageRepo(t, map[string]string{"apps/workloads.yaml": deploymentUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Image: []string{"nginx"}, + Version: "9.9.9", + ImageResolver: fakeImageVersionResolver{"nginx": {"1.27.0", "1.25.3"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 || plans[0].Written { + t.Fatalf("plans = %#v, want one unwritten plan", plans) + } + if !strings.Contains(plans[0].Skipped, "not available") { + t.Fatalf("skip = %q, want 'not available'", plans[0].Skipped) + } +} + +func TestUpdate_ResourceFilterKindNarrowsToHelm(t *testing.T) { + setupImageRepo(t, map[string]string{ + "apps/workloads.yaml": deploymentUpdateFixture, + "apps/helmrelease.yaml": helmReleaseUpdateFixture, + }) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage, ManagerHelm}, + Kind: []string{"HelmRelease"}, + Check: true, + ImageResolver: fakeImageVersionResolver{"podinfo": {"6.6.0", "6.5.0"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 || plans[0].Manager != ManagerHelm || plans[0].Name != "podinfo" { + t.Fatalf("plans = %#v, want only the HelmRelease chart", plans) + } + if plans[0].NewVersion != "6.6.0" || !plans[0].Checked { + t.Fatalf("plan = %#v, want checked 6.6.0", plans[0]) + } +} + +func TestUpdate_NoExpressionMatchesAll(t *testing.T) { + setupImageRepo(t, map[string]string{"apps/helmrelease.yaml": helmReleaseUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerHelm}, + Check: true, + ImageResolver: fakeImageVersionResolver{"podinfo": {"6.6.0", "6.5.0"}}, + }) + if err != nil { + t.Fatalf("empty expression should match all, got %v", err) + } + if len(plans) != 1 || plans[0].Name != "podinfo" { + t.Fatalf("plans = %#v, want podinfo", plans) + } +} + +func TestDiscoverUpdateCandidates_ChartRefOCI(t *testing.T) { + setupImageRepo(t, map[string]string{"clusters/app.yaml": chartRefOCIFixture}) + + got, err := DiscoverUpdateCandidates(".", []Manager{ManagerHelm}, DiscoverFilter{}) + if err != nil { + t.Fatal(err) + } + helm := findUpdateCandidate(got, ManagerHelm, "podinfo") + if helm == nil { + t.Fatalf("chartRef OCIRepository not discovered: %#v", got) + } + if helm.Current != "6.5.0" { + t.Errorf("current = %q, want 6.5.0 (from OCIRepository spec.ref.tag)", helm.Current) + } + if helm.Target == nil || helm.Target.RepoURL != "oci://ghcr.io/stefanprodan/charts" || !helm.Target.IsOCI { + t.Errorf("target repo not resolved: %#v", helm.Target) + } + // The edit anchor must be the OCIRepository file, not the HelmRelease. + if helm.Target.File != "clusters/app.yaml" || helm.Target.FieldJSONPath != "$.spec.ref.tag" { + t.Errorf("anchor = %s %s, want clusters/app.yaml $.spec.ref.tag", helm.Target.File, helm.Target.FieldJSONPath) + } +} + +func findPlan(plans []UpdatePlan, name string) *UpdatePlan { + for i := range plans { + if plans[i].Name == name { + return &plans[i] + } + } + return nil +} + +func readFileString(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} diff --git a/deps/update_plan.go b/deps/update_plan.go new file mode 100644 index 0000000..c78c4ce --- /dev/null +++ b/deps/update_plan.go @@ -0,0 +1,131 @@ +package deps + +import ( + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (p UpdatePlan) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("[%s] %s", p.Manager, p.Name), managerStyle(p.Manager)) + if p.OldVersion != "" || p.NewVersion != "" { + t = t.Space().Append(p.OldVersion, "font-mono text-muted") + if p.NewVersion != "" { + t = t.Append(" -> ", "text-muted").Append(p.NewVersion, "font-mono text-green-600") + } + } + switch { + case p.Skipped != "": + t = t.Space().Append("skipped: "+p.Skipped, "text-muted") + case p.Checked: + t = t.Space().Append("update available", "text-green-600") + case p.DryRun: + t = t.Space().Append("(dry-run)", "text-yellow-600") + case p.Written: + t = t.Space().Append("written", "text-green-600") + } + if len(p.Staged) > 0 { + t = t.Space().Append("staged "+strings.Join(p.Staged, ", "), "text-muted") + } + if p.StageError != "" { + t = t.Space().Append("stage failed: "+p.StageError, "text-yellow-600") + } + return t +} + +func (UpdatePlan) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("manager").Label("Manager").Build(), + api.Column("dependency").Label("Dependency").Build(), + api.Column("file").Label("File").Build(), + api.Column("scope").Label("Scope").Build(), + api.Column("change").Label("Change").Build(), + api.Column("status").Label("Status").Build(), + } +} + +func (p UpdatePlan) Row() map[string]any { + row := map[string]any{ + "manager": clicky.Text(string(p.Manager), managerStyle(p.Manager)), + "dependency": clicky.Text(p.Name, "font-bold text-cyan-600"), + "file": clicky.Text(p.File, "font-mono"), + "scope": clicky.Text(p.Scope, "text-muted"), + "change": updateChangeText(p.OldVersion, p.NewVersion), + } + switch { + case p.Skipped != "": + row["status"] = clicky.Text(p.Skipped, "text-muted") + case p.Checked: + row["status"] = clicky.Text("update available", "text-green-600") + case p.DryRun: + row["status"] = clicky.Text("dry-run", "text-yellow-600") + case p.Written: + status := clicky.Text("written", "text-green-600") + if len(p.Staged) > 0 { + status = status.Append(" + staged "+strings.Join(p.Staged, ", "), "text-muted") + } + if p.StageError != "" { + status = status.Append(" (stage failed: "+p.StageError+")", "text-yellow-600") + } + row["status"] = status + default: + row["status"] = clicky.Text("") + } + return row +} + +func updateChangeText(oldVersion, newVersion string) api.Text { + text := clicky.Text(oldVersion, "font-mono text-muted") + if newVersion != "" { + text = text.Append(" -> ", "text-muted").Append(newVersion, "font-mono text-green-600") + } + return text +} + +func planFromCandidate(candidate UpdateCandidate) UpdatePlan { + return UpdatePlan{ + Manager: candidate.Manager, + Name: candidate.Name, + File: candidate.File, + Scope: candidate.Scope, + OldVersion: candidate.Current, + } +} + +func skippedUpdatePlan(candidate UpdateCandidate, reason string) UpdatePlan { + plan := planFromCandidate(candidate) + plan.Skipped = reason + return plan +} + +func checkUpdatePlan(choice UpdateChoice) UpdatePlan { + plan := planFromCandidate(choice.Candidate) + plan.NewVersion = checkUpdateVersion(choice) + plan.Checked = true + return plan +} + +func checkUpdateVersion(choice UpdateChoice) string { + if choice.LatestStable != "" { + return choice.LatestStable + } + if choice.LatestPrerelease != "" { + return choice.LatestPrerelease + } + if len(choice.Versions) > 0 { + return choice.Versions[0] + } + return "" +} + +func orderedUpdatePlans(candidates []UpdateCandidate, plansByKey map[string]UpdatePlan) []UpdatePlan { + plans := make([]UpdatePlan, 0, len(plansByKey)) + for _, candidate := range candidates { + if plan, ok := plansByKey[candidate.key()]; ok { + plans = append(plans, plan) + } + } + return plans +} diff --git a/deps/update_prompt.go b/deps/update_prompt.go new file mode 100644 index 0000000..2740061 --- /dev/null +++ b/deps/update_prompt.go @@ -0,0 +1,78 @@ +package deps + +import ( + "fmt" + "sort" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func promptUpdateCandidates(choices []UpdateChoice) ([]UpdateChoice, bool) { + return runUpdateChoiceTreePicker(choices) +} + +func promptUpdateVersion(choice UpdateChoice) (string, bool) { + candidate := choice.Candidate + title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, candidate.File, candidate.Current) + return clicky.PromptSelect(choice.Versions, clicky.PromptSelectOptions[string]{ + Title: title, + PageSize: 12, + Render: func(version string) api.Textable { + text := clicky.Text(version, "font-mono") + var tags []string + if selectedVersionIsCurrent(candidate.Current, version) { + tags = append(tags, "current") + } + if version == choice.LatestStable { + tags = append(tags, "latest stable") + } + if version == choice.LatestPrerelease { + tags = append(tags, "latest pre-release") + } + if isPrerelease(version) { + tags = append(tags, "pre-release") + } + if len(tags) > 0 { + text = text.Space().Append("("+strings.Join(tags, ", ")+")", "text-muted") + } + return text + }, + }) +} + +type updateChoiceFileGroup struct { + File string + Choices []UpdateChoice +} + +func groupUpdateChoicesByFile(choices []UpdateChoice) []updateChoiceFileGroup { + byFile := map[string][]UpdateChoice{} + for _, choice := range choices { + file := choice.Candidate.File + byFile[file] = append(byFile[file], choice) + } + files := make([]string, 0, len(byFile)) + for file := range byFile { + files = append(files, file) + } + sort.Strings(files) + groups := make([]updateChoiceFileGroup, 0, len(files)) + for _, file := range files { + groupChoices := append([]UpdateChoice(nil), byFile[file]...) + sort.SliceStable(groupChoices, func(i, j int) bool { + return groupChoices[i].Candidate.less(groupChoices[j].Candidate) + }) + groups = append(groups, updateChoiceFileGroup{File: file, Choices: groupChoices}) + } + return groups +} + +func sortSelectedUpdateChoicesByFile(choices []UpdateChoice) []UpdateChoice { + var out []UpdateChoice + for _, group := range groupUpdateChoicesByFile(choices) { + out = append(out, group.Choices...) + } + return out +} diff --git a/deps/update_resolve.go b/deps/update_resolve.go new file mode 100644 index 0000000..caa8e55 --- /dev/null +++ b/deps/update_resolve.go @@ -0,0 +1,148 @@ +package deps + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/task" + flanksourceContext "github.com/flanksource/commons/context" +) + +func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) ([]UpdateChoice, map[string]UpdatePlan) { + type result struct { + versions []string + latestStable string + latestPrerelease string + err error + } + results := make([]result, len(candidates)) + group := task.StartGroup[int]("Resolving dependency versions", task.WithConcurrency(updateResolveConcurrency)) + for i, candidate := range candidates { + idx, c := i, candidate + group.Add(updateTaskName(c), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { + tk.Infof("looking up published versions") + versions, latestStable, latestPrerelease, err := resolveCandidateVersions(ctx, opts, c) + results[idx] = result{versions: versions, latestStable: latestStable, latestPrerelease: latestPrerelease, err: err} + if err != nil { + tk.Warnf("%s", err.Error()) + tk.Warning() + } else if len(versions) == 0 { + tk.Infof("already up to date") + tk.Success() + } else { + tk.Success() + } + return idx, nil + }) + } + _, _ = group.GetResults() + + plansByKey := map[string]UpdatePlan{} + choices := make([]UpdateChoice, 0, len(candidates)) + for i, candidate := range candidates { + result := results[i] + if result.err != nil { + plansByKey[candidate.key()] = skippedUpdatePlan(candidate, result.err.Error()) + continue + } + if len(result.versions) == 0 { + continue + } + choices = append(choices, UpdateChoice{ + Candidate: candidate, + Versions: result.versions, + LatestStable: result.latestStable, + LatestPrerelease: result.latestPrerelease, + }) + } + return choices, plansByKey +} + +func resolveCandidateVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, string, string, error) { + var ( + versions []string + err error + ) + switch candidate.Manager { + case ManagerImage, ManagerHelm: + versions, _, _, err = availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + default: + versions, err = AvailableDependencyVersions(ctx, opts.Runner, candidate) + } + if err != nil { + return nil, "", "", err + } + versions = updateableVersions(candidate.Current, versions) + return versions, latestStableVersion(versions), latestPrereleaseVersion(versions), nil +} + +func AvailableDependencyVersions(ctx context.Context, runner CommandRunner, candidate UpdateCandidate) ([]string, error) { + if runner == nil { + runner = ExecRunner{} + } + var ( + result CommandResult + err error + ) + switch candidate.Manager { + case ManagerGo: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "go", + Args: []string{"list", "-m", "-versions", "-json", candidate.Name}, + Env: []string{"GOFLAGS=-mod=readonly"}, + }) + case ManagerNPM: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "npm", + Args: []string{"view", candidate.Name, "versions", "--json"}, + }) + case ManagerPNPM: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "pnpm", + Args: []string{"view", candidate.Name, "versions", "--json"}, + }) + default: + return nil, fmt.Errorf("dependency version lookup does not support manager %q", candidate.Manager) + } + if err != nil { + if result.Stderr != "" { + return nil, fmt.Errorf("%s: %w", strings.TrimSpace(result.Stderr), err) + } + return nil, err + } + versions, err := parseAvailableVersions(candidate.Manager, result.Stdout) + if err != nil { + return nil, err + } + return sortDependencyVersions(versions), nil +} + +func parseAvailableVersions(manager Manager, stdout string) ([]string, error) { + stdout = strings.TrimSpace(stdout) + if stdout == "" { + return nil, nil + } + if manager == ManagerGo { + var payload struct { + Versions []string `json:"Versions"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + return nil, err + } + return payload.Versions, nil + } + var list []string + if err := json.Unmarshal([]byte(stdout), &list); err == nil { + return list, nil + } + var single string + if err := json.Unmarshal([]byte(stdout), &single); err == nil && single != "" { + return []string{single}, nil + } + return nil, fmt.Errorf("expected JSON version array") +} diff --git a/deps/update_test.go b/deps/update_test.go index ef85456..15ca9ec 100644 --- a/deps/update_test.go +++ b/deps/update_test.go @@ -38,7 +38,7 @@ replace github.com/acme/local => ../local }`) writeFile(t, filepath.Join(dir, "pnpm-app", "pnpm-lock.yaml"), `lockfileVersion: '9.0'`) - got, err := DiscoverUpdateCandidates(dir, nil) + got, err := DiscoverUpdateCandidates(dir, nil, DiscoverFilter{}) if err != nil { t.Fatal(err) } @@ -69,7 +69,7 @@ require github.com/acme/direct v1.2.3 }`) writeFile(t, filepath.Join(dir, "web", "package-lock.json"), `{"lockfileVersion": 3}`) - got, err := DiscoverUpdateCandidates(".", nil) + got, err := DiscoverUpdateCandidates(".", nil, DiscoverFilter{}) if err != nil { t.Fatal(err) } @@ -271,7 +271,7 @@ func TestDiscoverUpdateCandidates_ImageAndHelmTargets(t *testing.T) { writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) runGit(t, dir, "add", ".") - got, err := DiscoverUpdateCandidates(".", []Manager{ManagerImage, ManagerHelm}) + got, err := DiscoverUpdateCandidates(".", []Manager{ManagerImage, ManagerHelm}, DiscoverFilter{}) if err != nil { t.Fatal(err) } diff --git a/deps/update_version.go b/deps/update_version.go new file mode 100644 index 0000000..d675906 --- /dev/null +++ b/deps/update_version.go @@ -0,0 +1,94 @@ +package deps + +import ( + "sort" + "strings" + + "github.com/Masterminds/semver/v3" +) + +func sortDependencyVersions(versions []string) []string { + type parsed struct { + orig string + ver *semver.Version + } + seen := map[string]bool{} + var parsedVersions []parsed + for _, version := range versions { + version = strings.TrimSpace(version) + if version == "" || seen[version] { + continue + } + seen[version] = true + sv, err := semver.NewVersion(version) + if err != nil { + continue + } + parsedVersions = append(parsedVersions, parsed{orig: version, ver: sv}) + } + sort.Slice(parsedVersions, func(i, j int) bool { + return parsedVersions[i].ver.GreaterThan(parsedVersions[j].ver) + }) + out := make([]string, len(parsedVersions)) + for i, item := range parsedVersions { + out[i] = item.orig + } + return out +} + +func latestStableVersion(versions []string) string { + for _, version := range versions { + if !isPrerelease(version) { + return version + } + } + return "" +} + +func latestPrereleaseVersion(versions []string) string { + for _, version := range versions { + if isPrerelease(version) { + return version + } + } + return "" +} + +func isPrerelease(version string) bool { + sv, err := semver.NewVersion(version) + return err == nil && strings.TrimSpace(sv.Prerelease()) != "" +} + +func updateableVersions(current string, versions []string) []string { + current = normalizeCurrentVersion(current) + currentSemver, currentErr := semver.NewVersion(current) + out := make([]string, 0, len(versions)) + for _, version := range versions { + if version == "" || version == current { + continue + } + versionSemver, versionErr := semver.NewVersion(version) + if currentErr != nil || versionErr != nil { + out = append(out, version) + continue + } + if versionSemver.GreaterThan(currentSemver) { + out = append(out, version) + } + } + return out +} + +func selectedVersionIsCurrent(current, selected string) bool { + return normalizeCurrentVersion(current) == selected +} + +func normalizeCurrentVersion(current string) string { + current = strings.TrimSpace(current) + current = strings.TrimPrefix(current, "npm:") + current = strings.TrimLeft(current, "^~<>= ") + if idx := strings.IndexAny(current, " |,"); idx >= 0 { + current = current[:idx] + } + return strings.TrimSpace(current) +} diff --git a/imageupdate/chartref.go b/imageupdate/chartref.go new file mode 100644 index 0000000..5e8a83e --- /dev/null +++ b/imageupdate/chartref.go @@ -0,0 +1,215 @@ +package imageupdate + +import ( + "fmt" + "strings" + + "github.com/goccy/go-yaml/ast" + + "github.com/flanksource/repomap/kubernetes" +) + +// chartSource is a Flux source object that pins a concrete chart version and can +// therefore be the edit target of a chartRef HelmRelease: an OCIRepository +// (version in spec.ref.tag) or a HelmChart (version in spec.version). Unlike a +// HelmRepository, which is a bare URL, it carries the file and line of the +// version literal so Resolve can redirect the HelmRelease's target onto it. +type chartSource struct { + kind string // OCIRepository | HelmChart + file string + versionLine int + versionPath string + version string + repoURL string + isOCI bool + chartName string + + // srcRef chains a HelmChart to its HelmRepository for the repo URL. + srcRefName string + srcRefNamespace string +} + +type chartSourceCandidate struct { + rawNamespace string + effNamespace string + src chartSource +} + +// extractChartRef builds a deferred chart target from a HelmRelease's +// spec.chartRef. Only OCIRepository/HelmChart references resolve to a queryable +// version source; GitRepository/Bucket references carry no chart version. +func extractChartRef(file string, ref kubernetes.KubernetesRef, chartRef map[string]interface{}) (UpdateTarget, bool) { + kind, _ := chartRef["kind"].(string) + name, _ := chartRef["name"].(string) + namespace, _ := chartRef["namespace"].(string) + if name == "" || (kind != "OCIRepository" && kind != "HelmChart") { + return UpdateTarget{}, false + } + return UpdateTarget{ + Ref: ref, + Kind: TargetChart, + File: file, + ChartRefKind: kind, + ChartRefName: name, + ChartRefNamespace: namespace, + }, true +} + +func (idx *SourceIndex) indexOCIRepository(file string, ref kubernetes.KubernetesRef, m map[string]interface{}, doc *ast.DocumentNode, offset int) { + spec, _ := m["spec"].(map[string]interface{}) + url, _ := spec["url"].(string) + if url == "" { + return + } + // The OCIRepository url already points at the chart artifact; split off the + // last path segment so RepoURL+ChartName recompose to it (matching the + // inline OCI HelmRepository convention the resolver expects). + repoURL, chartName := splitOCIChartURL(url) + src := chartSource{ + kind: "OCIRepository", + file: file, + versionPath: "$.spec.ref.tag", + repoURL: repoURL, + isOCI: true, + chartName: chartName, + } + if refSpec, ok := spec["ref"].(map[string]interface{}); ok { + src.version, _ = refSpec["tag"].(string) + } + src.versionLine = valueLine(doc, src.versionPath, offset) + idx.addChartSource(file, ref, src) +} + +func (idx *SourceIndex) indexHelmChart(file string, ref kubernetes.KubernetesRef, m map[string]interface{}, doc *ast.DocumentNode, offset int) { + spec, _ := m["spec"].(map[string]interface{}) + if spec == nil { + return + } + src := chartSource{ + kind: "HelmChart", + file: file, + versionPath: "$.spec.version", + } + src.chartName, _ = spec["chart"].(string) + src.version, _ = spec["version"].(string) + if sr, ok := spec["sourceRef"].(map[string]interface{}); ok { + src.srcRefName, _ = sr["name"].(string) + src.srcRefNamespace, _ = sr["namespace"].(string) + } + src.versionLine = valueLine(doc, src.versionPath, offset) + idx.addChartSource(file, ref, src) +} + +func (idx *SourceIndex) addChartSource(file string, ref kubernetes.KubernetesRef, src chartSource) { + effNS := idx.effectiveNamespace(file, ref.Namespace) + idx.chartByKey[chartKey(src.kind, ref.Namespace, ref.Name)] = src + idx.chartByKey[chartKey(src.kind, effNS, ref.Name)] = src + nameKey := src.kind + "|" + ref.Name + idx.chartByName[nameKey] = append(idx.chartByName[nameKey], chartSourceCandidate{ + rawNamespace: ref.Namespace, + effNamespace: effNS, + src: src, + }) +} + +// resolveChartRef redirects a chartRef target onto the OCIRepository/HelmChart it +// references: File/FieldLine/CurrentValue become that object's version literal and +// RepoURL/IsOCI are resolved (chaining a HelmChart through its HelmRepository). +func (idx *SourceIndex) resolveChartRef(t *UpdateTarget) error { + wantNS := t.ChartRefNamespace + if wantNS == "" && idx.kt != nil { + wantNS = idx.kt.EffectiveNamespace(t.File) + } + src, err := idx.lookupChartSource(t.ChartRefKind, t.ChartRefName, wantNS, t.Ref) + if err != nil { + return err + } + + switch src.kind { + case "OCIRepository": + t.RepoURL = src.repoURL + t.IsOCI = src.isOCI + case "HelmChart": + repo, err := idx.lookupHelmRepository(src.srcRefName, chartSourceNamespace(src, wantNS), t.Ref) + if err != nil { + return fmt.Errorf("HelmChart %s/%s: %w", src.srcRefNamespace, src.chartName, err) + } + t.RepoURL = repo.URL + t.IsOCI = repo.IsOCI + } + + t.File = src.file + t.FieldLine = src.versionLine + t.FieldJSONPath = src.versionPath + t.CurrentValue = src.version + t.ChartName = src.chartName + if t.CurrentValue == "" { + return fmt.Errorf("HelmRelease %s/%s references %s %q which has no editable version (uses semver/digest, not a tag)", + t.Ref.Namespace, t.Ref.Name, src.kind, t.ChartRefName) + } + return nil +} + +func chartSourceNamespace(src chartSource, fallback string) string { + if src.srcRefNamespace != "" { + return src.srcRefNamespace + } + return fallback +} + +func (idx *SourceIndex) lookupChartSource(kind, name, wantNS string, requester kubernetes.KubernetesRef) (chartSource, error) { + if src, ok := idx.chartByKey[chartKey(kind, wantNS, name)]; ok { + return src, nil + } + candidates := idx.chartByName[kind+"|"+name] + switch { + case len(candidates) == 0: + return chartSource{}, fmt.Errorf("HelmRelease %s/%s references %s %s which was not found in the scanned manifests", + requester.Namespace, requester.Name, kind, sourceKey(wantNS, name)) + case len(candidates) == 1: + return candidates[0].src, nil + default: + if c, ok := pickByNamespace(candidates, wantNS, + func(c chartSourceCandidate) string { return c.rawNamespace }, + func(c chartSourceCandidate) string { return c.effNamespace }); ok { + return c.src, nil + } + return chartSource{}, fmt.Errorf("HelmRelease %s/%s references %s %q which is ambiguous across namespaces %s", + requester.Namespace, requester.Name, kind, name, chartCandidateNamespaces(candidates)) + } +} + +func chartKey(kind, namespace, name string) string { + return kind + "|" + namespace + "/" + name +} + +// splitOCIChartURL splits an OCIRepository url (oci://host/path/chart) into the +// parent repo url and chart name, mirroring how an inline OCI HelmRepository +// (parent url) plus chart name recompose to the full artifact path. +func splitOCIChartURL(url string) (repoURL, chartName string) { + trimmed := strings.TrimSuffix(strings.TrimPrefix(url, "oci://"), "/") + i := strings.LastIndex(trimmed, "/") + if i < 0 { + return "oci://" + trimmed, "" + } + return "oci://" + trimmed[:i], trimmed[i+1:] +} + +func chartCandidateNamespaces(candidates []chartSourceCandidate) string { + seen := map[string]bool{} + var out []string + for _, c := range candidates { + ns := c.effNamespace + if ns == "" { + ns = c.rawNamespace + } + if ns == "" { + ns = "(none)" + } + if !seen[ns] { + seen[ns] = true + out = append(out, ns) + } + } + return strings.Join(out, ", ") +} diff --git a/imageupdate/chartref_test.go b/imageupdate/chartref_test.go new file mode 100644 index 0000000..ff82a99 --- /dev/null +++ b/imageupdate/chartref_test.go @@ -0,0 +1,108 @@ +package imageupdate + +import ( + "strings" + "testing" +) + +// resolveSingleTarget indexes sources and extracts the one chart target from a +// fixture, resolving it through the index. It is the chartRef equivalent of the +// inline helpers in sourceref_test.go. +func resolveSingleTarget(t *testing.T, name string) UpdateTarget { + t.Helper() + content := readManifest(t, name) + idx := NewSourceIndex(nil) + if err := idx.IndexSources(name, content); err != nil { + t.Fatalf("index: %v", err) + } + targets, err := ExtractTargets(name, content) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(targets) != 1 { + t.Fatalf("want 1 target, got %d: %#v", len(targets), targets) + } + tg := targets[0] + if err := idx.Resolve(&tg); err != nil { + t.Fatalf("resolve: %v", err) + } + return tg +} + +func TestExtractTargets_ChartRefDeferred(t *testing.T) { + content := readManifest(t, "helmrelease-chartref-oci.yaml") + targets, err := ExtractTargets("helmrelease-chartref-oci.yaml", content) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 { + t.Fatalf("want 1 chart target (OCIRepository is a source, not a target), got %d", len(targets)) + } + tg := targets[0] + // Before Resolve the target is deferred: it names the chartRef but has no + // version or edit anchor yet. + if tg.ChartRefKind != "OCIRepository" || tg.ChartRefName != "podinfo" { + t.Errorf("chartRef = %q/%q, want OCIRepository/podinfo", tg.ChartRefKind, tg.ChartRefName) + } + if tg.CurrentValue != "" || tg.FieldLine != 0 { + t.Errorf("deferred target should have no version/line yet: %+v", tg) + } + if tg.Ref.Kind != "HelmRelease" || tg.Ref.Name != "podinfo" { + t.Errorf("ref should stay the HelmRelease: %+v", tg.Ref) + } +} + +func TestResolveChartRef_OCIRepository(t *testing.T) { + tg := resolveSingleTarget(t, "helmrelease-chartref-oci.yaml") + if !tg.IsOCI || tg.RepoURL != "oci://ghcr.io/stefanprodan/charts" { + t.Errorf("repo = %q oci=%v, want oci://ghcr.io/stefanprodan/charts true", tg.RepoURL, tg.IsOCI) + } + if tg.ChartName != "podinfo" { + t.Errorf("chart = %q, want podinfo", tg.ChartName) + } + if tg.CurrentValue != "6.5.0" { + t.Errorf("current = %q, want 6.5.0", tg.CurrentValue) + } + // The edit anchor must move onto the OCIRepository's spec.ref.tag (line 19). + if tg.FieldJSONPath != "$.spec.ref.tag" || tg.FieldLine != 19 { + t.Errorf("anchor = %s:%d, want $.spec.ref.tag:19", tg.FieldJSONPath, tg.FieldLine) + } + if tg.File != "helmrelease-chartref-oci.yaml" { + t.Errorf("file = %q, want the OCIRepository file", tg.File) + } +} + +func TestResolveChartRef_HelmChart(t *testing.T) { + tg := resolveSingleTarget(t, "helmrelease-chartref-helmchart.yaml") + if tg.IsOCI || tg.RepoURL != "https://charts.example.com/app" { + t.Errorf("repo = %q oci=%v, want https://charts.example.com/app false", tg.RepoURL, tg.IsOCI) + } + if tg.ChartName != "app" { + t.Errorf("chart = %q, want app", tg.ChartName) + } + if tg.CurrentValue != "1.2.0" { + t.Errorf("current = %q, want 1.2.0", tg.CurrentValue) + } + // Anchor moves onto the HelmChart's spec.version (line 18). + if tg.FieldJSONPath != "$.spec.version" || tg.FieldLine != 18 { + t.Errorf("anchor = %s:%d, want $.spec.version:18", tg.FieldJSONPath, tg.FieldLine) + } +} + +func TestResolveChartRef_MissingSourceFailsLoud(t *testing.T) { + idx := NewSourceIndex(nil) + tg := UpdateTarget{Kind: TargetChart, ChartRefKind: "OCIRepository", ChartRefName: "ghost"} + tg.Ref.Namespace = "apps" + tg.Ref.Name = "podinfo" + err := idx.Resolve(&tg) + if err == nil || !strings.Contains(err.Error(), "OCIRepository") { + t.Fatalf("expected loud error naming OCIRepository, got %v", err) + } +} + +func TestSplitOCIChartURL(t *testing.T) { + repo, chart := splitOCIChartURL("oci://ghcr.io/stefanprodan/charts/podinfo") + if repo != "oci://ghcr.io/stefanprodan/charts" || chart != "podinfo" { + t.Fatalf("split = %q/%q", repo, chart) + } +} diff --git a/imageupdate/discover.go b/imageupdate/discover.go new file mode 100644 index 0000000..9e76fe6 --- /dev/null +++ b/imageupdate/discover.go @@ -0,0 +1,140 @@ +package imageupdate + +import ( + "path/filepath" + "sort" + "strings" + + "github.com/flanksource/commons/collections" + + "github.com/flanksource/repomap" +) + +// TargetWarning is a non-fatal problem encountered while discovering a target — +// typically a chart whose Flux source (HelmRepository/OCIRepository/HelmChart) +// could not be resolved. Discovery keeps the target so it still surfaces. +type TargetWarning struct { + File string + Message string +} + +// DiscoverResult bundles the discovered targets, the source index used to resolve +// them, and any per-target resolution warnings. +type DiscoverResult struct { + Targets []UpdateTarget + Index *SourceIndex + Warnings []TargetWarning +} + +// DiscoverTargets runs the full image/chart discovery over a set of repo-relative +// YAML file contents: it builds the kustomize/Flux tree, indexes every chart +// source, extracts image/chart targets under scanPrefix, and resolves each chart +// target onto its source (URL + current version + edit anchor). Source-resolution +// failures become warnings (the target is kept), so a single bad HelmRelease +// never aborts discovery. scanPrefix is a repo-relative POSIX directory prefix +// ("" scans everything); sources are indexed repo-wide regardless of prefix so a +// HelmRelease under the prefix can resolve a source defined elsewhere. +func DiscoverTargets(contents map[string]string, scanPrefix string) DiscoverResult { + tree := BuildKustomizeTree(contents) + idx := NewSourceIndex(tree) + + files := make([]string, 0, len(contents)) + for f := range contents { + files = append(files, f) + } + sort.Strings(files) + + for _, f := range files { + _ = idx.IndexSources(f, contents[f]) + } + + var targets []UpdateTarget + var warnings []TargetWarning + for _, f := range files { + if scanPrefix != "" && !strings.HasPrefix(f, scanPrefix) { + continue + } + fileTargets, err := ExtractTargets(f, contents[f]) + if err != nil { + continue + } + for i := range fileTargets { + t := fileTargets[i] + if t.Kind == TargetChart { + if err := idx.Resolve(&t); err != nil { + warnings = append(warnings, TargetWarning{File: f, Message: err.Error()}) + } + } + targets = append(targets, t) + } + } + sortTargets(targets) + return DiscoverResult{Targets: targets, Index: idx, Warnings: warnings} +} + +// DiscoverRepoTargets is the convenience entry point used by the CLI: it reads +// every tracked YAML file under conf and discovers targets, scoping extraction to +// scanPath (which must live under the repo). +func DiscoverRepoTargets(conf *repomap.ArchConf, scanPath string) (DiscoverResult, error) { + contents, err := conf.TrackedYAMLContents() + if err != nil { + return DiscoverResult{}, err + } + return DiscoverTargets(contents, scanPrefix(conf.RepoPath(), scanPath)), nil +} + +// scanPrefix returns the repo-relative POSIX directory prefix for scanPath, or "" +// when scanPath is the repo root. +func scanPrefix(repoPath, scanPath string) string { + rel, err := filepath.Rel(repoPath, scanPath) + if err != nil || rel == "." || rel == "" { + return "" + } + return filepath.ToSlash(rel) + "/" +} + +func sortTargets(targets []UpdateTarget) { + sort.SliceStable(targets, func(i, j int) bool { + if targets[i].File != targets[j].File { + return targets[i].File < targets[j].File + } + if targets[i].FieldLine != targets[j].FieldLine { + return targets[i].FieldLine < targets[j].FieldLine + } + return targetName(targets[i]) < targetName(targets[j]) + }) +} + +func targetName(t UpdateTarget) string { + if t.Kind == TargetImage && t.Image != nil { + return t.Image.GetFullNameWithoutTag() + } + return t.ChartName +} + +// Filter applies a resource matcher plus image/chart name patterns to a target +// set. Empty matcher and empty patterns return the targets unchanged. +func Filter(targets []UpdateTarget, matcher repomap.ResourceMatcher, imagePatterns, chartPatterns []string) []UpdateTarget { + var out []UpdateTarget + for _, t := range targets { + if !matcher.MatchesRef(t.Ref) { + continue + } + if t.Kind == TargetImage && len(imagePatterns) > 0 { + name := "" + if t.Image != nil { + name = t.Image.GetFullNameWithoutTag() + } + if matched, _ := collections.MatchItem(name, imagePatterns...); !matched { + continue + } + } + if t.Kind == TargetChart && len(chartPatterns) > 0 { + if matched, _ := collections.MatchItem(t.ChartName, chartPatterns...); !matched { + continue + } + } + out = append(out, t) + } + return out +} diff --git a/imageupdate/extract.go b/imageupdate/extract.go index 4a79af9..0fe9bb1 100644 --- a/imageupdate/extract.go +++ b/imageupdate/extract.go @@ -28,38 +28,56 @@ var appsV1Kinds = map[string]bool{ // comment-only document, which would hide trailing HelmReleases/workloads. The // per-document line offset is added back so FieldLine stays absolute in the file. func ExtractTargets(file, content string) ([]UpdateTarget, error) { - docs, err := kubernetes.ParseYAMLDocuments(content) + var targets []UpdateTarget + err := forEachResourceDoc(content, func(d resourceDoc) { + switch { + case appsV1Kinds[d.ref.Kind]: + targets = append(targets, extractImages(file, d.ast, d.ref, d.m, d.offset)...) + case d.ref.Kind == "HelmRelease": + if t, ok := extractChart(file, d.ast, d.ref, d.m, d.offset); ok { + targets = append(targets, t) + } + } + }) if err != nil { return nil, fmt.Errorf("parse %s: %w", file, err) } + return targets, nil +} - var targets []UpdateTarget +// resourceDoc is a single Kubernetes resource document with both its decoded map +// content and its single-document AST, plus the line offset that converts AST +// line positions back to absolute file lines. +type resourceDoc struct { + ref kubernetes.KubernetesRef + m map[string]interface{} + ast *ast.DocumentNode + offset int +} + +// forEachResourceDoc splits content into documents (line-based, so trailing docs +// after a comment-only document survive), and yields each Kubernetes resource +// with its decoded map and single-doc AST. Documents whose AST cannot be parsed +// are skipped. Shared by target extraction and source indexing so both agree on +// document boundaries and line numbers. +func forEachResourceDoc(content string, fn func(resourceDoc)) error { + docs, err := kubernetes.ParseYAMLDocuments(content) + if err != nil { + return err + } for _, d := range docs { m := d.Content if !kubernetes.IsKubernetesResource(m) { continue } ref := kubernetes.ExtractKubernetesRef(kubernetes.YAMLDocument{StartLine: d.StartLine, Content: m}) - - // Parse just this document for accurate field line positions, then offset - // by the document's start line to get absolute file lines. - offset := d.StartLine - 1 astFile, err := parser.ParseBytes([]byte(docText(content, d)), 0) if err != nil || len(astFile.Docs) == 0 { continue } - doc := astFile.Docs[0] - - switch { - case appsV1Kinds[ref.Kind]: - targets = append(targets, extractImages(file, doc, ref, m, offset)...) - case ref.Kind == "HelmRelease": - if t, ok := extractChart(file, doc, ref, m, offset); ok { - targets = append(targets, t) - } - } + fn(resourceDoc{ref: ref, m: m, ast: astFile.Docs[0], offset: d.StartLine - 1}) } - return targets, nil + return nil } // docText returns the raw source lines of a single document (1-based inclusive @@ -145,6 +163,14 @@ func containerList(m map[string]interface{}) []interface{} { func extractChart(file string, doc *ast.DocumentNode, ref kubernetes.KubernetesRef, m map[string]interface{}, offset int) (UpdateTarget, bool) { spec, _ := m["spec"].(map[string]interface{}) + if spec == nil { + return UpdateTarget{}, false + } + // Flux v2 spec.chartRef points at an OCIRepository/HelmChart that holds the + // version literal; emit a deferred target that Resolve redirects onto it. + if chartRef, ok := spec["chartRef"].(map[string]interface{}); ok { + return extractChartRef(file, ref, chartRef) + } chart, _ := spec["chart"].(map[string]interface{}) chartSpec, _ := chart["spec"].(map[string]interface{}) if chartSpec == nil { diff --git a/imageupdate/sourceref.go b/imageupdate/sourceref.go index 0e95ef8..ececa96 100644 --- a/imageupdate/sourceref.go +++ b/imageupdate/sourceref.go @@ -27,20 +27,27 @@ type sourceCandidate struct { // SourceIndex resolves a HelmRelease sourceRef to its HelmRepository, accounting // for namespaces imposed by the Flux/kustomize tree rather than written into the // manifests. HelmRepositories are indexed under both their raw and effective -// namespace, with a name bucket for cross-namespace fallback. +// namespace, with a name bucket for cross-namespace fallback. OCIRepository and +// HelmChart objects (chartRef sources) are indexed the same way in chartBy*. type SourceIndex struct { byKey map[string]HelmRepositorySource byName map[string][]sourceCandidate - kt *KustomizeTree + + chartByKey map[string]chartSource + chartByName map[string][]chartSourceCandidate + + kt *KustomizeTree } // NewSourceIndex returns an empty index bound to a kustomize tree. The tree may // be nil, in which case effective namespaces equal raw namespaces. func NewSourceIndex(kt *KustomizeTree) *SourceIndex { return &SourceIndex{ - byKey: map[string]HelmRepositorySource{}, - byName: map[string][]sourceCandidate{}, - kt: kt, + byKey: map[string]HelmRepositorySource{}, + byName: map[string][]sourceCandidate{}, + chartByKey: map[string]chartSource{}, + chartByName: map[string][]chartSourceCandidate{}, + kt: kt, } } @@ -57,48 +64,54 @@ func (idx *SourceIndex) effectiveNamespace(file, rawNS string) string { return rawNS } -// IndexHelmRepositories parses one file and indexes every HelmRepository doc it -// contains under both its raw and effective (tree-derived) namespace. It uses -// the line-based document splitter rather than the YAML AST parser, which drops -// trailing documents in files that contain a comment-only document. -func (idx *SourceIndex) IndexHelmRepositories(file, content string) error { - docs, err := kubernetes.ParseYAMLDocuments(content) - if err != nil { - return err - } - for _, doc := range docs { - m := doc.Content - if kind, _ := m["kind"].(string); kind != "HelmRepository" { - continue - } - ref := kubernetes.ExtractKubernetesRef(kubernetes.YAMLDocument{Content: m}) - spec, _ := m["spec"].(map[string]interface{}) - url, _ := spec["url"].(string) - if url == "" { - continue +// IndexSources parses one file and indexes every Flux chart source it contains — +// HelmRepository (a bare URL) plus OCIRepository/HelmChart (chartRef sources that +// also pin a version) — under both their raw and effective (tree-derived) +// namespace, with a name bucket for cross-namespace fallback. +func (idx *SourceIndex) IndexSources(file, content string) error { + return forEachResourceDoc(content, func(d resourceDoc) { + switch d.ref.Kind { + case "HelmRepository": + idx.indexHelmRepository(file, d.ref, d.m) + case "OCIRepository": + idx.indexOCIRepository(file, d.ref, d.m, d.ast, d.offset) + case "HelmChart": + idx.indexHelmChart(file, d.ref, d.m, d.ast, d.offset) } - src := HelmRepositorySource{URL: url, IsOCI: strings.HasPrefix(url, "oci://")} - effNS := idx.effectiveNamespace(file, ref.Namespace) - - idx.byKey[sourceKey(ref.Namespace, ref.Name)] = src - idx.byKey[sourceKey(effNS, ref.Name)] = src - idx.byName[ref.Name] = append(idx.byName[ref.Name], sourceCandidate{ - rawNamespace: ref.Namespace, - effNamespace: effNS, - src: src, - }) + }) +} + +func (idx *SourceIndex) indexHelmRepository(file string, ref kubernetes.KubernetesRef, m map[string]interface{}) { + spec, _ := m["spec"].(map[string]interface{}) + url, _ := spec["url"].(string) + if url == "" { + return } - return nil + src := HelmRepositorySource{URL: url, IsOCI: strings.HasPrefix(url, "oci://")} + effNS := idx.effectiveNamespace(file, ref.Namespace) + + idx.byKey[sourceKey(ref.Namespace, ref.Name)] = src + idx.byKey[sourceKey(effNS, ref.Name)] = src + idx.byName[ref.Name] = append(idx.byName[ref.Name], sourceCandidate{ + rawNamespace: ref.Namespace, + effNamespace: effNS, + src: src, + }) } -// Resolve sets RepoURL and IsOCI on a chart target. It first tries an exact -// effective-namespace match, then falls back to matching by name in any -// namespace (a source often sits in the controller namespace, not the -// HelmRelease's). It fails loud only when the source is genuinely unresolvable. +// Resolve sets RepoURL/IsOCI (and, for chartRef targets, the edit anchor) on a +// chart target. For inline sourceRef targets it matches the HelmRepository by +// effective namespace then by name across namespaces; for chartRef targets it +// redirects onto the referenced OCIRepository/HelmChart. It is idempotent — an +// already-resolved target (RepoURL set) returns immediately — and fails loud only +// when the source is genuinely unresolvable. func (idx *SourceIndex) Resolve(t *UpdateTarget) error { - if t.Kind != TargetChart { + if t.Kind != TargetChart || t.RepoURL != "" { return nil } + if t.ChartRefName != "" { + return idx.resolveChartRef(t) + } if t.SourceRefName == "" { return fmt.Errorf("HelmRelease %s/%s has no sourceRef name", t.Ref.Namespace, t.Ref.Name) } @@ -107,49 +120,60 @@ func (idx *SourceIndex) Resolve(t *UpdateTarget) error { if wantNS == "" && idx.kt != nil { wantNS = idx.kt.EffectiveNamespace(t.File) } - - if src, ok := idx.byKey[sourceKey(wantNS, t.SourceRefName)]; ok { - return idx.apply(t, src) + src, err := idx.lookupHelmRepository(t.SourceRefName, wantNS, t.Ref) + if err != nil { + return err } + t.RepoURL = src.URL + t.IsOCI = src.IsOCI + return nil +} - candidates := idx.byName[t.SourceRefName] +// lookupHelmRepository finds a HelmRepository by effective namespace, then by +// name across namespaces (a source often sits in the controller namespace, not +// the consumer's). requester names the HelmRelease for error messages. +func (idx *SourceIndex) lookupHelmRepository(name, wantNS string, requester kubernetes.KubernetesRef) (HelmRepositorySource, error) { + if name == "" { + return HelmRepositorySource{}, fmt.Errorf("HelmRelease %s/%s has no HelmRepository sourceRef name", requester.Namespace, requester.Name) + } + if src, ok := idx.byKey[sourceKey(wantNS, name)]; ok { + return src, nil + } + candidates := idx.byName[name] switch { case len(candidates) == 0: - return fmt.Errorf("HelmRelease %s/%s references HelmRepository %s which was not found in the scanned manifests", - t.Ref.Namespace, t.Ref.Name, sourceKey(wantNS, t.SourceRefName)) + return HelmRepositorySource{}, fmt.Errorf("HelmRelease %s/%s references HelmRepository %s which was not found in the scanned manifests", + requester.Namespace, requester.Name, sourceKey(wantNS, name)) case len(candidates) == 1: - return idx.apply(t, candidates[0].src) + return candidates[0].src, nil default: - if c, ok := pickCandidate(candidates, wantNS); ok { - return idx.apply(t, c.src) + if c, ok := pickByNamespace(candidates, wantNS, + func(c sourceCandidate) string { return c.rawNamespace }, + func(c sourceCandidate) string { return c.effNamespace }); ok { + return c.src, nil } - return fmt.Errorf("HelmRelease %s/%s references HelmRepository %q which is ambiguous across namespaces %s", - t.Ref.Namespace, t.Ref.Name, t.SourceRefName, candidateNamespaces(candidates)) + return HelmRepositorySource{}, fmt.Errorf("HelmRelease %s/%s references HelmRepository %q which is ambiguous across namespaces %s", + requester.Namespace, requester.Name, name, candidateNamespaces(candidates)) } } -func (idx *SourceIndex) apply(t *UpdateTarget, src HelmRepositorySource) error { - t.RepoURL = src.URL - t.IsOCI = src.IsOCI - return nil -} - -// pickCandidate disambiguates multiple same-named sources: prefer an exact -// effective-namespace match, then a known controller namespace. -func pickCandidate(candidates []sourceCandidate, wantNS string) (sourceCandidate, bool) { +// pickByNamespace disambiguates multiple same-named candidates: prefer an exact +// raw/effective-namespace match, then a known controller namespace. +func pickByNamespace[C any](candidates []C, wantNS string, raw, eff func(C) string) (C, bool) { + var zero C for _, c := range candidates { - if c.effNamespace == wantNS || c.rawNamespace == wantNS { + if eff(c) == wantNS || raw(c) == wantNS { return c, true } } for _, ns := range controllerNamespaces { for _, c := range candidates { - if c.effNamespace == ns || c.rawNamespace == ns { + if eff(c) == ns || raw(c) == ns { return c, true } } } - return sourceCandidate{}, false + return zero, false } func candidateNamespaces(candidates []sourceCandidate) string { diff --git a/imageupdate/sourceref_test.go b/imageupdate/sourceref_test.go index 274c6f9..0140d92 100644 --- a/imageupdate/sourceref_test.go +++ b/imageupdate/sourceref_test.go @@ -8,7 +8,7 @@ import ( func TestSourceIndex_ResolveHTTP(t *testing.T) { content := readManifest(t, "helmrelease.yaml") idx := NewSourceIndex(nil) - if err := idx.IndexHelmRepositories("helmrelease.yaml", content); err != nil { + if err := idx.IndexSources("helmrelease.yaml", content); err != nil { t.Fatal(err) } targets, err := ExtractTargets("helmrelease.yaml", content) @@ -30,7 +30,7 @@ func TestSourceIndex_ResolveHTTP(t *testing.T) { func TestSourceIndex_ResolveOCI(t *testing.T) { content := readManifest(t, "helmrelease-oci.yaml") idx := NewSourceIndex(nil) - if err := idx.IndexHelmRepositories("helmrelease-oci.yaml", content); err != nil { + if err := idx.IndexSources("helmrelease-oci.yaml", content); err != nil { t.Fatal(err) } targets, _ := ExtractTargets("helmrelease-oci.yaml", content) @@ -73,7 +73,7 @@ func TestSourceIndex_TreeDerivedNamespace(t *testing.T) { kt := BuildKustomizeTree(files) idx := NewSourceIndex(kt) for f, content := range files { - if err := idx.IndexHelmRepositories(f, content); err != nil { + if err := idx.IndexSources(f, content); err != nil { t.Fatal(err) } } @@ -103,7 +103,7 @@ func TestSourceIndex_ControllerNamespaceFallback(t *testing.T) { // Two sources with the SAME name in different namespaces. repo := "apiVersion: source.toolkit.fluxcd.io/v1\nkind: HelmRepository\n" + "metadata:\n name: shared\n namespace: flux-system\nspec:\n url: oci://ghcr.io/acme/shared\n" - if err := idx.IndexHelmRepositories("sources.yaml", repo); err != nil { + if err := idx.IndexSources("sources.yaml", repo); err != nil { t.Fatal(err) } tg := UpdateTarget{Kind: TargetChart, SourceRefName: "shared", SourceRefNamespace: "my-app"} @@ -124,7 +124,7 @@ func TestSourceIndex_ControllerNamespaceFallback(t *testing.T) { func TestSourceIndex_CommentOnlyDocDoesNotDropTrailing(t *testing.T) { content := readManifest(t, "flux-multidoc.yaml") idx := NewSourceIndex(nil) - if err := idx.IndexHelmRepositories("flux.yaml", content); err != nil { + if err := idx.IndexSources("flux.yaml", content); err != nil { t.Fatal(err) } tg := UpdateTarget{Kind: TargetChart, SourceRefName: "flanksource-ecr", SourceRefNamespace: "flux-system"} @@ -144,8 +144,8 @@ func TestSourceIndex_AmbiguousNameFailsLoud(t *testing.T) { "metadata:\n name: dup\n namespace: team-a\nspec:\n url: https://a.example.com\n" b := "apiVersion: source.toolkit.fluxcd.io/v1\nkind: HelmRepository\n" + "metadata:\n name: dup\n namespace: team-b\nspec:\n url: https://b.example.com\n" - _ = idx.IndexHelmRepositories("a.yaml", a) - _ = idx.IndexHelmRepositories("b.yaml", b) + _ = idx.IndexSources("a.yaml", a) + _ = idx.IndexSources("b.yaml", b) tg := UpdateTarget{Kind: TargetChart, SourceRefName: "dup", SourceRefNamespace: "team-c"} tg.Ref.Namespace = "team-c" diff --git a/imageupdate/target.go b/imageupdate/target.go index 96b0968..be7aa37 100644 --- a/imageupdate/target.go +++ b/imageupdate/target.go @@ -37,4 +37,13 @@ type UpdateTarget struct { SourceRefNamespace string `json:"source_ref_namespace,omitempty"` RepoURL string `json:"repo_url,omitempty"` IsOCI bool `json:"oci,omitempty"` + + // ChartRef* are set when a HelmRelease selects its chart via spec.chartRef + // (Flux v2) instead of the inline spec.chart.spec template. They name the + // referenced OCIRepository/HelmChart source; SourceIndex.Resolve redirects + // File/FieldLine/CurrentValue/RepoURL onto that object (where the version + // literal actually lives) so the edit lands in the right place. + ChartRefKind string `json:"chart_ref_kind,omitempty"` + ChartRefName string `json:"chart_ref_name,omitempty"` + ChartRefNamespace string `json:"chart_ref_namespace,omitempty"` } diff --git a/imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml b/imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml new file mode 100644 index 0000000..db178dc --- /dev/null +++ b/imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml @@ -0,0 +1,29 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: app + namespace: apps +spec: + chartRef: + kind: HelmChart + name: app-chart +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmChart +metadata: + name: app-chart + namespace: apps +spec: + chart: app + version: 1.2.0 + sourceRef: + kind: HelmRepository + name: app-repo +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: app-repo + namespace: apps +spec: + url: https://charts.example.com/app diff --git a/imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml b/imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml new file mode 100644 index 0000000..8fbfd2a --- /dev/null +++ b/imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml @@ -0,0 +1,19 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: podinfo + namespace: apps +spec: + chartRef: + kind: OCIRepository + name: podinfo +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: podinfo + namespace: apps +spec: + url: oci://ghcr.io/stefanprodan/charts/podinfo + ref: + tag: 6.5.0 diff --git a/tracked_yaml.go b/tracked_yaml.go new file mode 100644 index 0000000..40f1baf --- /dev/null +++ b/tracked_yaml.go @@ -0,0 +1,33 @@ +package repomap + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/flanksource/repomap/kubernetes" +) + +// TrackedYAMLContents returns every git-tracked YAML file in the repo keyed by +// its repo-relative POSIX path. Empty or unreadable files are skipped. It is the +// shared first pass for image/chart discovery (imageupdate.DiscoverTargets), +// used by both `repomap deps update` and `repomap images list`. +func (conf *ArchConf) TrackedYAMLContents() (map[string]string, error) { + result, err := conf.Exec()("ls-files") + if err != nil { + return nil, fmt.Errorf("git ls-files failed: %w", err) + } + contents := map[string]string{} + for _, line := range strings.Split(result.Stdout, "\n") { + file := filepath.ToSlash(strings.TrimSpace(line)) + if file == "" || !kubernetes.IsYaml(file) { + continue + } + content, err := conf.ReadFileWithFallback(file, "") + if err != nil || content == "" { + continue + } + contents[file] = content + } + return contents, nil +} From f547842afe582378912ab6d9f292b861395bd3c8 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Wed, 22 Jul 2026 07:59:37 +0300 Subject: [PATCH 05/13] fix(imageupdate): preserve discovery context and source errors Apply effective Kustomize namespaces to discovered targets, distinguish exact-file scans from directory scans, and report untracked YAML files. Preserve chart source-resolution errors so callers can surface actionable diagnostics. --- imageupdate/discover.go | 81 +++++++++++++++++++++++++++++------- imageupdate/discover_test.go | 66 +++++++++++++++++++++++++++++ imageupdate/target.go | 6 +++ 3 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 imageupdate/discover_test.go diff --git a/imageupdate/discover.go b/imageupdate/discover.go index 9e76fe6..0438da1 100644 --- a/imageupdate/discover.go +++ b/imageupdate/discover.go @@ -1,6 +1,7 @@ package imageupdate import ( + "os" "path/filepath" "sort" "strings" @@ -8,6 +9,7 @@ import ( "github.com/flanksource/commons/collections" "github.com/flanksource/repomap" + "github.com/flanksource/repomap/kubernetes" ) // TargetWarning is a non-fatal problem encountered while discovering a target — @@ -19,22 +21,26 @@ type TargetWarning struct { } // DiscoverResult bundles the discovered targets, the source index used to resolve -// them, and any per-target resolution warnings. +// them, and any per-target resolution warnings. UntrackedTarget names a single +// scanned file that is a YAML manifest but is not git-tracked (so discovery could +// not see it); it is empty when the scan target is a directory or a tracked file. type DiscoverResult struct { - Targets []UpdateTarget - Index *SourceIndex - Warnings []TargetWarning + Targets []UpdateTarget + Index *SourceIndex + Warnings []TargetWarning + UntrackedTarget string } // DiscoverTargets runs the full image/chart discovery over a set of repo-relative // YAML file contents: it builds the kustomize/Flux tree, indexes every chart -// source, extracts image/chart targets under scanPrefix, and resolves each chart +// source, extracts image/chart targets under scope, and resolves each chart // target onto its source (URL + current version + edit anchor). Source-resolution // failures become warnings (the target is kept), so a single bad HelmRelease -// never aborts discovery. scanPrefix is a repo-relative POSIX directory prefix -// ("" scans everything); sources are indexed repo-wide regardless of prefix so a -// HelmRelease under the prefix can resolve a source defined elsewhere. -func DiscoverTargets(contents map[string]string, scanPrefix string) DiscoverResult { +// never aborts discovery. scope is a repo-relative POSIX match scope ("" scans +// everything, a "/"-terminated value is a directory prefix, anything else is an +// exact file path); sources are indexed repo-wide regardless of scope so a +// HelmRelease under the scope can resolve a source defined elsewhere. +func DiscoverTargets(contents map[string]string, scope string) DiscoverResult { tree := BuildKustomizeTree(contents) idx := NewSourceIndex(tree) @@ -51,7 +57,7 @@ func DiscoverTargets(contents map[string]string, scanPrefix string) DiscoverResu var targets []UpdateTarget var warnings []TargetWarning for _, f := range files { - if scanPrefix != "" && !strings.HasPrefix(f, scanPrefix) { + if !inScanScope(f, scope) { continue } fileTargets, err := ExtractTargets(f, contents[f]) @@ -60,8 +66,12 @@ func DiscoverTargets(contents map[string]string, scanPrefix string) DiscoverResu } for i := range fileTargets { t := fileTargets[i] + if effNS := tree.EffectiveNamespace(t.File); effNS != "" { + t.Ref.Namespace = effNS + } if t.Kind == TargetChart { if err := idx.Resolve(&t); err != nil { + t.SourceErr = err.Error() warnings = append(warnings, TargetWarning{File: f, Message: err.Error()}) } } @@ -74,23 +84,62 @@ func DiscoverTargets(contents map[string]string, scanPrefix string) DiscoverResu // DiscoverRepoTargets is the convenience entry point used by the CLI: it reads // every tracked YAML file under conf and discovers targets, scoping extraction to -// scanPath (which must live under the repo). +// scanPath (which must live under the repo). When scanPath is a single YAML file +// that is not git-tracked, the result's UntrackedTarget is set so the caller can +// report the reason instead of silently finding nothing. func DiscoverRepoTargets(conf *repomap.ArchConf, scanPath string) (DiscoverResult, error) { contents, err := conf.TrackedYAMLContents() if err != nil { return DiscoverResult{}, err } - return DiscoverTargets(contents, scanPrefix(conf.RepoPath(), scanPath)), nil + scope := scanScope(conf.RepoPath(), scanPath) + result := DiscoverTargets(contents, scope) + if untracked := untrackedFileTarget(scanPath, scope, contents); untracked != "" { + result.UntrackedTarget = untracked + } + return result, nil +} + +// untrackedFileTarget returns the repo-relative path of scanPath when it is a +// single YAML file (an exact-file scope) that is absent from the git-tracked +// contents, otherwise "". Directory scopes and tracked files return "". +func untrackedFileTarget(scanPath, scope string, contents map[string]string) string { + if scope == "" || strings.HasSuffix(scope, "/") || !kubernetes.IsYaml(scanPath) { + return "" + } + if _, tracked := contents[scope]; tracked { + return "" + } + return scope } -// scanPrefix returns the repo-relative POSIX directory prefix for scanPath, or "" -// when scanPath is the repo root. -func scanPrefix(repoPath, scanPath string) string { +// scanScope returns the repo-relative POSIX match scope for scanPath: "" for the +// repo root (match everything), a directory prefix ending in "/" for a directory, +// or the exact repo-relative file path for a single file. +func scanScope(repoPath, scanPath string) string { rel, err := filepath.Rel(repoPath, scanPath) if err != nil || rel == "." || rel == "" { return "" } - return filepath.ToSlash(rel) + "/" + rel = filepath.ToSlash(rel) + if info, err := os.Stat(scanPath); err == nil && !info.IsDir() { + return rel + } + return rel + "/" +} + +// inScanScope reports whether repo-relative file f falls under scope: an empty +// scope matches everything, a "/"-terminated scope is a directory prefix, and any +// other scope matches that exact file. +func inScanScope(f, scope string) bool { + switch { + case scope == "": + return true + case strings.HasSuffix(scope, "/"): + return strings.HasPrefix(f, scope) + default: + return f == scope + } } func sortTargets(targets []UpdateTarget) { diff --git a/imageupdate/discover_test.go b/imageupdate/discover_test.go new file mode 100644 index 0000000..8763cbe --- /dev/null +++ b/imageupdate/discover_test.go @@ -0,0 +1,66 @@ +package imageupdate + +import "testing" + +const deploymentNoNS = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: app +spec: + template: + spec: + containers: + - name: app + image: nginx:1.0.0 +` + +const deploymentPinnedNS = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: other +spec: + template: + spec: + containers: + - name: app + image: nginx:1.0.0 +` + +// imageTarget returns the single TargetImage in targets, failing if there is not +// exactly one. +func imageTarget(t *testing.T, targets []UpdateTarget) UpdateTarget { + t.Helper() + var found []UpdateTarget + for _, tg := range targets { + if tg.Kind == TargetImage { + found = append(found, tg) + } + } + if len(found) != 1 { + t.Fatalf("want exactly one image target, got %d: %+v", len(found), targets) + } + return found[0] +} + +func TestDiscoverTargets_KustomizationNamespaceApplied(t *testing.T) { + contents := map[string]string{ + "kenya/kustomization.yaml": "namespace: hf-qa-kenya\nresources:\n - app.yaml\n", + "kenya/app.yaml": deploymentNoNS, + } + res := DiscoverTargets(contents, "kenya/") + if ns := imageTarget(t, res.Targets).Ref.Namespace; ns != "hf-qa-kenya" { + t.Errorf("target namespace = %q, want hf-qa-kenya (imposed by kustomization)", ns) + } +} + +func TestDiscoverTargets_KustomizationNamespaceOverridesPinned(t *testing.T) { + contents := map[string]string{ + "kenya/kustomization.yaml": "namespace: hf-qa-kenya\nresources:\n - app.yaml\n", + "kenya/app.yaml": deploymentPinnedNS, + } + res := DiscoverTargets(contents, "kenya/") + if ns := imageTarget(t, res.Targets).Ref.Namespace; ns != "hf-qa-kenya" { + t.Errorf("target namespace = %q, want hf-qa-kenya (kustomization wins over metadata.namespace)", ns) + } +} diff --git a/imageupdate/target.go b/imageupdate/target.go index be7aa37..4e94d74 100644 --- a/imageupdate/target.go +++ b/imageupdate/target.go @@ -46,4 +46,10 @@ type UpdateTarget struct { ChartRefKind string `json:"chart_ref_kind,omitempty"` ChartRefName string `json:"chart_ref_name,omitempty"` ChartRefNamespace string `json:"chart_ref_namespace,omitempty"` + + // SourceErr records why a chart target's Flux source could not be resolved + // (for example, the referenced HelmRepository is absent from the scanned + // manifests). When set, RepoURL is empty and version resolution must surface + // this actionable message instead of querying an empty URL. + SourceErr string `json:"source_error,omitempty"` } From de03514711d4fcf1b5a607871b0e29db3aa10e42 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Thu, 6 Aug 2026 08:19:03 +0300 Subject: [PATCH 06/13] feat(cli): Add dependency metadata filters and implicit scan routing Expose Kubernetes metadata filters for image and Helm dependencies, while preserving positional paths and root help behavior. Restore implicit scan routing for bare paths, flags, and empty invocations. --- cmd/repomap/deps.go | 19 +++++++++++++++++-- cmd/repomap/deps_test.go | 21 +++++++++++++++++++++ cmd/repomap/images.go | 2 +- cmd/repomap/main.go | 30 ++++++++++++++++++++++-------- cmd/repomap/main_test.go | 31 +++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 cmd/repomap/main_test.go diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index 594e893..da30874 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -12,13 +12,17 @@ import ( ) type DepsOptions struct { - Path string `json:"path" args:"true" help:"Path to scan" default:"."` + Path string `json:"path" args:"true" help:"Path to scan (defaults to current directory)"` Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` + Kind []string `json:"kind,omitempty" flag:"kind,k" help:"Filter image/helm dependencies by Kubernetes kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` + Namespace []string `json:"namespace,omitempty" flag:"namespace,n" help:"Filter image/helm dependencies by namespace (MatchItem syntax)"` + Name []string `json:"name,omitempty" flag:"name" help:"Filter image/helm dependencies by resource name (MatchItem syntax)"` + Selector []string `json:"selector,omitempty" flag:"selector,l" help:"Filter image/helm dependencies by label selector, e.g. app=nginx"` Flat bool `json:"flat,omitempty" flag:"flat" help:"Export a flat node list with edges instead of the dependency tree"` IncludeIndirect bool `json:"include_indirect,omitempty" flag:"include-indirect" help:"Include Go indirect requirements in --depth 1 listings (ignored at other depths)"` - ShowDuplicates bool `json:"show_duplicates,omitempty" flag:"show-duplicates" help:"Render every occurrence of duplicated dependencies instead of collapsing them to the resolved node"` + ShowDuplicates bool `json:"show_duplicates,omitempty" flag:"show-duplicates" help:"Render every occurrence of duplicated dependencies and report duplicates/conflicts (default: collapse to a single resolved node with no duplicate reporting)"` } type DepsUpdateOptions struct { @@ -72,6 +76,12 @@ resolved node is tagged with the number of other parents and each parent that hid a duplicate shows a trailing count. Use --show-duplicates to render every occurrence instead. +Image and Helm dependencies discovered from Kubernetes manifests can be narrowed +by resource metadata with --kind/--namespace/--name/--selector (MatchItem +syntax). These filters only match manifest-sourced targets; Chart.yaml chart +directories have no Kubernetes metadata and are excluded whenever any of them is +set. + EXAMPLES: repomap deps repomap deps ./service --manager go @@ -80,6 +90,7 @@ EXAMPLES: repomap deps --manager go --include-indirect repomap deps --depth 0 --manager go --show-duplicates repomap deps --manager image,helm ./clusters/prod + repomap deps --manager helm -k HelmRelease -n default repomap deps --filter 'github.com/flanksource/*,!*test*'`) } @@ -138,6 +149,10 @@ func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { Managers: managers, MaxDepth: opts.Depth, Filters: splitCommaArgs(opts.Filter), + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, Flat: opts.Flat, IncludeIndirect: opts.IncludeIndirect, ShowDuplicates: opts.ShowDuplicates, diff --git a/cmd/repomap/deps_test.go b/cmd/repomap/deps_test.go index 65f7b8d..b8406bc 100644 --- a/cmd/repomap/deps_test.go +++ b/cmd/repomap/deps_test.go @@ -1,12 +1,33 @@ package main import ( + "context" "strings" "testing" + "github.com/flanksource/clicky" depgraph "github.com/flanksource/repomap/deps" + "github.com/spf13/cobra" ) +func TestDepsPositionalPathHonored(t *testing.T) { + var gotPath string + root := &cobra.Command{Use: "test"} + clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format") + clicky.AddNamedCommandWithContext("deps", root, DepsOptions{}, func(_ context.Context, opts DepsOptions) (*depgraph.Export, error) { + gotPath = opts.Path + return &depgraph.Export{}, nil + }) + + root.SetArgs([]string{"deps", "/tmp/some/scan/path", "--json"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + if gotPath != "/tmp/some/scan/path" { + t.Fatalf("positional path not bound: opts.Path = %q, want /tmp/some/scan/path", gotPath) + } +} + func TestParseManagers(t *testing.T) { got, err := parseManagers([]string{"go,npm", "pnpm", "image", "docker", "helm"}) if err != nil { diff --git a/cmd/repomap/images.go b/cmd/repomap/images.go index ec78f96..ae86b30 100644 --- a/cmd/repomap/images.go +++ b/cmd/repomap/images.go @@ -42,7 +42,7 @@ func init() { // `images update`. The first four mirror `scan`'s resource filters; --image and // --chart further narrow by image repo / chart name. type imageFilterOptions struct { - Path string `json:"path" args:"true" help:"Path to scan" default:"."` + Path string `json:"path" args:"true" help:"Path to scan (defaults to current directory)"` Kind []string `json:"kind" flag:"kind,k" help:"Filter by kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` Namespace []string `json:"namespace" flag:"namespace,n" help:"Filter by namespace (MatchItem syntax)"` Name []string `json:"name" flag:"name" help:"Filter by resource name (MatchItem syntax)"` diff --git a/cmd/repomap/main.go b/cmd/repomap/main.go index 5deaf0d..0940a18 100644 --- a/cmd/repomap/main.go +++ b/cmd/repomap/main.go @@ -49,17 +49,31 @@ func init() { func main() { defer shutdown.RecoverAndShutdown() - // Default to scan when no subcommand is given - if args := os.Args[1:]; len(args) == 0 || args[0] == "" || args[0][0] == '-' { - rootCmd.SetArgs(append([]string{"scan"}, args...)) - } else if args[0] != "help" && args[0] != "completion" { - if cmd, _, _ := rootCmd.Find(args); cmd == rootCmd { - rootCmd.SetArgs(append([]string{"scan"}, args...)) - } - } + rootCmd.SetArgs(defaultToScan(os.Args[1:])) if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } + +// defaultToScan rewrites CLI args so that `scan` is the implicit subcommand: a +// bare invocation, or one that leads with scan flags or a bare path, becomes +// `scan ...`. Help and completion are left untouched so `repomap --help` (and +// `-h`) shows the root command with its full subcommand list rather than scan's +// help. +func defaultToScan(args []string) []string { + if len(args) == 0 { + return []string{"scan"} + } + if args[0] == "-h" || args[0] == "--help" || args[0] == "help" || args[0] == "completion" { + return args + } + if args[0] == "" || args[0][0] == '-' { + return append([]string{"scan"}, args...) + } + if cmd, _, _ := rootCmd.Find(args); cmd == rootCmd { + return append([]string{"scan"}, args...) + } + return args +} diff --git a/cmd/repomap/main_test.go b/cmd/repomap/main_test.go new file mode 100644 index 0000000..57a3a08 --- /dev/null +++ b/cmd/repomap/main_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "reflect" + "testing" +) + +func TestDefaultToScan(t *testing.T) { + cases := []struct { + name string + in []string + want []string + }{ + {"bare invocation runs scan", nil, []string{"scan"}}, + {"leading scan flag prepends scan", []string{"-n", "ns"}, []string{"scan", "-n", "ns"}}, + {"bare path prepends scan", []string{"some/path"}, []string{"scan", "some/path"}}, + {"long help reaches root", []string{"--help"}, []string{"--help"}}, + {"short help reaches root", []string{"-h"}, []string{"-h"}}, + {"help command untouched", []string{"help"}, []string{"help"}}, + {"completion untouched", []string{"completion", "zsh"}, []string{"completion", "zsh"}}, + {"deps subcommand untouched", []string{"deps", "."}, []string{"deps", "."}}, + {"deps help untouched", []string{"deps", "-h"}, []string{"deps", "-h"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := defaultToScan(tc.in); !reflect.DeepEqual(got, tc.want) { + t.Errorf("defaultToScan(%v) = %v, want %v", tc.in, got, tc.want) + } + }) + } +} From eb85fa49da5a6bbbb943cb05c08b1b1f72e28b10 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Thu, 6 Aug 2026 08:19:19 +0300 Subject: [PATCH 07/13] feat(deps): add resource filtering and deduplicate version lookups Add Kubernetes resource filtering for image and Helm scans, with actionable errors for untracked manifest targets. Reuse published-version lookups across duplicate occurrences while preserving per-occurrence updates. Collapse package-manager duplicates silently by default and retain duplicate analysis only with --show-duplicates. BREAKING CHANGE: Remove Node.other_parents and Node.hidden_duplicates and their rendered markers; duplicate statistics and sections are no longer produced by default. --- deps/collapse.go | 26 +-------- deps/collapse_test.go | 27 ++++----- deps/go_graph_test.go | 20 ++++--- deps/model.go | 47 +++++++++------- deps/pretty.go | 14 +---- deps/pretty_tree.go | 19 +------ deps/pretty_tree_test.go | 22 +++++--- deps/scan.go | 30 +++++++--- deps/scan_image.go | 9 ++- deps/scan_test.go | 112 ++++++++++++++++++++++++++++++++++++++ deps/update_image.go | 6 ++ deps/update_match.go | 34 ++++++++++++ deps/update_modes_test.go | 63 +++++++++++++++++++++ deps/update_resolve.go | 109 ++++++++++++++++++++++--------------- deps/update_test.go | 102 ++++++++++++++++++++++++++++++++++ 15 files changed, 480 insertions(+), 160 deletions(-) diff --git a/deps/collapse.go b/deps/collapse.go index a5683ad..1fb8620 100644 --- a/deps/collapse.go +++ b/deps/collapse.go @@ -2,9 +2,8 @@ package deps // collapseDuplicates rewrites package-manager roots in place so each dependency // renders once at its resolved (shallowest) location instead of repeating under -// every parent. Image/Helm roots are left untouched: repeated images are real -// distinct deployments and the kubernetes display drops the root that would -// carry a collapse marker. +// every parent. Later occurrences are dropped silently with no marker. Image/Helm +// roots are left untouched: repeated images are real distinct deployments. func collapseDuplicates(roots []*Node) { for _, root := range roots { if root == nil || !isPackageManager(root.Manager) { @@ -23,11 +22,8 @@ func isPackageManager(manager Manager) bool { } // collapseRoot keeps the first BFS sighting (shallowest, then sorted) of each -// node ID within a single root, drops later sightings, and records the counts: -// the resolved node's OtherParents and each parent's HiddenDuplicates. +// node ID within a single root and silently drops later sightings. func collapseRoot(root *Node) { - refs := map[string]int{} - countRefs(root, refs) sortTree(root) seen := map[string]bool{} @@ -36,31 +32,15 @@ func collapseRoot(root *Node) { parent := queue[0] queue = queue[1:] kept := parent.Children[:0] - hidden := 0 for _, child := range parent.Children { if seen[child.ID] { - hidden++ continue } seen[child.ID] = true - if refs[child.ID] > 1 { - child.OtherParents = refs[child.ID] - 1 - } kept = append(kept, child) queue = append(queue, child) } parent.Children = kept - parent.HiddenDuplicates = hidden - } -} - -func countRefs(node *Node, refs map[string]int) { - if node == nil { - return - } - for _, child := range node.Children { - refs[child.ID]++ - countRefs(child, refs) } } diff --git a/deps/collapse_test.go b/deps/collapse_test.go index da97284..a9793ec 100644 --- a/deps/collapse_test.go +++ b/deps/collapse_test.go @@ -42,27 +42,22 @@ func TestCollapseKeepsSharedDependencyOnce(t *testing.T) { } } -func TestCollapseMarksOtherParentsAndHiddenCounts(t *testing.T) { +func TestCollapseDropsLaterSightingsSilently(t *testing.T) { root := diamondRoot() collapseDuplicates([]*Node{root}) a := findChild(root, "github.com/acme/a") - shared := findChild(a, "github.com/acme/shared") - if shared == nil { + if findChild(a, "github.com/acme/shared") == nil { t.Fatalf("shared should be retained under the first sorted parent 'a'") } - if shared.OtherParents != 2 { - t.Fatalf("resolved shared node should record 2 other parents, got %d", shared.OtherParents) - } - for _, name := range []string{"github.com/acme/b", "github.com/acme/c"} { parent := findChild(root, name) - if parent.HiddenDuplicates != 1 { - t.Fatalf("parent %s should hide 1 duplicate, got %d", name, parent.HiddenDuplicates) - } if findChild(parent, "github.com/acme/shared") != nil { t.Fatalf("parent %s should no longer carry the shared dependency", name) } + if len(parent.Children) != 0 { + t.Fatalf("parent %s should have no children after collapse, got %d", name, len(parent.Children)) + } } } @@ -79,9 +74,6 @@ func TestCollapseLeavesImageRootsUntouched(t *testing.T) { if len(root.Children) != 2 { t.Fatalf("image roots must keep every occurrence, got %d children", len(root.Children)) } - if root.HiddenDuplicates != 0 { - t.Fatalf("image root should not record hidden duplicates, got %d", root.HiddenDuplicates) - } } func TestCollapseIsolatesPerRoot(t *testing.T) { @@ -91,8 +83,13 @@ func TestCollapseIsolatesPerRoot(t *testing.T) { for _, root := range []*Node{first, second} { a := findChild(root, "github.com/acme/a") - if shared := findChild(a, "github.com/acme/shared"); shared == nil || shared.OtherParents != 2 { - t.Fatalf("each root should dedup independently with OtherParents=2") + if findChild(a, "github.com/acme/shared") == nil { + t.Fatalf("each root should retain shared under its first sorted parent") + } + for _, name := range []string{"github.com/acme/b", "github.com/acme/c"} { + if findChild(findChild(root, name), "github.com/acme/shared") != nil { + t.Fatalf("each root should dedup independently; %s should not keep shared", name) + } } } } diff --git a/deps/go_graph_test.go b/deps/go_graph_test.go index 8333318..4bf9948 100644 --- a/deps/go_graph_test.go +++ b/deps/go_graph_test.go @@ -147,7 +147,7 @@ require ( return got } -func TestScanCollapsesDuplicatesByDefault(t *testing.T) { +func TestScanCollapsesDuplicatesSilentlyByDefault(t *testing.T) { got := goDiamondScan(t, Options{}) root := got.Roots[0] lib := findChild(root, "github.com/acme/lib") @@ -158,15 +158,11 @@ func TestScanCollapsesDuplicatesByDefault(t *testing.T) { if (depUnderLib == nil) == (depUnderOther == nil) { t.Fatalf("dep should render under exactly one parent, lib=%v other=%v", depUnderLib != nil, depUnderOther != nil) } - resolved, hiddenParent := depUnderLib, other - if depUnderLib == nil { - resolved, hiddenParent = depUnderOther, lib + if got.Statistics.Duplicates != 0 { + t.Fatalf("default scan should not count duplicates, got %d", got.Statistics.Duplicates) } - if resolved.OtherParents != 1 { - t.Fatalf("resolved dep should record 1 other parent, got %d", resolved.OtherParents) - } - if hiddenParent.HiddenDuplicates != 1 { - t.Fatalf("the parent that lost dep should hide 1 duplicate, got %d", hiddenParent.HiddenDuplicates) + if len(got.Duplicates) != 0 { + t.Fatalf("default scan should not report a duplicates section, got %d", len(got.Duplicates)) } } @@ -181,6 +177,12 @@ func TestScanShowDuplicatesRendersEveryOccurrence(t *testing.T) { if !got.Metadata.ShowDuplicates { t.Fatalf("metadata should record show_duplicates") } + if got.Statistics.Duplicates == 0 { + t.Fatalf("--show-duplicates should count the shared dep as a duplicate") + } + if len(got.Duplicates) == 0 { + t.Fatalf("--show-duplicates should populate the duplicates section") + } } func TestScanFlatStaysFullGraph(t *testing.T) { diff --git a/deps/model.go b/deps/model.go index 0a30ace..b57ea48 100644 --- a/deps/model.go +++ b/deps/model.go @@ -28,8 +28,17 @@ type Options struct { Flat bool IncludeIndirect bool ShowDuplicates bool - Runner CommandRunner - Now func() time.Time + + // Resource filters for image/helm targets discovered from Kubernetes + // manifests (ignored by package managers and chart-directory scanning, which + // have no Kubernetes resource metadata to match against). + Kind []string + Namespace []string + Name []string + Selector []string + + Runner CommandRunner + Now func() time.Time // remote injects a pre-built remote resolver (cache + chart/image resolvers) // for tests; nil in production, where Scan builds a disk-backed one. remote *remoteDeps @@ -66,24 +75,22 @@ type Metadata struct { } type Node struct { - ID string `json:"id"` - Name string `json:"name"` - Version string `json:"version,omitempty"` - Manager Manager `json:"manager"` - Scope string `json:"scope,omitempty"` - Source string `json:"source,omitempty"` - Path string `json:"path,omitempty"` - Direct bool `json:"direct,omitempty"` - Dev bool `json:"dev,omitempty"` - Optional bool `json:"optional,omitempty"` - Local bool `json:"local,omitempty"` - Depth int `json:"depth"` - Circular bool `json:"circular,omitempty"` - Duplicate *DupRef `json:"duplicate,omitempty"` - OtherParents int `json:"other_parents,omitempty"` - HiddenDuplicates int `json:"hidden_duplicates,omitempty"` - Children []*Node `json:"children,omitempty"` - properties map[string]string + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Manager Manager `json:"manager"` + Scope string `json:"scope,omitempty"` + Source string `json:"source,omitempty"` + Path string `json:"path,omitempty"` + Direct bool `json:"direct,omitempty"` + Dev bool `json:"dev,omitempty"` + Optional bool `json:"optional,omitempty"` + Local bool `json:"local,omitempty"` + Depth int `json:"depth"` + Circular bool `json:"circular,omitempty"` + Duplicate *DupRef `json:"duplicate,omitempty"` + Children []*Node `json:"children,omitempty"` + properties map[string]string } type FlatNode struct { diff --git a/deps/pretty.go b/deps/pretty.go index d7fb604..ca0ab55 100644 --- a/deps/pretty.go +++ b/deps/pretty.go @@ -208,12 +208,7 @@ func statusTags(node *Node) []dependencyTag { if node.Circular { tags = append(tags, dependencyTag{label: "circular", style: "font-bold text-red-600"}) } - if node.OtherParents > 0 { - tags = append(tags, dependencyTag{label: parentMarker(node.OtherParents), style: "text-cyan-600"}) - if node.Duplicate != nil && node.Duplicate.Conflicts { - tags = append(tags, dependencyTag{label: "conflict", style: "font-bold text-red-600"}) - } - } else if node.Duplicate != nil { + if node.Duplicate != nil { tag := fmt.Sprintf("dup:%d", node.Duplicate.Count) style := "text-orange-500" if node.Duplicate.Conflicts { @@ -225,13 +220,6 @@ func statusTags(node *Node) []dependencyTag { return tags } -func parentMarker(other int) string { - if other == 1 { - return "+1 parent" - } - return fmt.Sprintf("+%d parents", other) -} - func sortTags(tags []dependencyTag) []dependencyTag { sort.Slice(tags, func(i, j int) bool { return tags[i].label < tags[j].label diff --git a/deps/pretty_tree.go b/deps/pretty_tree.go index ed00443..698d149 100644 --- a/deps/pretty_tree.go +++ b/deps/pretty_tree.go @@ -1,7 +1,6 @@ package deps import ( - "fmt" "path/filepath" "sort" "strings" @@ -64,32 +63,20 @@ func rootDisplayNode(root *Node, scanPath string) *displayNode { return &displayNode{text: label, children: packageChildNodes(root, showChildManager)} } -// packageChildNodes renders dependency children and, when duplicate occurrences -// were collapsed away, appends a trailing marker counting them. showManager -// keeps the manager prefix on children of roots that mix managers. +// packageChildNodes renders dependency children. showManager keeps the manager +// prefix on children of roots that mix managers. func packageChildNodes(parent *Node, showManager bool) []api.TreeNode { sorted := sortedNodes(parent.Children) - out := make([]api.TreeNode, 0, len(sorted)+1) + out := make([]api.TreeNode, 0, len(sorted)) for _, n := range sorted { out = append(out, &displayNode{ text: nodeText(n, showManager), children: packageChildNodes(n, showManager), }) } - if parent.HiddenDuplicates > 0 { - out = append(out, &displayNode{text: hiddenDuplicatesText(parent.HiddenDuplicates)}) - } return out } -func hiddenDuplicatesText(hidden int) api.Text { - noun := "dependencies" - if hidden == 1 { - noun = "dependency" - } - return clicky.Text(fmt.Sprintf("(and %d other %s)", hidden, noun), "text-muted italic") -} - // namespaceNodes is the top level of the kubernetes grouping. func namespaceNodes(leaves []*Node) []api.TreeNode { keys, groups := groupLeaves(sortedNodes(leaves), func(n *Node) string { diff --git a/deps/pretty_tree_test.go b/deps/pretty_tree_test.go index d968fed..1925d99 100644 --- a/deps/pretty_tree_test.go +++ b/deps/pretty_tree_test.go @@ -87,22 +87,26 @@ func TestTreeDropsDefaultScopeAndDirectTags(t *testing.T) { } } -func TestTreeShowsCollapseMarkers(t *testing.T) { +func TestTreeCollapsesSilently(t *testing.T) { root := diamondRoot() root.Path = "/abs/proj/go.mod" collapseDuplicates([]*Node{root}) out := (&Export{Metadata: Metadata{Path: "/abs/proj"}, Roots: []*Node{root}}).Pretty().String() - sharedLine := prettyLine(out, "github.com/acme/shared@v1.0.0") - if !strings.Contains(sharedLine, "+2 parents") { - t.Fatalf("resolved shared node should be tagged (+2 parents): %q", sharedLine) + var occurrences int + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "github.com/acme/shared@v1.0.0") { + occurrences++ + } } - - if prettyLine(out, "and 1 other dependency)") == "" { - t.Fatalf("a parent that hid one duplicate should show the singular marker:\n%s", out) + if occurrences != 1 { + t.Fatalf("collapsed tree should render shared exactly once, got %d:\n%s", occurrences, out) } - if strings.Contains(out, "and 1 other dependencies") { - t.Fatalf("singular hidden count must read 'dependency', not 'dependencies':\n%s", out) + if strings.Contains(out, "parent") { + t.Fatalf("collapsed tree must not carry parent-count markers:\n%s", out) + } + if strings.Contains(out, "other dependenc") { + t.Fatalf("collapsed tree must not carry hidden-duplicate markers:\n%s", out) } } diff --git a/deps/scan.go b/deps/scan.go index c3c61b1..496b34b 100644 --- a/deps/scan.go +++ b/deps/scan.go @@ -10,6 +10,8 @@ import ( "github.com/flanksource/clicky/task" flanksourceContext "github.com/flanksource/commons/context" + + "github.com/flanksource/repomap" ) func Scan(ctx context.Context, path string, opts Options) (*Export, error) { @@ -55,21 +57,28 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { roots = append(roots, projectRoots...) } + matcher := repomap.NewResourceMatcher(opts.Kind, opts.Namespace, opts.Name, opts.Selector) + var imageErr error if scanImages { // Chart-directory scanning is a filesystem walk and runs independently of // the git-backed k8s-manifest discovery, so a chart dir outside a git repo - // still resolves its own subcharts and images. - chartRoots, chartWarnings, chartErr := discoverChartDependencyRoots(absPath, opts.Managers) - warnings = append(warnings, chartWarnings...) - if chartErr != nil { - return nil, chartErr + // still resolves its own subcharts and images. Chart.yaml roots carry no + // Kubernetes resource metadata, so a resource filter (kind/namespace/name/ + // selector) can only be satisfied by manifest targets; skip chart dirs when + // one is active. + if matcher.IsEmpty() { + chartRoots, chartWarnings, chartErr := discoverChartDependencyRoots(absPath, opts.Managers) + warnings = append(warnings, chartWarnings...) + if chartErr != nil { + return nil, chartErr + } + roots = append(roots, chartRoots...) } - roots = append(roots, chartRoots...) var imageRoots []*Node var imageWarnings []Warning - imageRoots, imageWarnings, imageErr = discoverImageDependencyRoots(absPath, imageScanManagers(opts.Managers)) + imageRoots, imageWarnings, imageErr = discoverImageDependencyRoots(absPath, imageScanManagers(opts.Managers), matcher) warnings = append(warnings, imageWarnings...) roots = append(roots, imageRoots...) } @@ -111,8 +120,11 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { filteredRoots = append(filteredRoots, filtered) } } - dups := analyzeDuplicates(filteredRoots) - applyDuplicateRefs(filteredRoots, dups) + var dups map[string]*Duplicate + if opts.ShowDuplicates { + dups = analyzeDuplicates(filteredRoots) + applyDuplicateRefs(filteredRoots, dups) + } nodes, edges, stats := flatten(filteredRoots, dups) stats.Projects = projectsScanned diff --git a/deps/scan_image.go b/deps/scan_image.go index 139093a..1905851 100644 --- a/deps/scan_image.go +++ b/deps/scan_image.go @@ -1,6 +1,7 @@ package deps import ( + "fmt" "path/filepath" "sort" @@ -8,7 +9,7 @@ import ( "github.com/flanksource/repomap/imageupdate" ) -func discoverImageDependencyRoots(root string, managers []Manager) ([]*Node, []Warning, error) { +func discoverImageDependencyRoots(root string, managers []Manager, matcher repomap.ResourceMatcher) ([]*Node, []Warning, error) { selected := managerSet(managers) conf, err := repomap.GetConf(root) if err != nil { @@ -18,6 +19,10 @@ func discoverImageDependencyRoots(root string, managers []Manager) ([]*Node, []W if err != nil { return nil, nil, err } + if res.UntrackedTarget != "" { + return nil, nil, fmt.Errorf("%s is not tracked by git; deps only scans git-tracked manifests — run 'git add %s' first", res.UntrackedTarget, res.UntrackedTarget) + } + targets := imageupdate.Filter(res.Targets, matcher, nil, nil) var imageChildren []*Node var helmChildren []*Node @@ -25,7 +30,7 @@ func discoverImageDependencyRoots(root string, managers []Manager) ([]*Node, []W for _, w := range res.Warnings { warnings = append(warnings, Warning{Manager: ManagerHelm, Project: w.File, Message: w.Message}) } - for _, target := range res.Targets { + for _, target := range targets { manager := managerForUpdateTarget(target) if manager == "" || (len(selected) > 0 && !selected[manager]) { continue diff --git a/deps/scan_test.go b/deps/scan_test.go index 6589e39..c06df6d 100644 --- a/deps/scan_test.go +++ b/deps/scan_test.go @@ -193,6 +193,118 @@ func TestScanImageAndHelmManifestTargets(t *testing.T) { } } +func TestScanResourceFilterByKind(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "workloads.yaml"), deploymentUpdateFixture) + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + runGit(t, dir, "add", ".") + + cases := []struct { + name string + kind string + keep Manager + drop Manager + wantChild string + }{ + {name: "HelmRelease selects the chart", kind: "HelmRelease", keep: ManagerHelm, drop: ManagerImage, wantChild: "podinfo"}, + {name: "Deployment selects container images", kind: "Deployment", keep: ManagerImage, drop: ManagerHelm, wantChild: "nginx"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := Scan(context.Background(), ".", Options{ + Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Kind: []string{tc.kind}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if dropped := findRoot(got.Roots, tc.drop); dropped != nil { + t.Fatalf("kind=%s should exclude the %s root, got %#v", tc.kind, tc.drop, dropped) + } + kept := findRoot(got.Roots, tc.keep) + if kept == nil || findChild(kept, tc.wantChild) == nil { + t.Fatalf("kind=%s should keep %s/%s, got %#v", tc.kind, tc.keep, tc.wantChild, kept) + } + }) + } +} + +func TestScanResourceFilterExcludesChartDirectories(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + writeChartFixture(t, dir) // a Chart.yaml-sourced helm root with no Kubernetes resource metadata + runGit(t, dir, "add", ".") + + got, err := Scan(context.Background(), ".", Options{ + Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Kind: []string{"HelmRelease"}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + for _, root := range got.Roots { + if root.Source == "Chart.yaml" { + t.Fatalf("a resource filter must exclude Chart.yaml roots (no k8s metadata), got %#v", root) + } + } + if helmRoot := findRoot(got.Roots, ManagerHelm); helmRoot == nil || findChild(helmRoot, "podinfo") == nil { + t.Fatalf("kind=HelmRelease should keep the manifest-sourced podinfo chart, got %#v", got.Roots) + } +} + +func TestScanSingleFileTarget(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "workloads.yaml"), deploymentUpdateFixture) + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + runGit(t, dir, "add", ".") + + got, err := Scan(context.Background(), filepath.Join("apps", "helmrelease.yaml"), Options{ + Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if imageRoot := findRoot(got.Roots, ManagerImage); imageRoot != nil { + t.Fatalf("scanning a single HelmRelease file must not surface images from another file: %#v", imageRoot) + } + helmRoot := findRoot(got.Roots, ManagerHelm) + if helmRoot == nil || findChild(helmRoot, "podinfo") == nil { + t.Fatalf("single-file scan should resolve the file's own chart, got %#v", got.Roots) + } +} + +func TestScanUntrackedFileErrors(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + // Deliberately do NOT `git add` — deps only scans git-tracked manifests. + + _, err := Scan(context.Background(), filepath.Join("apps", "helmrelease.yaml"), Options{ + Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err == nil { + t.Fatal("scanning an untracked file should error") + } + if !strings.Contains(err.Error(), "not tracked") { + t.Fatalf("error should explain the file is untracked, got %q", err.Error()) + } +} + func TestNodeImplementsClickyTreeNode(t *testing.T) { var _ api.TreeNode = (*Node)(nil) diff --git a/deps/update_image.go b/deps/update_image.go index 8f2d1a7..5845dcc 100644 --- a/deps/update_image.go +++ b/deps/update_image.go @@ -2,6 +2,7 @@ package deps import ( "context" + "errors" "fmt" "path/filepath" "strings" @@ -134,6 +135,11 @@ func availableImageTargetVersions(ctx context.Context, resolver ImageVersionReso if candidate.Target == nil { return nil, "", "", fmt.Errorf("%s has no image or Helm target metadata", candidate.Name) } + // An unresolved Flux source leaves RepoURL empty; surface the actionable + // resolution error rather than querying an empty registry/Helm URL. + if candidate.Target.SourceErr != "" { + return nil, "", "", errors.New(candidate.Target.SourceErr) + } if resolver == nil { resolver = imageupdate.NewResolver() } diff --git a/deps/update_match.go b/deps/update_match.go index 0f9c7ae..fd564e0 100644 --- a/deps/update_match.go +++ b/deps/update_match.go @@ -6,6 +6,8 @@ import ( "strings" "github.com/flanksource/commons/collections" + + "github.com/flanksource/repomap/imageupdate" ) func updateManagers(managers []Manager) ([]Manager, error) { @@ -149,6 +151,38 @@ func (c UpdateCandidate) key() string { return strings.Join([]string{string(c.Manager), c.Dir, c.File, c.Scope, c.Name}, "\x00") } +// resolutionKey groups candidates whose published-version lookup yields the same +// result, so one network round-trip can serve every occurrence. Image tags come +// from the repository, chart versions from the chart+repo source, and package +// versions from the registry resolved in the project dir. +func (c UpdateCandidate) resolutionKey() string { + switch c.Manager { + case ManagerImage: + return strings.Join([]string{string(ManagerImage), c.Name}, "\x00") + case ManagerHelm: + return strings.Join([]string{string(ManagerHelm), c.Name, helmSourceKey(c.Target)}, "\x00") + default: + return strings.Join([]string{string(c.Manager), c.Name, c.Dir}, "\x00") + } +} + +// helmSourceKey identifies the chart repository a Helm candidate resolves +// against; charts sharing a name but a different repo (or an unresolved source) +// must not collapse onto the same lookup. +func helmSourceKey(target *imageupdate.UpdateTarget) string { + if target == nil { + return "" + } + if target.SourceErr != "" { + return "err:" + target.SourceErr + } + key := target.RepoURL + if target.IsOCI { + key += "|oci" + } + return key +} + func (c UpdateCandidate) less(other UpdateCandidate) bool { if c.Manager != other.Manager { return c.Manager < other.Manager diff --git a/deps/update_modes_test.go b/deps/update_modes_test.go index fca4ed1..100e4ab 100644 --- a/deps/update_modes_test.go +++ b/deps/update_modes_test.go @@ -137,6 +137,69 @@ func TestUpdate_ResourceFilterKindNarrowsToHelm(t *testing.T) { } } +func TestUpdate_DedupesVersionLookupsAcrossDuplicates(t *testing.T) { + setupImageRepo(t, map[string]string{ + "apps/a.yaml": deploymentUpdateFixture, + "apps/b.yaml": deploymentUpdateFixture, // the same nginx + proxy in a second file + }) + resolver := newCountingImageVersionResolver(map[string][]string{ + "nginx": {"1.27.0", "1.25.3"}, + "ghcr.io/flanksource/proxy": {"v0.5.0", "v0.4.1"}, + }) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Check: true, + ImageResolver: resolver, + }) + if err != nil { + t.Fatal(err) + } + // Two images each appear in two files, so four updates are planned but each + // image's published versions must be looked up exactly once. + if len(plans) != 4 { + t.Fatalf("plans = %d, want 4 (two images × two files): %#v", len(plans), plans) + } + if got := resolver.callCount("nginx"); got != 1 { + t.Fatalf("nginx version lookups = %d, want 1 deduped lookup", got) + } + if got := resolver.callCount("ghcr.io/flanksource/proxy"); got != 1 { + t.Fatalf("proxy version lookups = %d, want 1 deduped lookup", got) + } +} + +func TestUpdate_FilterThenChecksOnlyFilteredSet(t *testing.T) { + setupImageRepo(t, map[string]string{ + "apps/workloads.yaml": deploymentUpdateFixture, + "apps/helmrelease.yaml": helmReleaseUpdateFixture, + }) + resolver := newCountingImageVersionResolver(map[string][]string{ + "nginx": {"1.27.0", "1.25.3"}, + "ghcr.io/flanksource/proxy": {"v0.5.0", "v0.4.1"}, + "podinfo": {"6.6.0", "6.5.0"}, + }) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage, ManagerHelm}, + Kind: []string{"HelmRelease"}, + Check: true, + ImageResolver: resolver, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 || plans[0].Manager != ManagerHelm || plans[0].Name != "podinfo" { + t.Fatalf("plans = %#v, want only the filtered HelmRelease chart", plans) + } + // The kind filter must restrict the lookups: Deployment images are never queried. + if got := resolver.callCount("nginx") + resolver.callCount("ghcr.io/flanksource/proxy"); got != 0 { + t.Fatalf("filtered-out image lookups = %d, want 0 (filter before check)", got) + } + if got := resolver.callCount("podinfo"); got != 1 { + t.Fatalf("podinfo lookups = %d, want 1", got) + } +} + func TestUpdate_NoExpressionMatchesAll(t *testing.T) { setupImageRepo(t, map[string]string{"apps/helmrelease.yaml": helmReleaseUpdateFixture}) diff --git a/deps/update_resolve.go b/deps/update_resolve.go index caa8e55..4995dbd 100644 --- a/deps/update_resolve.go +++ b/deps/update_resolve.go @@ -11,26 +11,66 @@ import ( ) func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) ([]UpdateChoice, map[string]UpdatePlan) { - type result struct { - versions []string - latestStable string - latestPrerelease string - err error + rawByKey := resolveRawVersionsByKey(ctx, candidates, opts) + + plansByKey := map[string]UpdatePlan{} + choices := make([]UpdateChoice, 0, len(candidates)) + for _, candidate := range candidates { + raw := rawByKey[candidate.resolutionKey()] + if raw.err != nil { + plansByKey[candidate.key()] = skippedUpdatePlan(candidate, raw.err.Error()) + continue + } + // The "newer than current" filtering is per occurrence: duplicates of the + // same dependency may sit at different current versions even though they + // share one published-version list. + versions := updateableVersions(candidate.Current, raw.versions) + if len(versions) == 0 { + continue + } + choices = append(choices, UpdateChoice{ + Candidate: candidate, + Versions: versions, + LatestStable: latestStableVersion(versions), + LatestPrerelease: latestPrereleaseVersion(versions), + }) + } + return choices, plansByKey +} + +type rawVersionResult struct { + versions []string + err error +} + +// resolveRawVersionsByKey looks up the published versions for every distinct +// dependency source among candidates, running one lookup per source and sharing +// it across duplicate occurrences. The same image repository, chart+repo, or +// package resolves identically wherever it is referenced, so collapsing them +// avoids redundant registry/proxy round-trips. The result is keyed by +// UpdateCandidate.resolutionKey. +func resolveRawVersionsByKey(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) map[string]rawVersionResult { + keys := make([]string, 0, len(candidates)) + rep := map[string]UpdateCandidate{} + for _, candidate := range candidates { + key := candidate.resolutionKey() + if _, ok := rep[key]; !ok { + rep[key] = candidate + keys = append(keys, key) + } } - results := make([]result, len(candidates)) + + results := make([]rawVersionResult, len(keys)) group := task.StartGroup[int]("Resolving dependency versions", task.WithConcurrency(updateResolveConcurrency)) - for i, candidate := range candidates { - idx, c := i, candidate - group.Add(updateTaskName(c), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { + for i, key := range keys { + idx, candidate := i, rep[key] + group.Add(updateTaskName(candidate), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { tk.Infof("looking up published versions") - versions, latestStable, latestPrerelease, err := resolveCandidateVersions(ctx, opts, c) - results[idx] = result{versions: versions, latestStable: latestStable, latestPrerelease: latestPrerelease, err: err} + versions, err := resolveCandidateRawVersions(ctx, opts, candidate) + results[idx] = rawVersionResult{versions: versions, err: err} if err != nil { tk.Warnf("%s", err.Error()) tk.Warning() - } else if len(versions) == 0 { - tk.Infof("already up to date") - tk.Success() } else { tk.Success() } @@ -39,43 +79,24 @@ func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opt } _, _ = group.GetResults() - plansByKey := map[string]UpdatePlan{} - choices := make([]UpdateChoice, 0, len(candidates)) - for i, candidate := range candidates { - result := results[i] - if result.err != nil { - plansByKey[candidate.key()] = skippedUpdatePlan(candidate, result.err.Error()) - continue - } - if len(result.versions) == 0 { - continue - } - choices = append(choices, UpdateChoice{ - Candidate: candidate, - Versions: result.versions, - LatestStable: result.latestStable, - LatestPrerelease: result.latestPrerelease, - }) + byKey := make(map[string]rawVersionResult, len(keys)) + for i, key := range keys { + byKey[key] = results[i] } - return choices, plansByKey + return byKey } -func resolveCandidateVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, string, string, error) { - var ( - versions []string - err error - ) +// resolveCandidateRawVersions returns a candidate source's published versions +// without the per-occurrence "newer than current" filtering, so the result can +// be cached and reused across duplicate occurrences. +func resolveCandidateRawVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, error) { switch candidate.Manager { case ManagerImage, ManagerHelm: - versions, _, _, err = availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + versions, _, _, err := availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + return versions, err default: - versions, err = AvailableDependencyVersions(ctx, opts.Runner, candidate) - } - if err != nil { - return nil, "", "", err + return AvailableDependencyVersions(ctx, opts.Runner, candidate) } - versions = updateableVersions(candidate.Current, versions) - return versions, latestStableVersion(versions), latestPrereleaseVersion(versions), nil } func AvailableDependencyVersions(ctx context.Context, runner CommandRunner, candidate UpdateCandidate) ([]string, error) { diff --git a/deps/update_test.go b/deps/update_test.go index 15ca9ec..5723632 100644 --- a/deps/update_test.go +++ b/deps/update_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "testing" "github.com/flanksource/repomap/imageupdate" @@ -297,6 +298,52 @@ func TestDiscoverUpdateCandidates_ImageAndHelmTargets(t *testing.T) { } } +func TestUpdateHelmReleaseWithMissingSourceReportsClearError(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseMissingSourceFixture) + runGit(t, dir, "add", ".") + + resolver := &recordingImageVersionResolver{} + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerHelm}, + Check: true, + ImageResolver: resolver, + }) + if err != nil { + t.Fatalf("Update returned fatal error, want graceful per-chart skip: %v", err) + } + if len(plans) != 1 { + t.Fatalf("plans = %#v, want 1", plans) + } + if !strings.Contains(plans[0].Skipped, "HelmRepository flux-system/mission-control-oipa which was not found in the scanned manifests") { + t.Fatalf("skipped reason = %q, want the actionable HelmRepository-not-found error", plans[0].Skipped) + } + if resolver.calls.Load() != 0 { + t.Fatalf("resolver was queried %d times; an unresolved chart source must short-circuit before any lookup", resolver.calls.Load()) + } +} + +// recordingImageVersionResolver counts version lookups so a test can assert an +// unresolved chart source short-circuits before any network query. +type recordingImageVersionResolver struct{ calls atomic.Int64 } + +func (r *recordingImageVersionResolver) Available(context.Context, imageupdate.UpdateTarget) ([]string, error) { + r.calls.Add(1) + return nil, nil +} + +func (r *recordingImageVersionResolver) ResolveLatestVersions(context.Context, imageupdate.UpdateTarget) (imageupdate.LatestVersions, error) { + r.calls.Add(1) + return imageupdate.LatestVersions{}, nil +} + +func (r *recordingImageVersionResolver) NewImageValue(context.Context, imageupdate.UpdateTarget, string) (string, error) { + r.calls.Add(1) + return "", nil +} + func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { dir := t.TempDir() t.Chdir(dir) @@ -614,6 +661,44 @@ func (fakeImageVersionResolver) NewImageValue(_ context.Context, target imageupd return stripImageVersion(target.CurrentValue) + ":" + version, nil } +// countingImageVersionResolver records how many times each target's published +// versions are looked up, so tests can assert duplicate occurrences share a +// single network round-trip. +type countingImageVersionResolver struct { + versions map[string][]string + mu sync.Mutex + calls map[string]int +} + +func newCountingImageVersionResolver(versions map[string][]string) *countingImageVersionResolver { + return &countingImageVersionResolver{versions: versions, calls: map[string]int{}} +} + +func (r *countingImageVersionResolver) Available(_ context.Context, target imageupdate.UpdateTarget) ([]string, error) { + r.mu.Lock() + r.calls[updateTargetName(target)]++ + r.mu.Unlock() + return sortDependencyVersions(r.versions[updateTargetName(target)]), nil +} + +func (r *countingImageVersionResolver) ResolveLatestVersions(_ context.Context, target imageupdate.UpdateTarget) (imageupdate.LatestVersions, error) { + versions := sortDependencyVersions(r.versions[updateTargetName(target)]) + return imageupdate.LatestVersions{ + Stable: latestStableVersion(versions), + Prerelease: latestPrereleaseVersion(versions), + }, nil +} + +func (*countingImageVersionResolver) NewImageValue(_ context.Context, target imageupdate.UpdateTarget, version string) (string, error) { + return stripImageVersion(target.CurrentValue) + ":" + version, nil +} + +func (r *countingImageVersionResolver) callCount(name string) int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[name] +} + const deploymentUpdateFixture = `apiVersion: apps/v1 kind: Deployment metadata: @@ -652,3 +737,20 @@ metadata: spec: url: https://stefanprodan.github.io/podinfo ` + +// helmReleaseMissingSourceFixture references a HelmRepository that is absent from +// the scanned manifests, mirroring the real mission-control-oipa breakage. +const helmReleaseMissingSourceFixture = `apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: mission-control-oipa +spec: + chart: + spec: + chart: mission-control-oipa-chart + version: 0.0.9 + sourceRef: + kind: HelmRepository + name: mission-control-oipa + namespace: flux-system +` From 8ebe3cabfc4a339d8c66c680fb2a1a788759f798 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Thu, 6 Aug 2026 09:10:32 +0300 Subject: [PATCH 08/13] feat(deps): add dependency filters and grouped version prompts Expose MatchItem dependency filters through `deps update` and combine them with positional patterns. Deduplicate identical version prompts so one confirmation updates repeated declarations consistently. BREAKING CHANGE: rename `UpdateOptions.Expression` to `Filters` and change `VersionSelector` to receive `UpdateVersionPrompt`. --- .gitignore | 1 + cmd/repomap/deps.go | 61 +++++++----- cmd/repomap/deps_test.go | 115 +++++++++++++++++++++++ deps/update.go | 32 +++++-- deps/update_modes_test.go | 2 +- deps/update_prompt.go | 54 +++++++++-- deps/update_prompt_test.go | 185 +++++++++++++++++++++++++++++++++++++ deps/update_test.go | 60 ++++++------ 8 files changed, 440 insertions(+), 70 deletions(-) create mode 100644 deps/update_prompt_test.go diff --git a/.gitignore b/.gitignore index e0081f4..7a996d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .bin/ +.grite/ diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index da30874..6cd524d 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -28,6 +28,7 @@ type DepsOptions struct { type DepsUpdateOptions struct { Args []string `json:"args" args:"true" help:"Optional dependency MatchItem expression followed by optional path"` Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to update: go, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns (MatchItem syntax) matched against name, manager, manager:name, manager:name@version, scope, and version; use path:/file: for manifest paths; supports comma-separated values and !exclusions"` Kind []string `json:"kind,omitempty" flag:"kind,k" help:"Filter image/helm targets by kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` Namespace []string `json:"namespace,omitempty" flag:"namespace,n" help:"Filter image/helm targets by namespace (MatchItem syntax)"` Name []string `json:"name,omitempty" flag:"name" help:"Filter image/helm targets by resource name (MatchItem syntax)"` @@ -97,22 +98,29 @@ EXAMPLES: func (opts DepsUpdateOptions) Help() api.Text { return clicky.Text(`Update direct package, image, and Helm chart dependencies. -The optional expr argument uses commons MatchItem syntax and is matched against -dependency names, manager-qualified names, versions, and scopes. Manifest path -matching is explicit with path: or file:. With no expr, every -matched dependency is considered. Image and Helm targets (from git-tracked +Dependencies are narrowed with --filter and/or the optional positional expr; +both use commons MatchItem syntax, accept comma-separated values and +!exclusions, and are combined into a single pattern set. Patterns are matched +against dependency names, manager-qualified names, versions, and scopes. +Manifest path matching is explicit with path: or file:. With +no patterns, every matched dependency is considered. Image and Helm targets (from git-tracked Kubernetes/Flux manifests, including HelmRelease spec.chartRef OCIRepository and HelmChart sources) can be further narrowed with --kind/--namespace/--name/ --selector and the --image/--chart name patterns. -By default repomap prompts for which dependencies and versions to apply. Use ---latest to resolve each to its highest stable version, or --version to apply a -concrete version, both non-interactively. Applied updates are staged with git add +By default repomap prompts for which dependencies and versions to apply. A +dependency declared in several manifests at the same version is confirmed once +and the chosen version written to every occurrence; occurrences sitting at +different current versions are still confirmed separately. Use --latest to +resolve each to its highest stable version, or --version to apply a concrete +version, both non-interactively. Applied updates are staged with git add (manifests plus lockfiles); --dry-run and --check never stage. Use --check to list updateable dependencies without prompting or writing. EXAMPLES: + repomap deps update --filter 'github.com/flanksource/*' + repomap deps update --filter '*flanksource*,!*test*' --check repomap deps update 'github.com/flanksource/*' repomap deps update '*' --check repomap deps update --manager helm -k HelmRelease --latest @@ -172,23 +180,19 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { if err != nil { return nil, err } - var expression []string - if expr != "" { - expression = []string{expr} - } plans, err := depgraph.Update(ctx, path, depgraph.UpdateOptions{ - Managers: managers, - Expression: expression, - Kind: opts.Kind, - Namespace: opts.Namespace, - Name: opts.Name, - Selector: opts.Selector, - Image: opts.Image, - Chart: opts.Chart, - Latest: opts.Latest, - Version: opts.Version, - Check: opts.Check, - DryRun: opts.DryRun, + Managers: managers, + Filters: updateFilters(opts.Filter, expr), + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, + Image: opts.Image, + Chart: opts.Chart, + Latest: opts.Latest, + Version: opts.Version, + Check: opts.Check, + DryRun: opts.DryRun, }) if err != nil { return nil, err @@ -196,6 +200,17 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { return api.NewTableFrom(plans), nil } +// updateFilters combines --filter values with the optional positional expr into +// one MatchItem pattern set. Patterns are forwarded raw because depgraph.Update +// already comma-splits and trims them. +func updateFilters(filter []string, expr string) []string { + out := append([]string{}, filter...) + if expr != "" { + out = append(out, expr) + } + return out +} + // parseDepsUpdateArgs interprets the optional positional [expr] [path]. With one // argument, an existing directory is treated as the path and anything else as the // expression, so `deps update ./clusters` and `deps update 'left-pad'` both work. diff --git a/cmd/repomap/deps_test.go b/cmd/repomap/deps_test.go index b8406bc..647ca00 100644 --- a/cmd/repomap/deps_test.go +++ b/cmd/repomap/deps_test.go @@ -137,3 +137,118 @@ func TestDepsUpdateCommandRegistered(t *testing.T) { t.Fatalf("manager help should document update-supported managers, got %q", manager.Usage) } } + +// deps update must declare its own --filter, otherwise the flag silently binds +// to clicky's persistent format flag (`--filter string`, a CEL output filter) +// and the MatchItem patterns never reach depgraph.Update. +func TestDepsUpdateFilterFlagShadowsGlobalCELFilter(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps", "update"}) + if err != nil { + t.Fatal(err) + } + flag := cmd.Flags().Lookup("filter") + if flag == nil { + t.Fatal("filter flag not registered on deps update") + } + if got := flag.Value.Type(); got != "stringSlice" { + t.Fatalf("filter flag type = %q, want stringSlice (clicky's global CEL filter is a string)", got) + } + if !strings.Contains(flag.Usage, "MatchItem syntax") { + t.Fatalf("filter help should document MatchItem syntax, got %q", flag.Usage) + } +} + +func TestDepsUpdateFilterFlagBindsToOptions(t *testing.T) { + var got DepsUpdateOptions + root := &cobra.Command{Use: "test"} + clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format") + deps := clicky.AddNamedCommandWithContext("deps", root, DepsOptions{}, func(_ context.Context, _ DepsOptions) (*depgraph.Export, error) { + return &depgraph.Export{}, nil + }) + clicky.AddNamedCommandWithContext("update", deps, DepsUpdateOptions{}, func(_ context.Context, opts DepsUpdateOptions) (any, error) { + got = opts + return nil, nil + }) + + root.SetArgs([]string{"deps", "update", "--filter", "*flanksource*,!*test*", "--filter", "left-pad", "--json"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + // cobra's stringSlice comma-splits at parse time; depgraph.Update splits + // again for the positional expr, which arrives as one unsplit string. + want := []string{"*flanksource*", "!*test*", "left-pad"} + if len(got.Filter) != len(want) { + t.Fatalf("Filter = %#v, want %#v", got.Filter, want) + } + for i := range want { + if got.Filter[i] != want[i] { + t.Fatalf("Filter[%d] = %q, want %q", i, got.Filter[i], want[i]) + } + } +} + +func TestUpdateFiltersCombinesFlagAndPositionalExpr(t *testing.T) { + cases := []struct { + name string + filter []string + expr string + want []string + }{ + {name: "neither", want: nil}, + {name: "flag only", filter: []string{"npm:@scope/*"}, want: []string{"npm:@scope/*"}}, + {name: "expr only", expr: "left-pad", want: []string{"left-pad"}}, + { + name: "flag and expr combined", + filter: []string{"*flanksource*", "!*test*"}, + expr: "path:apps/*/package.json", + want: []string{"*flanksource*", "!*test*", "path:apps/*/package.json"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := updateFilters(tc.filter, tc.expr) + if len(got) != len(tc.want) { + t.Fatalf("updateFilters(%#v, %q) = %#v, want %#v", tc.filter, tc.expr, got, tc.want) + } + for i := range tc.want { + if got[i] != tc.want[i] { + t.Fatalf("pattern[%d] = %q, want %q", i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestParseDepsUpdateArgs(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name string + args []string + wantExpr string + wantPath string + wantErr bool + }{ + {name: "no args defaults to cwd", wantPath: "."}, + {name: "existing dir is the path", args: []string{dir}, wantPath: dir}, + {name: "non-dir is the expression", args: []string{"left-pad"}, wantExpr: "left-pad", wantPath: "."}, + {name: "expr then path", args: []string{"left-pad", dir}, wantExpr: "left-pad", wantPath: dir}, + {name: "too many args", args: []string{"a", "b", "c"}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + expr, path, err := parseDepsUpdateArgs(tc.args) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %#v", tc.args) + } + return + } + if err != nil { + t.Fatal(err) + } + if expr != tc.wantExpr || path != tc.wantPath { + t.Fatalf("parseDepsUpdateArgs(%#v) = (%q, %q), want (%q, %q)", tc.args, expr, path, tc.wantExpr, tc.wantPath) + } + }) + } +} diff --git a/deps/update.go b/deps/update.go index de17a78..ce5a087 100644 --- a/deps/update.go +++ b/deps/update.go @@ -25,7 +25,7 @@ var supportedUpdateManagers = map[Manager]bool{ } type CandidateSelector func([]UpdateChoice) ([]UpdateChoice, bool) -type VersionSelector func(UpdateChoice) (string, bool) +type VersionSelector func(UpdateVersionPrompt) (string, bool) type ImageVersionResolver interface { Available(context.Context, imageupdate.UpdateTarget) ([]string, error) @@ -34,8 +34,8 @@ type ImageVersionResolver interface { } type UpdateOptions struct { - Managers []Manager - Expression []string + Managers []Manager + Filters []string // Resource filters for image/helm targets (ignored by package managers). Kind []string @@ -85,6 +85,14 @@ type UpdateChoice struct { LatestPrerelease string `json:"latest_prerelease,omitempty"` } +// UpdateVersionPrompt is a single version question standing in for every +// selected occurrence of the same dependency. Candidate is the first +// occurrence; Files lists each distinct manifest the answer will be written to. +type UpdateVersionPrompt struct { + UpdateChoice + Files []string `json:"files"` +} + type UpdatePlan struct { Manager Manager `json:"manager"` Name string `json:"name"` @@ -112,7 +120,7 @@ func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, if err != nil { return nil, err } - patterns := splitUpdatePatterns(opts.Expression) + patterns := splitUpdatePatterns(opts.Filters) if opts.Runner == nil { opts.Runner = ExecRunner{} } @@ -187,13 +195,17 @@ func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, if selectVersion == nil { selectVersion = promptUpdateVersion } - for _, choice := range sortSelectedUpdateChoicesByFile(selected) { - version, ok := selectVersion(choice) - if !ok || strings.TrimSpace(version) == "" { - plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "no version selected") - continue + // A dependency repeated across manifests poses one question, so occurrences + // sharing a version prompt are confirmed once and the answer applied to all. + for _, group := range groupChoicesForVersionPrompt(sortSelectedUpdateChoicesByFile(selected)) { + version, ok := selectVersion(newUpdateVersionPrompt(group)) + for _, choice := range group { + if !ok || strings.TrimSpace(version) == "" { + plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "no version selected") + continue + } + plansByKey[choice.Candidate.key()] = applyDependencyUpdate(ctx, choice.Candidate, version, opts) } - plansByKey[choice.Candidate.key()] = applyDependencyUpdate(ctx, choice.Candidate, version, opts) } return orderedUpdatePlans(candidates, plansByKey), nil } diff --git a/deps/update_modes_test.go b/deps/update_modes_test.go index 100e4ab..5cf1851 100644 --- a/deps/update_modes_test.go +++ b/deps/update_modes_test.go @@ -200,7 +200,7 @@ func TestUpdate_FilterThenChecksOnlyFilteredSet(t *testing.T) { } } -func TestUpdate_NoExpressionMatchesAll(t *testing.T) { +func TestUpdate_NoFiltersMatchesAll(t *testing.T) { setupImageRepo(t, map[string]string{"apps/helmrelease.yaml": helmReleaseUpdateFixture}) plans, err := Update(context.Background(), ".", UpdateOptions{ diff --git a/deps/update_prompt.go b/deps/update_prompt.go index 2740061..a054934 100644 --- a/deps/update_prompt.go +++ b/deps/update_prompt.go @@ -13,10 +13,52 @@ func promptUpdateCandidates(choices []UpdateChoice) ([]UpdateChoice, bool) { return runUpdateChoiceTreePicker(choices) } -func promptUpdateVersion(choice UpdateChoice) (string, bool) { - candidate := choice.Candidate - title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, candidate.File, candidate.Current) - return clicky.PromptSelect(choice.Versions, clicky.PromptSelectOptions[string]{ +// versionPromptKey identifies occurrences that pose the identical version +// question: the same dependency, at the same current version, with the same +// available versions. Occurrences that differ in any of those are genuinely +// different decisions and stay separate prompts. +func (c UpdateChoice) versionPromptKey() string { + parts := append([]string{string(c.Candidate.Manager), c.Candidate.Name, c.Candidate.Current}, c.Versions...) + return strings.Join(parts, "\x00") +} + +// groupChoicesForVersionPrompt collapses occurrences sharing a version question +// into one group, keeping the input order of each group's first occurrence. +func groupChoicesForVersionPrompt(choices []UpdateChoice) [][]UpdateChoice { + indexByKey := map[string]int{} + var groups [][]UpdateChoice + for _, choice := range choices { + key := choice.versionPromptKey() + if i, ok := indexByKey[key]; ok { + groups[i] = append(groups[i], choice) + continue + } + indexByKey[key] = len(groups) + groups = append(groups, []UpdateChoice{choice}) + } + return groups +} + +func newUpdateVersionPrompt(group []UpdateChoice) UpdateVersionPrompt { + files := make([]string, 0, len(group)) + seen := map[string]bool{} + for _, choice := range group { + if file := choice.Candidate.File; !seen[file] { + seen[file] = true + files = append(files, file) + } + } + return UpdateVersionPrompt{UpdateChoice: group[0], Files: files} +} + +func promptUpdateVersion(prompt UpdateVersionPrompt) (string, bool) { + candidate := prompt.Candidate + location := candidate.File + if len(prompt.Files) > 1 { + location = fmt.Sprintf("%d files", len(prompt.Files)) + } + title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, location, candidate.Current) + return clicky.PromptSelect(prompt.Versions, clicky.PromptSelectOptions[string]{ Title: title, PageSize: 12, Render: func(version string) api.Textable { @@ -25,10 +67,10 @@ func promptUpdateVersion(choice UpdateChoice) (string, bool) { if selectedVersionIsCurrent(candidate.Current, version) { tags = append(tags, "current") } - if version == choice.LatestStable { + if version == prompt.LatestStable { tags = append(tags, "latest stable") } - if version == choice.LatestPrerelease { + if version == prompt.LatestPrerelease { tags = append(tags, "latest pre-release") } if isPrerelease(version) { diff --git a/deps/update_prompt_test.go b/deps/update_prompt_test.go new file mode 100644 index 0000000..6d05815 --- /dev/null +++ b/deps/update_prompt_test.go @@ -0,0 +1,185 @@ +package deps + +import ( + "context" + "path/filepath" + "strings" + "sync" + "testing" +) + +// A dependency declared in several manifests is one decision, so the version is +// confirmed once and the answer written to every occurrence. +func TestUpdatePromptsOnceForDependencyRepeatedAcrossFiles(t *testing.T) { + dir := t.TempDir() + for _, project := range []string{"api", "web"} { + writeFile(t, filepath.Join(dir, project, "package.json"), `{ + "name": "`+project+`", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, project, "package-lock.json"), `{"lockfileVersion": 3}`) + } + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.3.0","1.4.0"]`}, + }, + } + + var ( + mu sync.Mutex + prompts []UpdateVersionPrompt + ) + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, + SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { + if len(choices) != 2 { + t.Fatalf("both occurrences should be selectable, got %d", len(choices)) + } + return choices, true + }, + SelectVersion: func(prompt UpdateVersionPrompt) (string, bool) { + mu.Lock() + defer mu.Unlock() + prompts = append(prompts, prompt) + return "1.4.0", true + }, + }) + if err != nil { + t.Fatal(err) + } + if len(prompts) != 1 { + t.Fatalf("version prompts = %d, want 1: %#v", len(prompts), prompts) + } + // UpdateCandidate.File is cwd-relative, so assert on the manifest suffixes. + gotFiles := prompts[0].Files + wantSuffixes := []string{"api/package.json", "web/package.json"} + if len(gotFiles) != len(wantSuffixes) { + t.Fatalf("prompt files = %#v, want one entry per manifest %#v", gotFiles, wantSuffixes) + } + for i, suffix := range wantSuffixes { + if !strings.HasSuffix(filepath.ToSlash(gotFiles[i]), suffix) { + t.Fatalf("prompt file[%d] = %q, want a path ending in %q", i, gotFiles[i], suffix) + } + } + if len(plans) != 2 { + t.Fatalf("plans = %d, want 2: %#v", len(plans), plans) + } + for _, plan := range plans { + if plan.NewVersion != "1.4.0" || plan.Skipped != "" { + t.Fatalf("single answer should apply to every occurrence, got %#v", plan) + } + } +} + +// Declining the shared prompt skips every occurrence it covered. +func TestUpdateSharedPromptCancellationSkipsAllOccurrences(t *testing.T) { + dir := t.TempDir() + for _, project := range []string{"api", "web"} { + writeFile(t, filepath.Join(dir, project, "package.json"), `{ + "name": "`+project+`", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, project, "package-lock.json"), `{"lockfileVersion": 3}`) + } + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.3.0","1.4.0"]`}, + }, + } + + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, + SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { + return choices, true + }, + SelectVersion: func(UpdateVersionPrompt) (string, bool) { + return "", false + }, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 2 { + t.Fatalf("plans = %d, want 2: %#v", len(plans), plans) + } + for _, plan := range plans { + if plan.Skipped != "no version selected" || plan.Written { + t.Fatalf("declined prompt should skip every occurrence, got %#v", plan) + } + } +} + +func TestGroupChoicesForVersionPrompt(t *testing.T) { + choice := func(name, current, file string, versions ...string) UpdateChoice { + return UpdateChoice{ + Candidate: UpdateCandidate{Manager: ManagerNPM, Name: name, Current: current, File: file}, + Versions: versions, + } + } + cases := []struct { + name string + choices []UpdateChoice + want [][]string + }{ + { + name: "same dependency in different files shares one prompt", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.3.0", "web/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json", "web/package.json"}}, + }, + { + name: "different current versions are different decisions", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.2.0", "web/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json"}, {"web/package.json"}}, + }, + { + name: "different available versions are different decisions", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.3.0", "web/package.json", "1.4.0", "1.5.0"), + }, + want: [][]string{{"api/package.json"}, {"web/package.json"}}, + }, + { + name: "different dependencies never merge", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("right-pad", "^1.3.0", "api/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json"}, {"api/package.json"}}, + }, + { + name: "two occurrences in one file collapse to a single file entry", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + groups := groupChoicesForVersionPrompt(tc.choices) + if len(groups) != len(tc.want) { + t.Fatalf("groups = %d, want %d", len(groups), len(tc.want)) + } + for i, group := range groups { + got := newUpdateVersionPrompt(group).Files + if strings.Join(got, ",") != strings.Join(tc.want[i], ",") { + t.Fatalf("group[%d] files = %#v, want %#v", i, got, tc.want[i]) + } + } + }) + } +} diff --git a/deps/update_test.go b/deps/update_test.go index 5723632..3b02840 100644 --- a/deps/update_test.go +++ b/deps/update_test.go @@ -86,7 +86,7 @@ require github.com/acme/direct v1.2.3 } } -func TestUpdateCandidateMatchesMatchItemExpression(t *testing.T) { +func TestUpdateCandidateMatchesMatchItemFilter(t *testing.T) { candidates := []UpdateCandidate{ {Manager: ManagerGo, Name: "github.com/acme/lib", Current: "v1.0.0", File: "/work/flanksource/app/go.mod"}, {Manager: ManagerGo, Name: "github.com/flanksource/lib", Current: "v1.0.0", File: "/work/other/go.mod"}, @@ -160,14 +160,14 @@ func TestUpdateDryRunBuildsPackageManagerCommand(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - DryRun: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(choice UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.4.0", true }, }) @@ -203,10 +203,10 @@ func TestUpdateSkipsCandidatesWithoutChangesBeforePrompt(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - DryRun: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, SelectCandidates: func([]UpdateChoice) ([]UpdateChoice, bool) { t.Fatal("no-change candidates must not be shown for selection") return nil, false @@ -233,15 +233,15 @@ func TestUpdateCheckListsUpdatesWithoutPrompting(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - Check: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + Check: true, + Runner: runner, SelectCandidates: func([]UpdateChoice) ([]UpdateChoice, bool) { t.Fatal("--check must not prompt for dependency selection") return nil, false }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { t.Fatal("--check must not prompt for a version") return "", false }, @@ -353,7 +353,7 @@ func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { plans, err := Update(context.Background(), ".", UpdateOptions{ Managers: []Manager{ManagerImage}, - Expression: []string{"*nginx*"}, + Filters: []string{"*nginx*"}, DryRun: true, ImageResolver: fakeImageVersionResolver{"nginx": []string{"1.27.0", "1.25.3"}}, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { @@ -362,7 +362,7 @@ func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { } return choices, true }, - SelectVersion: func(choice UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.27.0", true }, }) @@ -397,13 +397,13 @@ func TestUpdateStagesChangedFilesAfterWrite(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + Runner: runner, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.4.0", true }, }) @@ -449,14 +449,14 @@ func TestUpdateDryRunDoesNotStage(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - DryRun: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.4.0", true }, }) @@ -501,12 +501,12 @@ func TestUpdateImageStagesEditedManifest(t *testing.T) { plans, err := Update(context.Background(), ".", UpdateOptions{ Managers: []Manager{ManagerImage}, - Expression: []string{"*nginx*"}, + Filters: []string{"*nginx*"}, ImageResolver: fakeImageVersionResolver{"nginx": []string{"1.27.0", "1.25.3"}}, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.27.0", true }, }) @@ -582,8 +582,8 @@ func TestUpdateCommandForManagers(t *testing.T) { func TestUpdateRejectsUnsupportedManagers(t *testing.T) { _, err := Update(context.Background(), t.TempDir(), UpdateOptions{ - Managers: []Manager{ManagerMaven}, - Expression: []string{"*"}, + Managers: []Manager{ManagerMaven}, + Filters: []string{"*"}, }) if err == nil || !strings.Contains(err.Error(), "unsupported manager") { t.Fatalf("expected unsupported manager error, got %v", err) From b233e8f9712cbba898766b4de029916c0eba8c49 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Fri, 7 Aug 2026 10:42:35 +0300 Subject: [PATCH 09/13] chore: remove grite --- .grite/export.json | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 .grite/export.json diff --git a/.grite/export.json b/.grite/export.json deleted file mode 100644 index f500e0b..0000000 --- a/.grite/export.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "meta": { - "schema_version": 1, - "generated_ts": 1782230638451, - "event_count": 2 - }, - "issues": [ - { - "issue_id": "a701ab0f379f53c67a4006a29274b673", - "title": "Fix repomap update", - "state": "closed", - "labels": [ - "priority:medium", - "session:0eee6b92-6966-4968-93a5-486b87b18e25" - ], - "assignees": [], - "created_ts": 1782228570737, - "updated_ts": 1782230638424, - "comment_count": 1 - } - ], - "events": [ - { - "event_id": "eebcf55cb0fa3528523dd496c0982c02289e77beb9a4fbf2be52ebee84108d22", - "issue_id": "a701ab0f379f53c67a4006a29274b673", - "actor": "b1f0880938c36f0a054030e48c354418", - "ts_unix_ms": 1782230638397, - "kind": { - "StateChanged": { - "state": "closed" - } - } - }, - { - "event_id": "0654b3d43459dedc11799a46e7ba51943d4ea07f3ae0379f8e6d9d5a9c11aa90", - "issue_id": "a701ab0f379f53c67a4006a29274b673", - "actor": "b1f0880938c36f0a054030e48c354418", - "ts_unix_ms": 1782230638424, - "kind": { - "LabelRemoved": { - "label": "status:in_progress" - } - } - } - ] -} \ No newline at end of file From 4ebcc91ccbba46085fe924c650c145a9ef742d58 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Fri, 7 Aug 2026 10:42:55 +0300 Subject: [PATCH 10/13] feat(deps): add cache warming for Go, npm, and pnpm dependencies Add cache warming for CI, sandbox, and air-gapped environments. Use real package managers to populate shared caches, optionally compile dependencies, and verify offline readiness while preserving partial results and actionable command errors. --- README.md | 36 ++++ cmd/repomap/cache_warm.go | 97 ++++++++++ cmd/repomap/cache_warm_test.go | 114 +++++++++++ deps/cachewarm.go | 330 ++++++++++++++++++++++++++++++++ deps/cachewarm_plan.go | 90 +++++++++ deps/cachewarm_test.go | 272 ++++++++++++++++++++++++++ deps/manager/gomod/warm.go | 95 +++++++++ deps/manager/gomod/warm_test.go | 159 +++++++++++++++ deps/manager/node/warm.go | 83 ++++++++ deps/manager/node/warm_test.go | 127 ++++++++++++ deps/manager/npm/warm.go | 28 +++ deps/manager/npm/warm_test.go | 69 +++++++ deps/manager/pnpm/warm.go | 66 +++++++ deps/manager/pnpm/warm_test.go | 118 ++++++++++++ deps/manifest/command.go | 46 +++++ deps/manifest/manager.go | 17 ++ deps/manifest/warm.go | 82 ++++++++ deps/model.go | 26 ++- deps/runner.go | 46 +---- deps/update_test.go | 18 ++ 20 files changed, 1874 insertions(+), 45 deletions(-) create mode 100644 cmd/repomap/cache_warm.go create mode 100644 cmd/repomap/cache_warm_test.go create mode 100644 deps/cachewarm.go create mode 100644 deps/cachewarm_plan.go create mode 100644 deps/cachewarm_test.go create mode 100644 deps/manager/gomod/warm.go create mode 100644 deps/manager/gomod/warm_test.go create mode 100644 deps/manager/node/warm.go create mode 100644 deps/manager/node/warm_test.go create mode 100644 deps/manager/npm/warm.go create mode 100644 deps/manager/npm/warm_test.go create mode 100644 deps/manager/pnpm/warm.go create mode 100644 deps/manager/pnpm/warm_test.go create mode 100644 deps/manifest/command.go create mode 100644 deps/manifest/manager.go create mode 100644 deps/manifest/warm.go diff --git a/README.md b/README.md index 0b8fd6b..c3333ae 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,42 @@ lockfile content without running package-manager commands. It prints direct dependencies by default; use `--depth 0` for the full graph available from the local files. +### `cache-warm` + +Prime the local package caches for dependencies this machine has not checked out. + +```bash +# Download a module and its full transitive closure into GOMODCACHE +repomap cache-warm go github.com/flanksource/clicky@v1.21.14 + +# Take the current version, compile every package, and prove it works offline +repomap cache-warm go github.com/flanksource/commons --build --verify + +# Warm several npm packages into the pnpm store +repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0 +``` + +For each `name@version` spec, repomap creates a throwaway single-dependency +project in a temporary directory, drives the real package manager to download the +closure into the machine's shared cache, then deletes the project. Nothing in the +working tree is touched; what persists is the warmed cache (`GOMODCACHE`, the pnpm +store, or the npm cache). Omit the version to take whatever the manager considers +current — the concrete resolved version is reported back, including Go +pseudo-versions. + +`--build` goes further than downloading. For Go it compiles every package in the +module so `GOCACHE` holds build artifacts and not just source. For npm and pnpm it +lets dependency lifecycle scripts run so native addons are compiled; it does not +run the package's own build script. + +`--verify` proves the result rather than assuming it, replaying the work with the +network disabled (`GOPROXY=off`, or an `--offline` install against a frozen +lockfile), so a cache that could not actually build offline fails loudly. + +Supported managers are `go`, `npm`, and `pnpm`. This is aimed at CI images, +sandboxes, and air-gapped builds, where a later build must succeed with no network +access. + ### `version` Print version, commit hash, build date, and Go version. diff --git a/cmd/repomap/cache_warm.go b/cmd/repomap/cache_warm.go new file mode 100644 index 0000000..4d83c81 --- /dev/null +++ b/cmd/repomap/cache_warm.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + depgraph "github.com/flanksource/repomap/deps" +) + +type CacheWarmOptions struct { + // No default: tag — a positional field carrying one has its positional values + // silently discarded. Emptiness is validated in runCacheWarm instead. + Args []string `json:"args" args:"true" help:"Package manager (go, npm, pnpm) followed by one or more name@version specs"` + Build bool `json:"build,omitempty" flag:"build" help:"Compile every package after downloading (Go) or run dependency lifecycle and native builds (npm, pnpm)"` + Verify bool `json:"verify,omitempty" flag:"verify" help:"Replay the warm with the network disabled to prove the cache is complete"` +} + +func (opts CacheWarmOptions) GetName() string { return "cache-warm ..." } + +func (opts CacheWarmOptions) Help() api.Text { + return clicky.Text(`Prime the local package caches for dependencies this machine has not checked out. + +For each name@version spec, repomap creates a throwaway single-dependency +project in a temporary directory, drives the real package manager to download +the dependency's full transitive closure into the machine's shared cache, then +deletes the project. Nothing in the working tree is touched; what persists is +the warmed cache (GOMODCACHE, the pnpm store, or the npm cache). + +Omit the version to take whatever the manager considers current. The concrete +resolved version is reported back, including Go pseudo-versions. + +Use --build to go further than downloading. For Go it compiles every package in +the module so GOCACHE holds the build artifacts, not just the source. For npm and +pnpm it lets dependency lifecycle scripts run so native addons are compiled. It +does not run the package's own build script. + +Use --verify to prove the result rather than assume it: the work is replayed with +the network disabled (GOPROXY=off, or an --offline install against a frozen +lockfile), so a cache that could not actually build offline fails loudly. + +This is aimed at CI images, sandboxes, and air-gapped builds, where a later build +must succeed with no network access. + +EXAMPLES: + repomap cache-warm go github.com/flanksource/clicky@v1.21.14 + repomap cache-warm go github.com/flanksource/commons --build --verify + repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0 + repomap cache-warm npm @flanksource/icons@1.0.0 --verify + repomap cache-warm go github.com/flanksource/clicky@v1.21.14 --json`) +} + +func init() { + cmd := clicky.AddNamedCommandWithContext("cache-warm", rootCmd, CacheWarmOptions{}, runCacheWarm) + cmd.Short = "Warm the Go, npm, or pnpm cache for a dependency and optionally build it" +} + +func runCacheWarm(ctx context.Context, opts CacheWarmOptions) (any, error) { + manager, specs, err := parseCacheWarmArgs(opts.Args) + if err != nil { + return nil, err + } + results, err := depgraph.WarmCache(ctx, depgraph.WarmOptions{ + Manager: manager, + Specs: specs, + Build: opts.Build, + Verify: opts.Verify, + }) + // Returned as a slice rather than api.NewTableFrom so --json keeps the full + // WarmResult — per-step commands, durations, and errors, which is what a CI + // debugging session needs. Pretty output still renders as a table via + // WarmResult's Columns/Row. The error is returned alongside the results so a + // partial run still reports which specs succeeded. + return results, err +} + +// parseCacheWarmArgs splits the positional arguments into the manager and its +// specs. Only managers repomap can actually warm are accepted; maven and gradle +// are scan-only, and image/helm are not package caches. +func parseCacheWarmArgs(args []string) (depgraph.Manager, []string, error) { + if len(args) == 0 { + return "", nil, fmt.Errorf("expected a package manager (go, npm, or pnpm) followed by one or more name@version specs") + } + manager := depgraph.Manager(strings.ToLower(strings.TrimSpace(args[0]))) + switch manager { + case depgraph.ManagerGo, depgraph.ManagerNPM, depgraph.ManagerPNPM: + default: + return "", nil, fmt.Errorf("cache warming does not support %q (expected go, npm, or pnpm)", args[0]) + } + specs := splitCommaArgs(args[1:]) + if len(specs) == 0 { + return "", nil, fmt.Errorf("expected at least one name@version spec to warm for %s", manager) + } + return manager, specs, nil +} diff --git a/cmd/repomap/cache_warm_test.go b/cmd/repomap/cache_warm_test.go new file mode 100644 index 0000000..170f03a --- /dev/null +++ b/cmd/repomap/cache_warm_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/flanksource/clicky" + depgraph "github.com/flanksource/repomap/deps" + "github.com/spf13/cobra" +) + +// runCacheWarmArgs parses argv through a real cobra tree with a stub handler, so +// the assertions cover clicky's struct-tag binding rather than a hand-built +// options struct. +func runCacheWarmArgs(t *testing.T, argv ...string) (CacheWarmOptions, error) { + t.Helper() + var got CacheWarmOptions + root := &cobra.Command{Use: "test"} + clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format") + clicky.AddNamedCommandWithContext("cache-warm", root, CacheWarmOptions{}, func(_ context.Context, opts CacheWarmOptions) (any, error) { + got = opts + return nil, nil + }) + root.SetArgs(argv) + return got, root.Execute() +} + +// The positional args field must carry no default: tag, or clicky silently drops +// the positional values. +func TestCacheWarmBindsPositionalArgsAndFlags(t *testing.T) { + got, err := runCacheWarmArgs(t, "cache-warm", "go", "github.com/acme/lib@v1.2.3", "left-pad@1.3.0", "--build", "--verify") + if err != nil { + t.Fatal(err) + } + want := []string{"go", "github.com/acme/lib@v1.2.3", "left-pad@1.3.0"} + if strings.Join(got.Args, ",") != strings.Join(want, ",") { + t.Fatalf("Args = %v, want %v", got.Args, want) + } + if !got.Build || !got.Verify { + t.Fatalf("Build = %v, Verify = %v, want both true", got.Build, got.Verify) + } +} + +func TestCacheWarmFlagsDefaultOff(t *testing.T) { + got, err := runCacheWarmArgs(t, "cache-warm", "go", "github.com/acme/lib@v1.2.3") + if err != nil { + t.Fatal(err) + } + if got.Build || got.Verify { + t.Fatalf("Build = %v, Verify = %v, want both false", got.Build, got.Verify) + } +} + +func TestParseCacheWarmArgs(t *testing.T) { + cases := []struct { + name string + args []string + wantManager depgraph.Manager + wantSpecs []string + wantErr string + }{ + { + name: "single spec", args: []string{"go", "github.com/acme/lib@v1.2.3"}, + wantManager: depgraph.ManagerGo, wantSpecs: []string{"github.com/acme/lib@v1.2.3"}, + }, + { + name: "several specs", args: []string{"pnpm", "left-pad@1.3.0", "@scope/pkg@2.0.0"}, + wantManager: depgraph.ManagerPNPM, wantSpecs: []string{"left-pad@1.3.0", "@scope/pkg@2.0.0"}, + }, + { + name: "manager casing is normalised", args: []string{"NPM", "left-pad@1.3.0"}, + wantManager: depgraph.ManagerNPM, wantSpecs: []string{"left-pad@1.3.0"}, + }, + {name: "no args", args: nil, wantErr: "manager"}, + {name: "manager but no spec", args: []string{"go"}, wantErr: "spec"}, + // Managers repomap can scan but cannot warm must be rejected by name. + {name: "unwarmable manager", args: []string{"maven", "org.acme:lib@1.0.0"}, wantErr: "maven"}, + {name: "unknown manager", args: []string{"cargo", "serde@1.0.0"}, wantErr: "cargo"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + manager, specs, err := parseCacheWarmArgs(tc.args) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("expected an error mentioning %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error %q should mention %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if manager != tc.wantManager { + t.Errorf("manager = %q, want %q", manager, tc.wantManager) + } + if strings.Join(specs, ",") != strings.Join(tc.wantSpecs, ",") { + t.Errorf("specs = %v, want %v", specs, tc.wantSpecs) + } + }) + } +} + +// defaultToScan rewrites anything it does not recognise into `scan ...`, so a +// misregistered name would turn this command into a silent repo scan. +func TestCacheWarmIsNotRewrittenToScan(t *testing.T) { + argv := []string{"cache-warm", "go", "github.com/acme/lib@v1.2.3"} + got := defaultToScan(argv) + if strings.Join(got, " ") != strings.Join(argv, " ") { + t.Fatalf("defaultToScan(%v) = %v, want it unchanged", argv, got) + } +} diff --git a/deps/cachewarm.go b/deps/cachewarm.go new file mode 100644 index 0000000..9b9b128 --- /dev/null +++ b/deps/cachewarm.go @@ -0,0 +1,330 @@ +package deps + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/clicky/task" + flanksourceContext "github.com/flanksource/commons/context" + "github.com/flanksource/repomap/deps/manager/gomod" + "github.com/flanksource/repomap/deps/manager/npm" + "github.com/flanksource/repomap/deps/manager/pnpm" + "github.com/flanksource/repomap/deps/manifest" +) + +// cacheWarmConcurrency bounds how many specs warm at once. GOMODCACHE and the +// pnpm store are both lock-protected against concurrent writers, so the limit is +// about not saturating the network rather than correctness. +const cacheWarmConcurrency = 4 + +// warmers is the single place an ecosystem is wired into cache warming. +var warmers = map[Manager]manifest.Warmer{ + ManagerGo: gomod.Warmer{}, + ManagerNPM: npm.Warmer{}, + ManagerPNPM: pnpm.Warmer{}, +} + +type WarmOptions struct { + Manager Manager + Specs []string + // Build compiles every package of the target (Go) or lets dependency + // lifecycle scripts run so native addons are built (npm, pnpm). + Build bool + // Verify replays the work with the network disabled, turning "a download ran" + // into "this cache can build offline". + Verify bool + Runner CommandRunner +} + +type WarmStep struct { + Name string `json:"name"` + Command string `json:"command"` + Duration time.Duration `json:"duration"` + Error string `json:"error,omitempty"` +} + +type WarmResult struct { + Manager Manager `json:"manager"` + // Spec is the requested name@version; Version is what the manager resolved. + Spec string `json:"spec"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Packages int `json:"packages,omitempty"` + Cache string `json:"cache,omitempty"` + Built bool `json:"built,omitempty"` + Verified bool `json:"verified,omitempty"` + Steps []WarmStep `json:"steps,omitempty"` + Error string `json:"error,omitempty"` + // SummaryError records a failure to read back what was warmed. The cache is + // still warm when this is set, so it does not fail the spec — but it is + // reported rather than swallowed. + SummaryError string `json:"summary_error,omitempty"` +} + +func (r WarmResult) failed() bool { return r.Error != "" } + +// WarmCache downloads each spec's full dependency closure into the machine's +// shared package cache, using a throwaway project per spec so nothing in the +// user's working tree is touched. +func WarmCache(ctx context.Context, opts WarmOptions) ([]WarmResult, error) { + warmer, ok := warmers[opts.Manager] + if !ok { + return nil, fmt.Errorf("cache warming does not support manager %q (expected go, npm, or pnpm)", opts.Manager) + } + if len(opts.Specs) == 0 { + return nil, fmt.Errorf("cache warming needs at least one name@version spec") + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + // Only preflight when we are really going to exec: an injected runner is a + // test double or a recorder, and need not correspond to a binary on PATH. + if _, err := exec.LookPath(warmer.Binary()); err != nil { + return nil, fmt.Errorf("%s not found on PATH, which is required to warm %s caches: %w", warmer.Binary(), opts.Manager, err) + } + } + + results := make([]WarmResult, len(opts.Specs)) + group := task.StartGroup[int]("Warming package caches", task.WithConcurrency(cacheWarmConcurrency)) + for i, spec := range opts.Specs { + idx, spec := i, spec + group.Add(fmt.Sprintf("%s %s", opts.Manager, spec), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { + results[idx] = warmSpec(ctx, warmer, runner, spec, opts, tk) + switch { + case results[idx].failed(): + tk.Errorf("%s", results[idx].Error) + tk.Failed() + case results[idx].SummaryError != "": + tk.Warnf("%s", results[idx].SummaryError) + tk.Warning() + default: + tk.Success() + } + return idx, nil + }) + } + _, _ = group.GetResults() + + // Each failure carries its own command and the tool's stderr. They are folded + // into the returned error rather than left on the results, because the scratch + // directory is already gone and a caller that only prints the error would + // otherwise have nothing to reproduce from. + var failed []string + for _, result := range results { + if result.failed() { + failed = append(failed, fmt.Sprintf("%s: %s", result.Spec, result.Error)) + } + } + if len(failed) > 0 { + return results, fmt.Errorf("failed to warm %d of %d specs:\n %s", len(failed), len(results), strings.Join(failed, "\n ")) + } + return results, nil +} + +func warmSpec(ctx context.Context, warmer manifest.Warmer, runner CommandRunner, spec string, opts WarmOptions, tk *task.Task) WarmResult { + result := WarmResult{Manager: warmer.Manager(), Spec: spec} + name, version, err := parseWarmSpec(spec) + if err != nil { + result.Error = err.Error() + return result + } + result.Name = name + + dir, err := os.MkdirTemp("", "repomap-cache-warm-*") + if err != nil { + result.Error = err.Error() + return result + } + // The scratch project is disposable: the durable result is what landed in the + // module cache or package store. + defer func() { _ = os.RemoveAll(dir) }() + + probe, err := runProbe(ctx, warmer, runner, dir) + if err != nil { + result.Error = err.Error() + return result + } + steps, err := warmer.Steps(manifest.WarmRequest{ + Dir: dir, + Name: name, + Version: version, + Build: opts.Build, + Verify: opts.Verify, + }, probe) + if err != nil { + result.Error = err.Error() + return result + } + + tk.SetProgress(0, len(steps)) + for i, step := range steps { + tk.Infof("%s", step.Name) + started := time.Now() + err := runWarmStep(ctx, runner, dir, step) + record := WarmStep{Name: step.Name, Command: step.Detail(), Duration: time.Since(started)} + if err != nil { + record.Error = err.Error() + result.Steps = append(result.Steps, record) + result.Error = err.Error() + return result + } + result.Steps = append(result.Steps, record) + tk.SetProgress(i+1, len(steps)) + } + + result.Built = opts.Build + result.Verified = opts.Verify + summarizeWarm(ctx, &result, dir, runner) + return result +} + +// runProbe executes a manager's optional version probe inside the scratch dir and +// returns its trimmed stdout, which Steps consumes as an ordinary input. +func runProbe(ctx context.Context, warmer manifest.Warmer, runner CommandRunner, dir string) (string, error) { + cmd := warmer.Probe() + if cmd == nil { + return "", nil + } + cmd.Dir = dir + result, err := runner.Run(ctx, *cmd) + if err != nil { + return "", commandError(*cmd, result, err) + } + return strings.TrimSpace(result.Stdout), nil +} + +func runWarmStep(ctx context.Context, runner CommandRunner, dir string, step manifest.Step) error { + switch step.Kind { + case manifest.StepWrite: + return os.WriteFile(filepath.Join(dir, step.Path), step.Content, 0o600) + case manifest.StepRemove: + return os.RemoveAll(filepath.Join(dir, step.Path)) + case manifest.StepExec: + result, err := runner.Run(ctx, step.Command) + if err != nil { + return commandError(step.Command, result, err) + } + return nil + default: + return fmt.Errorf("unknown warm step kind %q for step %q", step.Kind, step.Name) + } +} + +// commandError names the exact invocation and the tool's own diagnostics. The +// scratch dir is gone by the time a caller sees this, so the message is the only +// reproduction handle there is. +func commandError(cmd manifest.Command, result CommandResult, err error) error { + if stderr := strings.TrimSpace(result.Stderr); stderr != "" { + return fmt.Errorf("%s: %w: %s", cmd.String(), err, stderr) + } + return fmt.Errorf("%s: %w", cmd.String(), err) +} + +// summarizeWarm reports what actually landed by reading the warmed scratch +// project back through repomap's own manifest resolvers, rather than re-parsing +// go.mod or a lockfile here. The manager did the resolving, so this is also where +// the concrete version (including a Go pseudo-version) is discovered. +// +// It calls discoverOffline/resolveManifest rather than Scan deliberately: Scan +// renders its own task group, which would surface read-back warnings about a +// temporary directory the user never asked about. +func summarizeWarm(ctx context.Context, result *WarmResult, dir string, runner CommandRunner) { + result.Cache = warmCachePath(ctx, result.Manager, runner, dir) + + projects, _, err := discoverOffline(dir, []Manager{result.Manager}) + if err != nil { + result.SummaryError = fmt.Sprintf("could not read back the warmed project: %s", err) + return + } + if len(projects) == 0 { + result.SummaryError = fmt.Sprintf("no %s manifest was produced in the scratch project", result.Manager) + return + } + // MaxDepth 1 keeps the read-back offline — it parses the manifest and lockfile + // the warm just wrote instead of shelling out again. + root, _, err := resolveManifest(ctx, projects[0], Options{MaxDepth: 1, IncludeIndirect: true}) + if err != nil { + result.SummaryError = fmt.Sprintf("could not read back the warmed project: %s", err) + return + } + result.Packages, result.Version = walkWarmed(root, result.Name) +} + +// walkWarmed counts every resolved dependency below root and finds the target's +// version. It walks rather than reading root.Children directly because pnpm nests +// dependencies under a synthetic importer node while go.mod lists them flat. +// +// Only nodes carrying a version are counted, which is what distinguishes a real +// package from pnpm's importer (and from the project root itself). +func walkWarmed(root *Node, name string) (count int, version string) { + seen := map[*Node]bool{} + stack := []*Node{root} + for len(stack) > 0 { + node := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if node == nil || seen[node] { + continue + } + seen[node] = true + for _, child := range node.Children { + if child.Version != "" { + count++ + if version == "" && child.Name == name { + version = child.Version + } + } + stack = append(stack, child) + } + } + return count, version +} + +// warmCachePath asks the manager where its cache lives, so the output names the +// directory that grew. A failure here is not worth reporting: it costs only the +// display of a path. +func warmCachePath(ctx context.Context, manager Manager, runner CommandRunner, dir string) string { + var cmd manifest.Command + switch manager { + case ManagerGo: + cmd = manifest.Command{Dir: dir, Name: "go", Args: []string{"env", "GOMODCACHE"}} + case ManagerPNPM: + cmd = manifest.Command{Dir: dir, Name: "pnpm", Args: []string{"store", "path"}} + case ManagerNPM: + cmd = manifest.Command{Dir: dir, Name: "npm", Args: []string{"config", "get", "cache"}} + default: + return "" + } + result, err := runner.Run(ctx, cmd) + if err != nil { + return "" + } + return strings.TrimSpace(result.Stdout) +} + +// parseWarmSpec splits "name@version". The split uses the last @ at a non-zero +// index so a scoped npm name such as @scope/pkg keeps its leading @. An omitted +// version becomes "latest" and the manager decides what that means; the concrete +// version is read back after warming. +func parseWarmSpec(spec string) (name, version string, err error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return "", "", fmt.Errorf("empty dependency spec: expected name@version") + } + at := strings.LastIndex(spec, "@") + if at <= 0 { + if spec == "@" { + return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) + } + return spec, "latest", nil + } + name, version = spec[:at], spec[at+1:] + if name == "" || version == "" { + return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) + } + return name, version, nil +} diff --git a/deps/cachewarm_plan.go b/deps/cachewarm_plan.go new file mode 100644 index 0000000..4847bac --- /dev/null +++ b/deps/cachewarm_plan.go @@ -0,0 +1,90 @@ +package deps + +import ( + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r WarmResult) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("[%s] %s", r.Manager, r.Name), managerStyle(r.Manager)) + if version := r.displayVersion(); version != "" { + t = t.Space().Append(version, "font-mono text-muted") + } + if r.Error != "" { + return t.Space().Append("failed: "+r.Error, "text-red-600") + } + if r.Packages > 0 { + t = t.Space().Append(fmt.Sprintf("%d packages", r.Packages), "text-muted") + } + t = t.Space().Append(strings.Join(r.badges(), " "), "text-green-600") + if r.Cache != "" { + t = t.Space().Append(r.Cache, "font-mono text-muted") + } + if r.SummaryError != "" { + t = t.Space().Append("("+r.SummaryError+")", "text-yellow-600") + } + return t +} + +func (WarmResult) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("manager").Label("Manager").Build(), + api.Column("dependency").Label("Dependency").Build(), + api.Column("version").Label("Version").Build(), + api.Column("packages").Label("Packages").Build(), + api.Column("status").Label("Status").Build(), + api.Column("cache").Label("Cache").Build(), + } +} + +func (r WarmResult) Row() map[string]any { + row := map[string]any{ + "manager": clicky.Text(string(r.Manager), managerStyle(r.Manager)), + "dependency": clicky.Text(r.Name, "font-bold text-cyan-600"), + "version": clicky.Text(r.displayVersion(), "font-mono"), + "cache": clicky.Text(r.Cache, "font-mono text-muted"), + } + if r.Packages > 0 { + row["packages"] = clicky.Text(fmt.Sprintf("%d", r.Packages), "text-muted") + } else { + row["packages"] = clicky.Text("") + } + if r.Error != "" { + row["status"] = clicky.Text("failed: "+r.Error, "text-red-600") + return row + } + status := clicky.Text(strings.Join(r.badges(), " "), "text-green-600") + if r.SummaryError != "" { + status = status.Append(" ("+r.SummaryError+")", "text-yellow-600") + } + row["status"] = status + return row +} + +// badges names what the warm actually did, so "warmed" is never confused with +// "compiled" or with "proven to work offline". +func (r WarmResult) badges() []string { + badges := []string{"warmed"} + if r.Built { + badges = append(badges, "built") + } + if r.Verified { + badges = append(badges, "verified offline") + } + return badges +} + +// displayVersion prefers the version the manager resolved, falling back to what +// was requested when the read-back could not determine it. +func (r WarmResult) displayVersion() string { + if r.Version != "" { + return r.Version + } + if _, version, err := parseWarmSpec(r.Spec); err == nil { + return version + } + return "" +} diff --git a/deps/cachewarm_test.go b/deps/cachewarm_test.go new file mode 100644 index 0000000..a1c9ca8 --- /dev/null +++ b/deps/cachewarm_test.go @@ -0,0 +1,272 @@ +package deps + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestParseWarmSpec(t *testing.T) { + cases := []struct { + spec string + wantName string + wantVersion string + wantErr bool + }{ + {spec: "github.com/acme/lib@v1.2.3", wantName: "github.com/acme/lib", wantVersion: "v1.2.3"}, + // No version means "whatever the manager considers current"; the concrete + // version is read back after warming. + {spec: "github.com/acme/lib", wantName: "github.com/acme/lib", wantVersion: "latest"}, + {spec: "left-pad@1.3.0", wantName: "left-pad", wantVersion: "1.3.0"}, + {spec: "left-pad@^1.3.0", wantName: "left-pad", wantVersion: "^1.3.0"}, + {spec: "left-pad@latest", wantName: "left-pad", wantVersion: "latest"}, + // A scoped npm name leads with @, so splitting must use the last @ and + // ignore one at index 0. + {spec: "@scope/pkg@1.0.0", wantName: "@scope/pkg", wantVersion: "1.0.0"}, + {spec: "@scope/pkg", wantName: "@scope/pkg", wantVersion: "latest"}, + // A Go branch or commit reference must survive untouched. + {spec: "github.com/acme/lib@main", wantName: "github.com/acme/lib", wantVersion: "main"}, + {spec: "", wantErr: true}, + {spec: " ", wantErr: true}, + {spec: "left-pad@", wantErr: true}, + {spec: "@", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.spec, func(t *testing.T) { + name, version, err := parseWarmSpec(tc.spec) + if tc.wantErr { + if err == nil { + t.Fatalf("parseWarmSpec(%q) = (%q, %q), want an error", tc.spec, name, version) + } + return + } + if err != nil { + t.Fatal(err) + } + if name != tc.wantName || version != tc.wantVersion { + t.Fatalf("parseWarmSpec(%q) = (%q, %q), want (%q, %q)", tc.spec, name, version, tc.wantName, tc.wantVersion) + } + }) + } +} + +// pnpm nests dependencies under a synthetic importer node, so counting +// root.Children would report the importer instead of the packages, and the +// version lookup would miss the target entirely. +func TestWalkWarmedSkipsSyntheticNodes(t *testing.T) { + leftPad := NewNode(ManagerPNPM, "left-pad", "1.3.0") + nested := NewNode(ManagerPNPM, "big-int", "1.0.2") + leftPad.Children = []*Node{nested} + importer := NewNode(ManagerPNPM, ".", "") + importer.Source = "importer" + importer.Children = []*Node{leftPad} + root := NewNode(ManagerPNPM, "scratch", "") + root.Children = []*Node{importer} + + count, version := walkWarmed(root, "left-pad") + if count != 2 { + t.Errorf("count = %d, want 2: the importer carries no version and is not a package", count) + } + if version != "1.3.0" { + t.Errorf("version = %q, want 1.3.0: the target sits below the importer", version) + } +} + +func TestWalkWarmedFindsFlatGoRequires(t *testing.T) { + root := NewNode(ManagerGo, "repomap.local/cachewarm", "") + root.Children = []*Node{ + NewNode(ManagerGo, "rsc.io/quote", "v1.5.2"), + NewNode(ManagerGo, "rsc.io/sampler", "v1.3.0"), + } + count, version := walkWarmed(root, "rsc.io/quote") + if count != 2 || version != "v1.5.2" { + t.Fatalf("walkWarmed = (%d, %q), want (2, v1.5.2)", count, version) + } +} + +func TestWarmCacheRejectsUnsupportedManager(t *testing.T) { + for _, manager := range []Manager{ManagerMaven, ManagerGradle, ManagerHelm, ManagerImage, Manager("cargo")} { + _, err := WarmCache(context.Background(), WarmOptions{ + Manager: manager, + Specs: []string{"something@1.0.0"}, + Runner: &updateFakeRunner{}, + }) + if err == nil { + t.Errorf("manager %q: expected an error", manager) + continue + } + if !strings.Contains(err.Error(), "go, npm, or pnpm") { + t.Errorf("manager %q: error should list the supported managers, got %v", manager, err) + } + } +} + +func TestWarmCacheRequiresSpecs(t *testing.T) { + if _, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Runner: &updateFakeRunner{}, + }); err == nil { + t.Fatal("expected an error when no specs are given") + } +} + +func TestWarmCacheRunsTheGoStepsInOrder(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"github.com/acme/lib@v1.2.3"}, + Build: true, + Verify: true, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("want 1 result, got %d", len(results)) + } + result := results[0] + if result.Error != "" { + t.Fatalf("unexpected warm error: %s", result.Error) + } + if result.Name != "github.com/acme/lib" || result.Spec != "github.com/acme/lib@v1.2.3" { + t.Errorf("result identity = %q / %q", result.Name, result.Spec) + } + if !result.Built || !result.Verified { + t.Errorf("Built = %v, Verified = %v, want both true", result.Built, result.Verified) + } + steps := []string{ + "go mod init repomap.local/cachewarm", + "go get github.com/acme/lib/...@v1.2.3", + "go mod download all", + "go build github.com/acme/lib/...", + // The verify replay is the same build with GOPROXY=off, which the argv + // alone does not show; deps/manager/gomod pins the env. + "go build github.com/acme/lib/...", + } + // Reporting where the cache lives runs after the steps, not as one of them. + want := append(append([]string{}, steps...), "go env GOMODCACHE") + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } + if len(result.Steps) != len(steps) { + t.Errorf("recorded %d steps, want %d", len(result.Steps), len(steps)) + } +} + +// The scratch directory is deleted before the error surfaces, so the message is +// the only handle the user has for reproducing the failure. +func TestWarmCacheFailureAbortsRemainingStepsAndNamesTheCommand(t *testing.T) { + runner := &updateFakeRunner{ + succeedByDefault: true, + errors: map[string]error{ + "go get github.com/acme/lib/...@v1.2.3": errors.New("exit status 1"), + }, + responses: map[string]CommandResult{ + "go get github.com/acme/lib/...@v1.2.3": {Stderr: "module github.com/acme/lib: not found"}, + }, + } + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"github.com/acme/lib@v1.2.3"}, + Build: true, + Runner: runner, + }) + if err == nil { + t.Fatal("WarmCache should report an error when a spec fails, so the CLI exits non-zero") + } + if len(results) != 1 { + t.Fatalf("want 1 result, got %d", len(results)) + } + for _, want := range []string{"go get github.com/acme/lib/...@v1.2.3", "not found"} { + if !strings.Contains(results[0].Error, want) { + t.Errorf("result error %q should contain %q", results[0].Error, want) + } + // The returned error is what a caller that only prints err sees, and the + // scratch dir is gone by then, so the detail has to survive into it too. + if !strings.Contains(err.Error(), want) { + t.Errorf("returned error %q should contain %q", err, want) + } + } + if results[0].Built { + t.Error("Built should stay false when the warm failed") + } + // init ran, get failed, and download/build must not have been attempted. + want := []string{"go mod init repomap.local/cachewarm", "go get github.com/acme/lib/...@v1.2.3"} + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } +} + +// pnpm's probe output decides the build flags, so the orchestrator has to run it +// and feed the result into Steps. +func TestWarmCacheFeedsTheProbeIntoSteps(t *testing.T) { + runner := &updateFakeRunner{ + succeedByDefault: true, + responses: map[string]CommandResult{ + "pnpm --version": {Stdout: "10.7.0\n"}, + }, + } + if _, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerPNPM, + Specs: []string{"left-pad@1.3.0"}, + Build: true, + Runner: runner, + }); err != nil { + t.Fatal(err) + } + want := []string{ + "pnpm --version", + "pnpm install --config.dangerouslyAllowAllBuilds=true", + "pnpm store path", + } + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestWarmCacheWarmsEverySpec(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"github.com/acme/one@v1.0.0", "github.com/acme/two@v2.0.0"}, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if len(results) != 2 { + t.Fatalf("want 2 results, got %d", len(results)) + } + // Specs are warmed concurrently, so assert identity by position rather than + // relying on command ordering. + if results[0].Spec != "github.com/acme/one@v1.0.0" || results[1].Spec != "github.com/acme/two@v2.0.0" { + t.Fatalf("results should stay in spec order, got %q and %q", results[0].Spec, results[1].Spec) + } + for _, result := range results { + if result.Error != "" { + t.Errorf("spec %q failed: %s", result.Spec, result.Error) + } + } +} + +// A bad spec must be reported against that spec rather than aborting the run +// before the others are warmed. +func TestWarmCacheReportsABadSpecWithoutSkippingTheRest(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"left-pad@", "github.com/acme/lib@v1.0.0"}, + Runner: runner, + }) + if err == nil { + t.Fatal("expected a non-nil error because one spec is invalid") + } + if results[0].Error == "" { + t.Error("the invalid spec should carry an error") + } + if results[1].Error != "" { + t.Errorf("the valid spec should still have been warmed, got %q", results[1].Error) + } +} diff --git a/deps/manager/gomod/warm.go b/deps/manager/gomod/warm.go new file mode 100644 index 0000000..11ad1e3 --- /dev/null +++ b/deps/manager/gomod/warm.go @@ -0,0 +1,95 @@ +// Package gomod warms the Go module and build caches for a single module. +// +// It is named gomod rather than go because a directory named "go" reads badly at +// import sites. +package gomod + +import ( + "fmt" + + "github.com/flanksource/repomap/deps/manifest" +) + +// ModulePath is the synthetic module the scratch project declares. It is never +// published, but it must be a valid module path for `go mod init` to accept it. +const ModulePath = "repomap.local/cachewarm" + +// Warmer drives the go toolchain against a throwaway module that requires the +// target, populating GOMODCACHE and — with Build — GOCACHE. +type Warmer struct{} + +func (Warmer) Manager() manifest.Manager { return manifest.ManagerGo } + +func (Warmer) Binary() string { return "go" } + +// Probe returns nil: the go commands used here have been stable for many +// releases, so no runtime version check is needed. +func (Warmer) Probe() *manifest.Command { return nil } + +func (Warmer) Steps(req manifest.WarmRequest, _ string) ([]manifest.Step, error) { + switch { + case req.Dir == "": + return nil, fmt.Errorf("go warming needs a scratch directory") + case req.Name == "": + return nil, fmt.Errorf("go warming needs a module path") + case req.Version == "": + return nil, fmt.Errorf("go warming needs a version for %s", req.Name) + } + + // The /... package pattern, rather than the bare module, is what records the + // go.sum entries needed to *build* every package in the module. Resolving the + // module alone records only enough to reference it, which leaves a later + // offline build short of its dependencies. + packages := req.Name + "/..." + + steps := []manifest.Step{ + goStep("init", req, false, "mod", "init", ModulePath), + goStep("resolve", req, false, "get", packages+"@"+req.Version), + // The synthetic module imports nothing, so a bare `go mod download` would + // have no packages to work from. The `all` pattern materialises zips for + // the whole resolved graph, which is what makes a later build offline-able. + goStep("download", req, false, "mod", "download", "all"), + } + if req.Build { + steps = append(steps, goStep("build", req, false, "build", packages)) + } + if req.Verify { + steps = append(steps, verifyStep(req, packages)) + } + return steps, nil +} + +// verifyStep replays the most demanding work already done, with the proxy +// disabled so any cache miss is a hard failure rather than a silent refetch. +// With Build there are compiled packages to reproduce; without it, the strongest +// available claim is that every module zip is already resident. +func verifyStep(req manifest.WarmRequest, packages string) manifest.Step { + if req.Build { + return goStep("verify", req, true, "build", packages) + } + return goStep("verify", req, true, "mod", "download", "all") +} + +func goStep(name string, req manifest.WarmRequest, offline bool, args ...string) manifest.Step { + return manifest.Step{ + Kind: manifest.StepExec, + Name: name, + Command: manifest.Command{ + Dir: req.Dir, + Name: "go", + Args: args, + Env: goEnv(offline), + }, + } +} + +// goEnv pins the module mode explicitly. GOWORK=off matters most: without it a +// scratch dir that happens to sit inside a go.work tree silently joins that +// workspace and warms the wrong module graph. +func goEnv(offline bool) []string { + env := []string{"GOWORK=off", "GOFLAGS=-mod=mod"} + if offline { + env = append(env, "GOPROXY=off") + } + return env +} diff --git a/deps/manager/gomod/warm_test.go b/deps/manager/gomod/warm_test.go new file mode 100644 index 0000000..e0c350a --- /dev/null +++ b/deps/manager/gomod/warm_test.go @@ -0,0 +1,159 @@ +package gomod + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +// formatSteps renders each step as "name: argv [env]" so a single assertion pins +// the command sequence, the exact arguments, and the environment together. +func formatSteps(t *testing.T, steps []manifest.Step) []string { + t.Helper() + out := make([]string, 0, len(steps)) + for _, step := range steps { + if step.Kind != manifest.StepExec { + t.Fatalf("go warming should only produce exec steps, got %q for %q", step.Kind, step.Name) + } + out = append(out, step.Name+": "+step.Command.String()+" ["+strings.Join(step.Command.Env, " ")+"]") + } + return out +} + +func TestStepsPerFlagCombination(t *testing.T) { + const ( + online = "[GOWORK=off GOFLAGS=-mod=mod]" + offline = "[GOWORK=off GOFLAGS=-mod=mod GOPROXY=off]" + module = "github.com/acme/lib" + version = "v1.2.3" + ) + cases := []struct { + name string + build bool + verif bool + want []string + }{ + { + name: "download only", + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + }, + }, + { + name: "build compiles every package", + build: true, + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + "build: go build github.com/acme/lib/... " + online, + }, + }, + { + // Without --build there is nothing compiled to replay, so the offline + // proof is that every module zip is already resident. + name: "verify without build replays the download", + verif: true, + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + "verify: go mod download all " + offline, + }, + }, + { + name: "verify with build replays the build", + build: true, + verif: true, + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + "build: go build github.com/acme/lib/... " + online, + "verify: go build github.com/acme/lib/... " + offline, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", + Name: module, + Version: version, + Build: tc.build, + Verify: tc.verif, + }, "") + if err != nil { + t.Fatal(err) + } + got := formatSteps(t, steps) + if strings.Join(got, "\n") != strings.Join(tc.want, "\n") { + t.Fatalf("steps mismatch\n got: %s\nwant: %s", strings.Join(got, "\n "), strings.Join(tc.want, "\n ")) + } + }) + } +} + +// GOPROXY=off must never leak onto a warming step, or the warm would fail on a +// cold cache instead of populating it. +func TestOnlyVerifyStepDisablesTheProxy(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "github.com/acme/lib", Version: "v1.0.0", Build: true, Verify: true, + }, "") + if err != nil { + t.Fatal(err) + } + for _, step := range steps { + offline := strings.Contains(strings.Join(step.Command.Env, " "), "GOPROXY=off") + if offline != (step.Name == "verify") { + t.Errorf("step %q: GOPROXY=off present = %v, want %v", step.Name, offline, step.Name == "verify") + } + } +} + +// Every step must run inside the scratch project, otherwise go would resolve +// against whatever module happens to contain the process working directory. +func TestStepsRunInTheScratchDir(t *testing.T) { + const dir = "/tmp/repomap-cache-warm-xyz" + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: dir, Name: "github.com/acme/lib", Version: "v1.0.0", Build: true, Verify: true, + }, "") + if err != nil { + t.Fatal(err) + } + for _, step := range steps { + if step.Command.Dir != dir { + t.Errorf("step %q dir = %q, want %q", step.Name, step.Command.Dir, dir) + } + } +} + +func TestStepsRejectsIncompleteRequest(t *testing.T) { + cases := []struct { + name string + request manifest.WarmRequest + }{ + {name: "no module path", request: manifest.WarmRequest{Dir: "/scratch", Version: "v1.0.0"}}, + {name: "no version", request: manifest.WarmRequest{Dir: "/scratch", Name: "github.com/acme/lib"}}, + {name: "no dir", request: manifest.WarmRequest{Name: "github.com/acme/lib", Version: "v1.0.0"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := (Warmer{}).Steps(tc.request, ""); err == nil { + t.Fatal("expected an error for an incomplete request") + } + }) + } +} + +func TestWarmerIdentity(t *testing.T) { + if got := (Warmer{}).Manager(); got != manifest.ManagerGo { + t.Errorf("Manager() = %q, want %q", got, manifest.ManagerGo) + } + if probe := (Warmer{}).Probe(); probe != nil { + t.Errorf("Probe() = %v, want nil: go needs no runtime version check", probe) + } +} diff --git a/deps/manager/node/warm.go b/deps/manager/node/warm.go new file mode 100644 index 0000000..05b56e2 --- /dev/null +++ b/deps/manager/node/warm.go @@ -0,0 +1,83 @@ +// Package node holds the warming logic npm and pnpm share: generating the +// single-dependency package.json and assembling the install/verify step +// sequence. The npm and pnpm packages supply only the argv that differs. +package node + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/repomap/deps/manifest" +) + +// ProjectName is the synthetic package the scratch project declares. It is +// marked private so the registry client never treats it as publishable. +const ProjectName = "repomap-cache-warm" + +// Commands is how one node package manager spells the operations warming needs. +type Commands struct { + Binary string + // Install is the base argv that downloads and links the dependency. + Install []string + // IgnoreScripts suppresses dependency lifecycle scripts. It is applied unless + // the caller asked to build, since compiling native addons is the point of + // building. + IgnoreScripts string + // BuildArgs are appended when building. pnpm uses this for its version-gated + // lifecycle-script allowlist; npm needs nothing. + BuildArgs []string + // Offline is the full argv for the replay that proves the cache is complete. + Offline []string +} + +// Manifest renders the scratch package.json. This is the one place repomap +// generates a package manifest rather than editing an existing one. +func Manifest(name, version string) ([]byte, error) { + return json.MarshalIndent(struct { + Name string `json:"name"` + Version string `json:"version"` + Private bool `json:"private"` + Dependencies map[string]string `json:"dependencies"` + }{ + Name: ProjectName, + Version: "0.0.0", + Private: true, + Dependencies: map[string]string{name: version}, + }, "", " ") +} + +func Steps(req manifest.WarmRequest, cmds Commands) ([]manifest.Step, error) { + switch { + case req.Dir == "": + return nil, fmt.Errorf("%s warming needs a scratch directory", cmds.Binary) + case req.Name == "": + return nil, fmt.Errorf("%s warming needs a package name", cmds.Binary) + case req.Version == "": + return nil, fmt.Errorf("%s warming needs a version for %s", cmds.Binary, req.Name) + } + content, err := Manifest(req.Name, req.Version) + if err != nil { + return nil, err + } + + install := append([]string{}, cmds.Install...) + if req.Build { + install = append(install, cmds.BuildArgs...) + } else if cmds.IgnoreScripts != "" { + install = append(install, cmds.IgnoreScripts) + } + + steps := []manifest.Step{ + {Kind: manifest.StepWrite, Name: "manifest", Path: "package.json", Content: content}, + {Kind: manifest.StepExec, Name: "download", Command: manifest.Command{Dir: req.Dir, Name: cmds.Binary, Args: install}}, + } + if req.Verify { + // Installing over a populated node_modules is a no-op, so the tree has to + // go before the offline replay can prove the cache holds the packages. + steps = append(steps, + manifest.Step{Kind: manifest.StepRemove, Name: "clean", Path: "node_modules"}, + manifest.Step{Kind: manifest.StepExec, Name: "verify", Command: manifest.Command{Dir: req.Dir, Name: cmds.Binary, Args: cmds.Offline}}, + ) + } + return steps, nil +} diff --git a/deps/manager/node/warm_test.go b/deps/manager/node/warm_test.go new file mode 100644 index 0000000..542026a --- /dev/null +++ b/deps/manager/node/warm_test.go @@ -0,0 +1,127 @@ +package node + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +func TestManifestDeclaresExactlyOneDependency(t *testing.T) { + cases := []struct { + name string + pkg string + version string + want string + }{ + { + name: "plain package", pkg: "left-pad", version: "1.3.0", + want: `{ + "name": "repomap-cache-warm", + "version": "0.0.0", + "private": true, + "dependencies": { + "left-pad": "1.3.0" + } +}`, + }, + { + // A scoped name must survive verbatim as the dependency key. + name: "scoped package", pkg: "@flanksource/clicky-ui", version: "^2.1.0", + want: `{ + "name": "repomap-cache-warm", + "version": "0.0.0", + "private": true, + "dependencies": { + "@flanksource/clicky-ui": "^2.1.0" + } +}`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := Manifest(tc.pkg, tc.version) + if err != nil { + t.Fatal(err) + } + if string(got) != tc.want { + t.Fatalf("manifest mismatch\n got: %s\nwant: %s", got, tc.want) + } + }) + } +} + +func testCommands() Commands { + return Commands{ + Binary: "fakepm", + Install: []string{"install"}, + IgnoreScripts: "--ignore-scripts", + Offline: []string{"install", "--offline"}, + } +} + +func TestStepsWriteManifestBeforeInstalling(t *testing.T) { + steps, err := Steps(manifest.WarmRequest{Dir: "/scratch", Name: "left-pad", Version: "1.3.0"}, testCommands()) + if err != nil { + t.Fatal(err) + } + want := []string{ + "manifest: write package.json", + "download: exec fakepm install --ignore-scripts", + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestBuildDropsIgnoreScriptsAndAppendsBuildArgs(t *testing.T) { + cmds := testCommands() + cmds.BuildArgs = []string{"--allow-builds"} + steps, err := Steps(manifest.WarmRequest{Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: true}, cmds) + if err != nil { + t.Fatal(err) + } + want := []string{ + "manifest: write package.json", + "download: exec fakepm install --allow-builds", + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +// Reinstalling over a populated node_modules is a no-op, so without the removal +// the offline replay would pass without proving the cache holds anything. +func TestVerifyRemovesNodeModulesBeforeReplaying(t *testing.T) { + steps, err := Steps(manifest.WarmRequest{Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Verify: true}, testCommands()) + if err != nil { + t.Fatal(err) + } + want := []string{ + "manifest: write package.json", + "download: exec fakepm install --ignore-scripts", + "clean: remove node_modules", + "verify: exec fakepm install --offline", + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestStepsRejectsIncompleteRequest(t *testing.T) { + cases := []struct { + name string + request manifest.WarmRequest + }{ + {name: "no package name", request: manifest.WarmRequest{Dir: "/scratch", Version: "1.3.0"}}, + {name: "no version", request: manifest.WarmRequest{Dir: "/scratch", Name: "left-pad"}}, + {name: "no dir", request: manifest.WarmRequest{Name: "left-pad", Version: "1.3.0"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := Steps(tc.request, testCommands()); err == nil { + t.Fatal("expected an error for an incomplete request") + } + }) + } +} diff --git a/deps/manager/npm/warm.go b/deps/manager/npm/warm.go new file mode 100644 index 0000000..c0b487e --- /dev/null +++ b/deps/manager/npm/warm.go @@ -0,0 +1,28 @@ +// Package npm warms the npm cache for a single package. +package npm + +import ( + "github.com/flanksource/repomap/deps/manager/node" + "github.com/flanksource/repomap/deps/manifest" +) + +type Warmer struct{} + +func (Warmer) Manager() manifest.Manager { return manifest.ManagerNPM } + +func (Warmer) Binary() string { return "npm" } + +// Probe returns nil: npm has kept its lifecycle-script default, so no runtime +// version check is needed. +func (Warmer) Probe() *manifest.Command { return nil } + +func (Warmer) Steps(req manifest.WarmRequest, _ string) ([]manifest.Step, error) { + return node.Steps(req, node.Commands{ + Binary: "npm", + Install: []string{"install"}, + IgnoreScripts: "--ignore-scripts", + // ci rather than install: the lockfile the download step wrote makes it the + // stricter replay, and it refuses to reach the network for anything missing. + Offline: []string{"ci", "--offline"}, + }) +} diff --git a/deps/manager/npm/warm_test.go b/deps/manager/npm/warm_test.go new file mode 100644 index 0000000..7fc5f5f --- /dev/null +++ b/deps/manager/npm/warm_test.go @@ -0,0 +1,69 @@ +package npm + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +func TestSteps(t *testing.T) { + cases := []struct { + name string + build bool + verif bool + want []string + }{ + { + name: "download suppresses lifecycle scripts", + want: []string{ + "manifest: write package.json", + "download: exec npm install --ignore-scripts", + }, + }, + { + // Compiling native addons is the point of building, so the suppression + // has to come off. + name: "build allows lifecycle scripts", + build: true, + want: []string{ + "manifest: write package.json", + "download: exec npm install", + }, + }, + { + // npm ci over npm install: the lockfile written by the download step + // makes it the stricter replay. + name: "verify replays from the lockfile offline", + verif: true, + want: []string{ + "manifest: write package.json", + "download: exec npm install --ignore-scripts", + "clean: remove node_modules", + "verify: exec npm ci --offline", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: tc.build, Verify: tc.verif, + }, "") + if err != nil { + t.Fatal(err) + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(tc.want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, tc.want) + } + }) + } +} + +func TestWarmerIdentity(t *testing.T) { + if got := (Warmer{}).Manager(); got != manifest.ManagerNPM { + t.Errorf("Manager() = %q, want %q", got, manifest.ManagerNPM) + } + if probe := (Warmer{}).Probe(); probe != nil { + t.Errorf("Probe() = %v, want nil: npm needs no runtime version check", probe) + } +} diff --git a/deps/manager/pnpm/warm.go b/deps/manager/pnpm/warm.go new file mode 100644 index 0000000..c5ddc8f --- /dev/null +++ b/deps/manager/pnpm/warm.go @@ -0,0 +1,66 @@ +// Package pnpm warms the pnpm content-addressable store for a single package. +package pnpm + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/flanksource/repomap/deps/manager/node" + "github.com/flanksource/repomap/deps/manifest" +) + +// allowAllBuilds opts every dependency into running its lifecycle scripts. pnpm +// 10.6 stopped honouring them by default — dropping --ignore-scripts is no longer +// enough — and older pnpm does not recognise the flag, hence the version gate. +const allowAllBuilds = "--config.dangerouslyAllowAllBuilds=true" + +const ( + allowAllBuildsMajor = 10 + allowAllBuildsMinor = 6 +) + +type Warmer struct{} + +func (Warmer) Manager() manifest.Manager { return manifest.ManagerPNPM } + +func (Warmer) Binary() string { return "pnpm" } + +// Probe reports the pnpm version, which decides how --build has to ask for +// lifecycle scripts. Dir is left for the orchestrator to fill in. +func (Warmer) Probe() *manifest.Command { + return &manifest.Command{Name: "pnpm", Args: []string{"--version"}} +} + +func (Warmer) Steps(req manifest.WarmRequest, probe string) ([]manifest.Step, error) { + cmds := node.Commands{ + Binary: "pnpm", + Install: []string{"install"}, + IgnoreScripts: "--ignore-scripts", + Offline: []string{"install", "--offline", "--frozen-lockfile"}, + } + // The probe only matters for building; a plain warm must not depend on it. + if req.Build { + buildArgs, err := buildArgsFor(probe) + if err != nil { + return nil, err + } + cmds.BuildArgs = buildArgs + } + return node.Steps(req, cmds) +} + +func buildArgsFor(probe string) ([]string, error) { + if probe == "" { + return nil, fmt.Errorf("--build needs the pnpm version to decide how to enable dependency builds, but `pnpm --version` reported nothing") + } + version, err := semver.NewVersion(probe) + if err != nil { + return nil, fmt.Errorf("--build needs the pnpm version to decide how to enable dependency builds, but `pnpm --version` reported %q: %w", probe, err) + } + if version.Major() > allowAllBuildsMajor || + (version.Major() == allowAllBuildsMajor && version.Minor() >= allowAllBuildsMinor) { + return []string{allowAllBuilds}, nil + } + // Before 10.6, dropping --ignore-scripts is enough on its own. + return nil, nil +} diff --git a/deps/manager/pnpm/warm_test.go b/deps/manager/pnpm/warm_test.go new file mode 100644 index 0000000..2f95c20 --- /dev/null +++ b/deps/manager/pnpm/warm_test.go @@ -0,0 +1,118 @@ +package pnpm + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +func TestSteps(t *testing.T) { + cases := []struct { + name string + build bool + verif bool + probe string + want []string + }{ + { + name: "download suppresses lifecycle scripts", probe: "10.7.0", + want: []string{ + "manifest: write package.json", + "download: exec pnpm install --ignore-scripts", + }, + }, + { + name: "verify replays offline against the frozen lockfile", verif: true, probe: "10.7.0", + want: []string{ + "manifest: write package.json", + "download: exec pnpm install --ignore-scripts", + "clean: remove node_modules", + "verify: exec pnpm install --offline --frozen-lockfile", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: tc.build, Verify: tc.verif, + }, tc.probe) + if err != nil { + t.Fatal(err) + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(tc.want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, tc.want) + } + }) + } +} + +// pnpm 10.6 stopped running dependency lifecycle scripts even without +// --ignore-scripts, so on those versions building needs an explicit allowlist +// flag that older pnpm does not recognise. +func TestBuildFlagIsGatedOnTheProbedVersion(t *testing.T) { + cases := []struct { + version string + want string + }{ + {version: "9.12.0", want: "download: exec pnpm install"}, + {version: "10.5.9", want: "download: exec pnpm install"}, + {version: "10.6.0", want: "download: exec pnpm install --config.dangerouslyAllowAllBuilds=true"}, + {version: "10.7.1", want: "download: exec pnpm install --config.dangerouslyAllowAllBuilds=true"}, + {version: "11.0.0", want: "download: exec pnpm install --config.dangerouslyAllowAllBuilds=true"}, + } + for _, tc := range cases { + t.Run(tc.version, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: true, + }, tc.version) + if err != nil { + t.Fatal(err) + } + got := manifest.FormatSteps(steps) + if len(got) != 2 || got[1] != tc.want { + t.Fatalf("pnpm %s download step = %q, want %q", tc.version, got[len(got)-1], tc.want) + } + }) + } +} + +// Guessing the flag set would either skip the builds the user asked for or pass +// an argument older pnpm rejects, so an unreadable probe is a hard failure. +func TestBuildRejectsAnUnreadableProbe(t *testing.T) { + for _, probe := range []string{"", "not-a-version"} { + if _, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: true, + }, probe); err == nil { + t.Errorf("probe %q: expected an error when --build cannot determine the pnpm version", probe) + } + } +} + +// Without --build the version is irrelevant, so a missing probe must not block a +// plain warm. +func TestDownloadIgnoresTheProbe(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", + }, "") + if err != nil { + t.Fatal(err) + } + want := []string{"manifest: write package.json", "download: exec pnpm install --ignore-scripts"} + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestWarmerIdentity(t *testing.T) { + if got := (Warmer{}).Manager(); got != manifest.ManagerPNPM { + t.Errorf("Manager() = %q, want %q", got, manifest.ManagerPNPM) + } + probe := (Warmer{}).Probe() + if probe == nil { + t.Fatal("Probe() = nil, want pnpm --version: the build flag depends on it") + } + if got := probe.String(); got != "pnpm --version" { + t.Errorf("Probe() = %q, want %q", got, "pnpm --version") + } +} diff --git a/deps/manifest/command.go b/deps/manifest/command.go new file mode 100644 index 0000000..19f460c --- /dev/null +++ b/deps/manifest/command.go @@ -0,0 +1,46 @@ +package manifest + +import ( + "context" + "os" + "os/exec" + "strings" +) + +type Command struct { + Dir string + Name string + Args []string + Env []string +} + +// String renders the command as it would be typed, so failures can name the +// exact invocation to rerun. +func (c Command) String() string { + return strings.Join(append([]string{c.Name}, c.Args...), " ") +} + +type CommandResult struct { + Stdout string + Stderr string +} + +type CommandRunner interface { + Run(ctx context.Context, cmd Command) (CommandResult, error) +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, spec Command) (CommandResult, error) { + cmd := exec.CommandContext(ctx, spec.Name, spec.Args...) + cmd.Dir = spec.Dir + if len(spec.Env) > 0 { + cmd.Env = append(os.Environ(), spec.Env...) + } + out, err := cmd.Output() + result := CommandResult{Stdout: string(out)} + if exit, ok := err.(*exec.ExitError); ok { + result.Stderr = string(exit.Stderr) + } + return result, err +} diff --git a/deps/manifest/manager.go b/deps/manifest/manager.go new file mode 100644 index 0000000..edf4716 --- /dev/null +++ b/deps/manifest/manager.go @@ -0,0 +1,17 @@ +// Package manifest holds the dependency-manager taxonomy and the process +// execution seam shared between the deps orchestrator and the per-manager +// packages under deps/manager. It is a leaf: nothing here imports deps, so the +// manager packages can depend on it without creating an import cycle. +package manifest + +type Manager string + +const ( + ManagerGo Manager = "go" + ManagerMaven Manager = "maven" + ManagerGradle Manager = "gradle" + ManagerNPM Manager = "npm" + ManagerPNPM Manager = "pnpm" + ManagerImage Manager = "image" + ManagerHelm Manager = "helm" +) diff --git a/deps/manifest/warm.go b/deps/manifest/warm.go new file mode 100644 index 0000000..1011239 --- /dev/null +++ b/deps/manifest/warm.go @@ -0,0 +1,82 @@ +package manifest + +// WarmRequest describes one dependency to warm into the machine's shared +// package cache. Dir is a scratch project the orchestrator has already created; +// a Warmer only decides what to run inside it. +type WarmRequest struct { + Dir string + Name string + Version string + Build bool + Verify bool +} + +type StepKind string + +const ( + StepExec StepKind = "exec" + StepWrite StepKind = "write" + StepRemove StepKind = "remove" +) + +// Step is one unit of warming work. Warming is not purely exec: node ecosystems +// need a package.json written before installing, and need node_modules removed +// before an offline replay can prove anything. +type Step struct { + Kind StepKind + Name string + Command Command // StepExec + Path string // StepWrite, StepRemove — relative to WarmRequest.Dir + Content []byte // StepWrite +} + +// Detail renders just the action, with no step name, for callers that already +// report the name separately. +func (s Step) Detail() string { + switch s.Kind { + case StepExec: + return s.Command.String() + case StepWrite: + return "write " + s.Path + case StepRemove: + return "remove " + s.Path + default: + return "unknown step kind " + string(s.Kind) + } +} + +// String renders the step as "name: kind detail", so a failure can name what was +// being attempted and tests can assert a whole sequence in one comparison. +func (s Step) String() string { + if s.Kind == StepExec { + return s.Name + ": exec " + s.Detail() + } + return s.Name + ": " + s.Detail() +} + +// FormatSteps renders a sequence one line per step. +func FormatSteps(steps []Step) []string { + out := make([]string, 0, len(steps)) + for _, step := range steps { + out = append(out, step.String()) + } + return out +} + +// Warmer decides which commands warm one ecosystem. Implementations do no I/O: +// Steps is a pure function of its arguments, which is what lets the per-manager +// packages be tested by comparing argv without a toolchain, a network, or a +// process. +// +// Probe is the escape hatch for a manager that genuinely needs runtime +// information (pnpm's lifecycle-script policy changed in 10.6). The orchestrator +// runs it and feeds the trimmed stdout back into Steps as an ordinary input, +// keeping Steps pure. +type Warmer interface { + Manager() Manager + // Binary is the executable that must be on PATH, so the orchestrator can + // fail before it creates a scratch directory. + Binary() string + Probe() *Command + Steps(req WarmRequest, probe string) ([]Step, error) +} diff --git a/deps/model.go b/deps/model.go index b57ea48..87d8e50 100644 --- a/deps/model.go +++ b/deps/model.go @@ -1,17 +1,25 @@ package deps -import "time" +import ( + "time" -type Manager string + "github.com/flanksource/repomap/deps/manifest" +) + +// Manager is defined in deps/manifest so the per-manager packages under +// deps/manager can name it without importing deps. It stays an alias here, and +// the constants stay re-declared, so every existing switch and caller is +// unaffected; the underlying type is still string, so JSON is unchanged. +type Manager = manifest.Manager const ( - ManagerGo Manager = "go" - ManagerMaven Manager = "maven" - ManagerGradle Manager = "gradle" - ManagerNPM Manager = "npm" - ManagerPNPM Manager = "pnpm" - ManagerImage Manager = "image" - ManagerHelm Manager = "helm" + ManagerGo = manifest.ManagerGo + ManagerMaven = manifest.ManagerMaven + ManagerGradle = manifest.ManagerGradle + ManagerNPM = manifest.ManagerNPM + ManagerPNPM = manifest.ManagerPNPM + ManagerImage = manifest.ManagerImage + ManagerHelm = manifest.ManagerHelm ) type Mode string diff --git a/deps/runner.go b/deps/runner.go index f24f607..dc3f26f 100644 --- a/deps/runner.go +++ b/deps/runner.go @@ -1,39 +1,13 @@ package deps -import ( - "context" - "os" - "os/exec" +import "github.com/flanksource/repomap/deps/manifest" + +// The process execution seam lives in deps/manifest so the per-manager packages +// under deps/manager can declare commands without importing deps. These aliases +// keep it spelled deps.Command / deps.CommandRunner for every existing caller. +type ( + Command = manifest.Command + CommandResult = manifest.CommandResult + CommandRunner = manifest.CommandRunner + ExecRunner = manifest.ExecRunner ) - -type Command struct { - Dir string - Name string - Args []string - Env []string -} - -type CommandResult struct { - Stdout string - Stderr string -} - -type CommandRunner interface { - Run(ctx context.Context, cmd Command) (CommandResult, error) -} - -type ExecRunner struct{} - -func (ExecRunner) Run(ctx context.Context, spec Command) (CommandResult, error) { - cmd := exec.CommandContext(ctx, spec.Name, spec.Args...) - cmd.Dir = spec.Dir - if len(spec.Env) > 0 { - cmd.Env = append(os.Environ(), spec.Env...) - } - out, err := cmd.Output() - result := CommandResult{Stdout: string(out)} - if exit, ok := err.(*exec.ExitError); ok { - result.Stderr = string(exit.Stderr) - } - return result, err -} diff --git a/deps/update_test.go b/deps/update_test.go index 3b02840..e1f7598 100644 --- a/deps/update_test.go +++ b/deps/update_test.go @@ -595,6 +595,10 @@ type updateFakeRunner struct { responses map[string]CommandResult errors map[string]error commands []Command + // succeedByDefault makes an unlisted command succeed with empty output, + // for callers that assert on which commands ran rather than what they + // printed. Left false, an unlisted command is an error. + succeedByDefault bool } func (r *updateFakeRunner) Run(_ context.Context, cmd Command) (CommandResult, error) { @@ -610,9 +614,23 @@ func (r *updateFakeRunner) Run(_ context.Context, cmd Command) (CommandResult, e return result, nil } } + if r.succeedByDefault { + return CommandResult{}, nil + } return CommandResult{}, errors.New("unexpected command: " + key) } +// ran returns each command as it would have been typed, in the order received. +func (r *updateFakeRunner) ran() []string { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]string, 0, len(r.commands)) + for _, cmd := range r.commands { + out = append(out, cmd.String()) + } + return out +} + func updateCandidateLabels(candidates []UpdateCandidate) []string { labels := make([]string, 0, len(candidates)) for _, candidate := range candidates { From b668a8a72db99857be566a24aa59017740ca3b26 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Fri, 7 Aug 2026 11:11:57 +0300 Subject: [PATCH 11/13] refactor(imageupdate): use go-containerregistry for image metadata Replace Argo-specific registry APIs with go-containerregistry to simplify image metadata resolution while preserving Docker credential authentication and OCI label discovery Claude-Session-Id: 86a07da7-fd6f-46e2-bcbd-abaa7cadf735 --- cmd/repomap/images.go | 1 - cmd/repomap/images_test_helpers.go | 21 +++++++----- imageupdate/labels.go | 55 +++++++++++++++++++----------- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/cmd/repomap/images.go b/cmd/repomap/images.go index ae86b30..b595b7f 100644 --- a/cmd/repomap/images.go +++ b/cmd/repomap/images.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/flanksource/commons/collections" "github.com/spf13/cobra" "github.com/flanksource/repomap" diff --git a/cmd/repomap/images_test_helpers.go b/cmd/repomap/images_test_helpers.go index 700b081..1648cc9 100644 --- a/cmd/repomap/images_test_helpers.go +++ b/cmd/repomap/images_test_helpers.go @@ -7,11 +7,6 @@ import ( "path/filepath" "testing" - "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/image" - "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/registry" - "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/registry/mocks" - "github.com/stretchr/testify/mock" - "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" ) @@ -29,12 +24,20 @@ spec: image: nginx:1.25.3 # keep me ` +type fakeRegistryClient struct{ tags []string } + +func (f fakeRegistryClient) Tags(context.Context) ([]string, error) { + return f.tags, nil +} + +func (f fakeRegistryClient) Digest(context.Context, string) (string, error) { + return "", nil +} + func fakeImageResolver(tags []string) *imageupdate.Resolver { return &imageupdate.Resolver{ - NewRegistryClient: func(ctx context.Context, img *image.ContainerImage) (registry.RegistryClient, error) { - m := &mocks.RegistryClient{} - m.On("Tags", mock.Anything).Return(tags, nil) - return m, nil + NewRegistryClient: func(context.Context, *imageupdate.ContainerImage) (imageupdate.RegistryClient, error) { + return fakeRegistryClient{tags: tags}, nil }, } } diff --git a/imageupdate/labels.go b/imageupdate/labels.go index 04e8218..1b6a3ea 100644 --- a/imageupdate/labels.go +++ b/imageupdate/labels.go @@ -4,43 +4,58 @@ import ( "context" "fmt" - "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/image" - "github.com/argoproj-labs/argocd-image-updater/registry-scanner/pkg/options" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" ) -// LabelResolver reads an image's config digest and OCI labels from its registry, -// authenticating via the local Docker credential store. It is used by the deps -// recursion to discover an image's source repository +// LabelResolver reads an image's manifest digest and OCI labels from its +// registry, authenticating via the local Docker credential store. It is used by +// the deps recursion to discover an image's source repository // (org.opencontainers.image.source) and declared base image // (org.opencontainers.image.base.name/base.digest). type LabelResolver struct { - newClient RegistryClientFactory + creds CredentialResolver } -// NewLabelResolver returns a LabelResolver wired to the live registry client. +// NewLabelResolver returns a LabelResolver wired to the local Docker keychain. func NewLabelResolver() *LabelResolver { - return &LabelResolver{newClient: liveRegistryClientFactory(NewKeychainResolver())} + return &LabelResolver{creds: NewKeychainResolver()} } -// Labels resolves the config digest and OCI labels for an image reference of the -// form registry/repo:tag[@digest]. +// Labels resolves the manifest digest and OCI labels for an image reference of +// the form registry/repo:tag[@digest]. func (l *LabelResolver) Labels(ctx context.Context, ref string) (digest string, labels map[string]string, err error) { - img := image.NewFromIdentifier(ref) - client, err := l.newClient(ctx, img) - if err != nil { - return "", nil, err - } + img := NewContainerImage(ref) tagName := "latest" if img.ImageTag != nil && img.ImageTag.TagName != "" { tagName = img.ImageTag.TagName } - manifest, err := client.ManifestForTag(ctx, tagName) + tag, err := name.NewTag(img.GetFullNameWithoutTag() + ":" + tagName) + if err != nil { + return "", nil, fmt.Errorf("parse reference %s:%s: %w", img.GetFullNameWithoutTag(), tagName, err) + } + + user, pass, err := l.creds.Resolve(ctx, tag.RegistryStr()) + if err != nil { + return "", nil, err + } + var auth authn.Authenticator = authn.Anonymous + if user != "" || pass != "" { + auth = authn.FromConfig(authn.AuthConfig{Username: user, Password: pass}) + } + + remoteImage, err := remote.Image(tag, remote.WithContext(ctx), remote.WithAuth(auth)) + if err != nil { + return "", nil, fmt.Errorf("manifest for %s: %w", tag, err) + } + manifestDigest, err := remoteImage.Digest() if err != nil { - return "", nil, fmt.Errorf("manifest for %s:%s: %w", img.GetFullNameWithoutTag(), tagName, err) + return "", nil, fmt.Errorf("digest for %s: %w", tag, err) } - info, err := client.TagMetadata(ctx, manifest, options.NewManifestOptions()) + config, err := remoteImage.ConfigFile() if err != nil { - return "", nil, fmt.Errorf("metadata for %s:%s: %w", img.GetFullNameWithoutTag(), tagName, err) + return "", nil, fmt.Errorf("config for %s: %w", tag, err) } - return info.EncodedDigest(), info.Labels, nil + return manifestDigest.String(), config.Config.Labels, nil } From 5665d702038f3bcd9b42a227ba8aca45a5298f66 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Fri, 7 Aug 2026 11:11:57 +0300 Subject: [PATCH 12/13] chore: update generated and lock files Claude-Session-Id: 86a07da7-fd6f-46e2-bcbd-abaa7cadf735 --- go.mod | 14 ++++++-------- go.sum | 42 ++++++++++++++++-------------------------- 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index a67a082..be5c7b2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/flanksource/repomap go 1.26.1 require ( - github.com/Masterminds/semver/v3 v3.4.0 + github.com/Masterminds/semver/v3 v3.5.0 github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/flanksource/clicky v1.21.48 @@ -17,6 +17,7 @@ require ( github.com/spf13/cobra v1.10.2 golang.org/x/mod v0.36.0 golang.org/x/sync v0.21.0 + helm.sh/helm/v3 v3.21.3 ) require ( @@ -161,14 +162,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect - golang.org/x/image v0.41.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.46.0 // indirect @@ -181,10 +179,10 @@ require ( gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.36.1 // indirect - k8s.io/apiextensions-apiserver v0.36.1 // indirect - k8s.io/apimachinery v0.36.1 // indirect - k8s.io/client-go v0.36.1 // indirect + k8s.io/api v0.36.2 // indirect + k8s.io/apiextensions-apiserver v0.36.2 // indirect + k8s.io/apimachinery v0.36.2 // indirect + k8s.io/client-go v0.36.2 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect diff --git a/go.sum b/go.sum index 6de3cb7..324f698 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,14 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw= @@ -127,24 +129,10 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= -github.com/flanksource/clicky v1.21.14 h1:72Ax9EAzeu/SEQiR9pgvUmpfj54W/hM+0VXTv5Qdno4= -github.com/flanksource/clicky v1.21.14/go.mod h1:+JLpose/DqOYsXWdjF+J2/rAZ7PbThzAGXWm2GUPA1s= -github.com/flanksource/clicky v1.21.40 h1:96JqmjBSNe3nTRM43KBLrAXLE2e5OIK7X4uy9Dub8Co= -github.com/flanksource/clicky v1.21.40/go.mod h1:wyMaDJleu6wHw6jgg3/BfHFXY5JSsyGG6X1vnFRLfHc= -github.com/flanksource/clicky v1.21.42 h1:MC0uklJWq4KDg2Inq9OxT4IE3SJVqVA3Zio8DftSvK0= -github.com/flanksource/clicky v1.21.42/go.mod h1:wxgvRVic4B8cWCxQ7EO57Q6zBnV49VUoGJ0i9SMdVt4= -github.com/flanksource/clicky v1.21.43 h1:xQxHUDyKQdRFTQfhGAk5G6j2r9Du2Qq1Gfn+T7dGs98= -github.com/flanksource/clicky v1.21.43/go.mod h1:wxgvRVic4B8cWCxQ7EO57Q6zBnV49VUoGJ0i9SMdVt4= -github.com/flanksource/clicky v1.21.44 h1:EwDaWXOPF7Tu5UeL/TE168n89LZqjRMUptcg1kw6884= -github.com/flanksource/clicky v1.21.44/go.mod h1:wxgvRVic4B8cWCxQ7EO57Q6zBnV49VUoGJ0i9SMdVt4= github.com/flanksource/clicky v1.21.48 h1:yCbCk6rDEgtxdknuQV6uZLsvVqCPGYNO7dHuLgbtOp4= github.com/flanksource/clicky v1.21.48/go.mod h1:2gAph3Uy90AYirGmkEP/D4okiFZlrH2XdcoXf69oZHY= -github.com/flanksource/commons v1.51.3 h1:sgQZ2s0XJTub4qmIlzRyH+eYXJP6UXmreCataP9mE7E= -github.com/flanksource/commons v1.51.3/go.mod h1:BxXJzAsRxsw0la7Y/ShEABa8ZbtGIdRi7PCRjiHDCJE= github.com/flanksource/commons v1.53.1 h1:WiMvY9XGG//L4ndYKTcmp0NjWXg4B7Wbn4hYWkITUmY= github.com/flanksource/commons v1.53.1/go.mod h1:ZII22jIDJ3fd/Mz7l0SzLFEv8aoSUg+xwfJAAVf8up4= -github.com/flanksource/gomplate/v3 v3.24.82 h1:22HOZYeNRMX40G8OF3qRAqRoR5Lkf8Vm4x3WDqugZkE= -github.com/flanksource/gomplate/v3 v3.24.82/go.mod h1:NMMZkFsjbLy/8iY8Fip5N86Y0PP6lZeq+kmPwpVVIL0= github.com/flanksource/gomplate/v3 v3.24.84 h1:UOE0yCJsczTIKRaHUvhD6tjCYrbNvOugAizuy0FVlhE= github.com/flanksource/gomplate/v3 v3.24.84/go.mod h1:NMMZkFsjbLy/8iY8Fip5N86Y0PP6lZeq+kmPwpVVIL0= github.com/flanksource/is-healthy v1.0.88 h1:ATQuKoNdp8Qfzf41/eMFajmT0qzOmZlZNG5eLK41RFo= @@ -392,8 +380,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 h1:s/1iRkCKDfhlh1JF26knRneorus8aOwVIDhvYx9WoDw= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0/go.mod h1:UI3wi0FXg1Pofb8ZBiBLhtMzgoTm1TYkMvn71fAqDzs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= @@ -530,15 +518,17 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +helm.sh/helm/v3 v3.21.3 h1:wkamdwI3liEkW6wI1l9aGqQZGxcTKyt8kx0qJLPcmCg= +helm.sh/helm/v3 v3.21.3/go.mod h1:iaJ0iNsPoTZl++7h6vzQFyT0VEVtLYJiyRBDkPOOBTs= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= -k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= -k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= -k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= -k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= -k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= -k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= -k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= +k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= +k8s.io/apiextensions-apiserver v0.36.2 h1:3O5gqOj/dt2XWWbpMe+TXWpE9yU6pjM/tXxtHHJT/K4= +k8s.io/apiextensions-apiserver v0.36.2/go.mod h1:cL1tBWe8XSaP1H30iWKGo7hf6iAUUUJPEU70dskmAnA= +k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= +k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= +k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= +k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= From 4bd659687062188c3bfddcda717b9ad8c672a3d2 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 9 Aug 2026 10:26:22 +0300 Subject: [PATCH 13/13] feat(deps): normalize Go dependency specs before warming Accept GitHub slugs and repository URLs for Go cache warming by canonicalising inputs before invoking the toolchain. Preserve the user's original spec while reporting the canonical module identity, and share dependency-spec parsing across managers. BREAKING CHANGE: manifest.Warmer implementations must implement NormalizeSpec --- README.md | 8 ++- cmd/repomap/cache_warm.go | 6 ++ deps/cachewarm.go | 56 ++++++--------- deps/cachewarm_plan.go | 5 +- deps/cachewarm_test.go | 119 ++++++++++++++++++++------------ deps/manager/gomod/warm.go | 61 ++++++++++++++++ deps/manager/gomod/warm_test.go | 63 +++++++++++++++++ deps/manager/npm/warm.go | 4 ++ deps/manager/pnpm/warm.go | 4 ++ deps/manifest/warm.go | 33 +++++++++ deps/manifest/warm_test.go | 47 +++++++++++++ 11 files changed, 323 insertions(+), 83 deletions(-) create mode 100644 deps/manifest/warm_test.go diff --git a/README.md b/README.md index c3333ae..f377903 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,10 @@ repomap cache-warm go github.com/flanksource/clicky@v1.21.14 # Take the current version, compile every package, and prove it works offline repomap cache-warm go github.com/flanksource/commons --build --verify +# A Go dependency may also be a GitHub slug or a repository URL +repomap cache-warm go flanksource/commons +repomap cache-warm go https://github.com/flanksource/commons + # Warm several npm packages into the pnpm store repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0 ``` @@ -117,7 +121,9 @@ closure into the machine's shared cache, then deletes the project. Nothing in th working tree is touched; what persists is the warmed cache (`GOMODCACHE`, the pnpm store, or the npm cache). Omit the version to take whatever the manager considers current — the concrete resolved version is reported back, including Go -pseudo-versions. +pseudo-versions. A Go dependency can be named as a module path, a GitHub +`owner/repo` slug, or a repository URL; all three are canonicalised to the module +path before the go toolchain sees them. `--build` goes further than downloading. For Go it compiles every package in the module so `GOCACHE` holds build artifacts and not just source. For npm and pnpm it diff --git a/cmd/repomap/cache_warm.go b/cmd/repomap/cache_warm.go index 4d83c81..5c5e72d 100644 --- a/cmd/repomap/cache_warm.go +++ b/cmd/repomap/cache_warm.go @@ -32,6 +32,10 @@ the warmed cache (GOMODCACHE, the pnpm store, or the npm cache). Omit the version to take whatever the manager considers current. The concrete resolved version is reported back, including Go pseudo-versions. +A Go dependency can be given as a module path, as a GitHub owner/repo slug, or as +a repository URL — all three are canonicalised to the module path before the go +toolchain sees them. + Use --build to go further than downloading. For Go it compiles every package in the module so GOCACHE holds the build artifacts, not just the source. For npm and pnpm it lets dependency lifecycle scripts run so native addons are compiled. It @@ -47,6 +51,8 @@ must succeed with no network access. EXAMPLES: repomap cache-warm go github.com/flanksource/clicky@v1.21.14 repomap cache-warm go github.com/flanksource/commons --build --verify + repomap cache-warm go flanksource/commons + repomap cache-warm go https://github.com/flanksource/commons repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0 repomap cache-warm npm @flanksource/icons@1.0.0 --verify repomap cache-warm go github.com/flanksource/clicky@v1.21.14 --json`) diff --git a/deps/cachewarm.go b/deps/cachewarm.go index 9b9b128..f940bbb 100644 --- a/deps/cachewarm.go +++ b/deps/cachewarm.go @@ -50,16 +50,21 @@ type WarmStep struct { type WarmResult struct { Manager Manager `json:"manager"` - // Spec is the requested name@version; Version is what the manager resolved. - Spec string `json:"spec"` - Name string `json:"name"` - Version string `json:"version,omitempty"` - Packages int `json:"packages,omitempty"` - Cache string `json:"cache,omitempty"` - Built bool `json:"built,omitempty"` - Verified bool `json:"verified,omitempty"` - Steps []WarmStep `json:"steps,omitempty"` - Error string `json:"error,omitempty"` + // Spec is the requested name@version, verbatim; Name is the manager's + // canonical form of it, and Version is what the manager resolved. + Spec string `json:"spec"` + Name string `json:"name"` + // RequestedVersion is the version the spec asked for ("latest" when it was + // omitted), kept so output has something to show when the read-back could not + // determine a concrete one. + RequestedVersion string `json:"requested_version,omitempty"` + Version string `json:"version,omitempty"` + Packages int `json:"packages,omitempty"` + Cache string `json:"cache,omitempty"` + Built bool `json:"built,omitempty"` + Verified bool `json:"verified,omitempty"` + Steps []WarmStep `json:"steps,omitempty"` + Error string `json:"error,omitempty"` // SummaryError records a failure to read back what was warmed. The cache is // still warm when this is set, so it does not fail the spec — but it is // reported rather than swallowed. @@ -128,12 +133,18 @@ func WarmCache(ctx context.Context, opts WarmOptions) ([]WarmResult, error) { func warmSpec(ctx context.Context, warmer manifest.Warmer, runner CommandRunner, spec string, opts WarmOptions, tk *task.Task) WarmResult { result := WarmResult{Manager: warmer.Manager(), Spec: spec} - name, version, err := parseWarmSpec(spec) + normalized, err := warmer.NormalizeSpec(spec) + if err != nil { + result.Error = err.Error() + return result + } + name, version, err := manifest.SplitSpec(normalized) if err != nil { result.Error = err.Error() return result } result.Name = name + result.RequestedVersion = version dir, err := os.MkdirTemp("", "repomap-cache-warm-*") if err != nil { @@ -305,26 +316,3 @@ func warmCachePath(ctx context.Context, manager Manager, runner CommandRunner, d } return strings.TrimSpace(result.Stdout) } - -// parseWarmSpec splits "name@version". The split uses the last @ at a non-zero -// index so a scoped npm name such as @scope/pkg keeps its leading @. An omitted -// version becomes "latest" and the manager decides what that means; the concrete -// version is read back after warming. -func parseWarmSpec(spec string) (name, version string, err error) { - spec = strings.TrimSpace(spec) - if spec == "" { - return "", "", fmt.Errorf("empty dependency spec: expected name@version") - } - at := strings.LastIndex(spec, "@") - if at <= 0 { - if spec == "@" { - return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) - } - return spec, "latest", nil - } - name, version = spec[:at], spec[at+1:] - if name == "" || version == "" { - return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) - } - return name, version, nil -} diff --git a/deps/cachewarm_plan.go b/deps/cachewarm_plan.go index 4847bac..43db1a3 100644 --- a/deps/cachewarm_plan.go +++ b/deps/cachewarm_plan.go @@ -83,8 +83,5 @@ func (r WarmResult) displayVersion() string { if r.Version != "" { return r.Version } - if _, version, err := parseWarmSpec(r.Spec); err == nil { - return version - } - return "" + return r.RequestedVersion } diff --git a/deps/cachewarm_test.go b/deps/cachewarm_test.go index a1c9ca8..623b97a 100644 --- a/deps/cachewarm_test.go +++ b/deps/cachewarm_test.go @@ -7,50 +7,6 @@ import ( "testing" ) -func TestParseWarmSpec(t *testing.T) { - cases := []struct { - spec string - wantName string - wantVersion string - wantErr bool - }{ - {spec: "github.com/acme/lib@v1.2.3", wantName: "github.com/acme/lib", wantVersion: "v1.2.3"}, - // No version means "whatever the manager considers current"; the concrete - // version is read back after warming. - {spec: "github.com/acme/lib", wantName: "github.com/acme/lib", wantVersion: "latest"}, - {spec: "left-pad@1.3.0", wantName: "left-pad", wantVersion: "1.3.0"}, - {spec: "left-pad@^1.3.0", wantName: "left-pad", wantVersion: "^1.3.0"}, - {spec: "left-pad@latest", wantName: "left-pad", wantVersion: "latest"}, - // A scoped npm name leads with @, so splitting must use the last @ and - // ignore one at index 0. - {spec: "@scope/pkg@1.0.0", wantName: "@scope/pkg", wantVersion: "1.0.0"}, - {spec: "@scope/pkg", wantName: "@scope/pkg", wantVersion: "latest"}, - // A Go branch or commit reference must survive untouched. - {spec: "github.com/acme/lib@main", wantName: "github.com/acme/lib", wantVersion: "main"}, - {spec: "", wantErr: true}, - {spec: " ", wantErr: true}, - {spec: "left-pad@", wantErr: true}, - {spec: "@", wantErr: true}, - } - for _, tc := range cases { - t.Run(tc.spec, func(t *testing.T) { - name, version, err := parseWarmSpec(tc.spec) - if tc.wantErr { - if err == nil { - t.Fatalf("parseWarmSpec(%q) = (%q, %q), want an error", tc.spec, name, version) - } - return - } - if err != nil { - t.Fatal(err) - } - if name != tc.wantName || version != tc.wantVersion { - t.Fatalf("parseWarmSpec(%q) = (%q, %q), want (%q, %q)", tc.spec, name, version, tc.wantName, tc.wantVersion) - } - }) - } -} - // pnpm nests dependencies under a synthetic importer node, so counting // root.Children would report the importer instead of the packages, and the // version lookup would miss the target entirely. @@ -251,6 +207,81 @@ func TestWarmCacheWarmsEverySpec(t *testing.T) { } } +// A GitHub slug and a repository URL are what a user has to hand, and both used +// to reach `go get` verbatim and fail with the toolchain's "malformed module +// path". +func TestWarmCacheNormalisesGoRepoShorthand(t *testing.T) { + for _, spec := range []string{ + "flanksource/commons", + "https://github.com/flanksource/commons", + "git@github.com:flanksource/commons.git", + } { + t.Run(spec, func(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{spec}, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if results[0].Name != "github.com/flanksource/commons" { + t.Errorf("Name = %q, want the canonical module path", results[0].Name) + } + if results[0].Spec != spec { + t.Errorf("Spec = %q, should stay the spec the user typed", results[0].Spec) + } + want := []string{ + "go mod init repomap.local/cachewarm", + "go get github.com/flanksource/commons/...@latest", + "go mod download all", + "go env GOMODCACHE", + } + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } + }) + } +} + +// A spec with no host to infer is repomap's to reject, before a scratch project +// is created or the go toolchain is asked anything. +func TestWarmCacheRejectsAGoSpecThatIsNotAModulePath(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"commons"}, + Runner: runner, + }) + if err == nil { + t.Fatal("expected an error: \"commons\" names no host and cannot become a module path") + } + if !strings.Contains(results[0].Error, "github.com/owner/repo") { + t.Errorf("error %q should say what a module path looks like", results[0].Error) + } + if got := runner.ran(); len(got) != 0 { + t.Errorf("no command should run for an unusable spec, got %v", got) + } +} + +// npm names are already registry names, and a scoped one must not be mistaken +// for a repo shorthand. +func TestWarmCacheLeavesScopedNPMNamesAlone(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerNPM, + Specs: []string{"@scope/pkg@1.0.0"}, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if results[0].Name != "@scope/pkg" || results[0].RequestedVersion != "1.0.0" { + t.Fatalf("result identity = %q@%q, want @scope/pkg@1.0.0", results[0].Name, results[0].RequestedVersion) + } +} + // A bad spec must be reported against that spec rather than aborting the run // before the others are warmed. func TestWarmCacheReportsABadSpecWithoutSkippingTheRest(t *testing.T) { diff --git a/deps/manager/gomod/warm.go b/deps/manager/gomod/warm.go index 11ad1e3..4a4d92d 100644 --- a/deps/manager/gomod/warm.go +++ b/deps/manager/gomod/warm.go @@ -6,6 +6,9 @@ package gomod import ( "fmt" + "strings" + + "golang.org/x/mod/module" "github.com/flanksource/repomap/deps/manifest" ) @@ -14,6 +17,14 @@ import ( // published, but it must be a valid module path for `go mod init` to accept it. const ModulePath = "repomap.local/cachewarm" +// DefaultHost is prepended to a spec whose first path element carries no dot, +// which is what a bare owner/repo shorthand looks like. +const DefaultHost = "github.com" + +// schemes are stripped from a spec typed as a URL. Only the scheme goes: the +// rest of a repository URL already reads as a module path. +var schemes = []string{"https://", "http://", "ssh://", "git://"} + // Warmer drives the go toolchain against a throwaway module that requires the // target, populating GOMODCACHE and — with Build — GOCACHE. type Warmer struct{} @@ -26,6 +37,56 @@ func (Warmer) Binary() string { return "go" } // releases, so no runtime version check is needed. func (Warmer) Probe() *manifest.Command { return nil } +// NormalizeSpec turns what a user actually has to hand — a GitHub slug, a +// browser URL, a clone URL — into the canonical module path `go get` demands, +// and rejects anything that still is not one. Without it the only diagnostic is +// the go toolchain's own "malformed module path", which never says what repomap +// wanted. +// +// A repository URL is not always a module path (vanity domains, modules nested +// in a monorepo), so this canonicalises the common GitHub shape rather than +// claiming to resolve every repository. +func (Warmer) NormalizeSpec(spec string) (string, error) { + original := strings.TrimSpace(spec) + trimmed := original + for _, scheme := range schemes { + trimmed = strings.TrimPrefix(trimmed, scheme) + } + + name, version, err := manifest.SplitSpec(trimGitURL(trimmed)) + if err != nil { + return "", err + } + name = strings.TrimSuffix(strings.TrimSuffix(name, "/"), ".git") + + // A dotless first element is a bare owner/repo shorthand: no module path can + // start without a host, and every host has a dot. A single element is left + // alone so it fails below rather than becoming github.com/. + if head, _, ok := strings.Cut(name, "/"); ok && !strings.Contains(head, ".") { + name = DefaultHost + "/" + name + } + if err := module.CheckPath(name); err != nil { + return "", fmt.Errorf("%q is not a Go module path (expected github.com/owner/repo or owner/repo): %w", original, err) + } + return name + "@" + version, nil +} + +// trimGitURL rewrites the two clone-URL shapes a module path cannot express: +// scp syntax (git@host:owner/repo) and a leftover userinfo prefix +// (git@host/owner/repo, what stripping ssh:// leaves behind). Both must go +// before the name@version split, which would otherwise read git@ as the version +// separator. A spec with no slash is never a clone URL, so name@version is left +// intact. +func trimGitURL(spec string) string { + head, path, ok := strings.Cut(spec, "/") + at := strings.Index(head, "@") + if !ok || at < 0 { + return spec + } + // In scp syntax the colon plays the role of the first slash. + return strings.Replace(head[at+1:], ":", "/", 1) + "/" + path +} + func (Warmer) Steps(req manifest.WarmRequest, _ string) ([]manifest.Step, error) { switch { case req.Dir == "": diff --git a/deps/manager/gomod/warm_test.go b/deps/manager/gomod/warm_test.go index e0c350a..2e31ea2 100644 --- a/deps/manager/gomod/warm_test.go +++ b/deps/manager/gomod/warm_test.go @@ -149,6 +149,69 @@ func TestStepsRejectsIncompleteRequest(t *testing.T) { } } +func TestNormalizeSpec(t *testing.T) { + const module = "github.com/flanksource/commons" + cases := []struct { + spec string + want string + }{ + // A canonical module path is already what go get wants. + {spec: "github.com/acme/lib@v1.2.3", want: "github.com/acme/lib@v1.2.3"}, + {spec: "github.com/acme/lib", want: "github.com/acme/lib@latest"}, + // A dot in the first element marks it as a host, so a vanity domain and a + // major-version suffix both survive untouched. + {spec: "gopkg.in/yaml.v3", want: "gopkg.in/yaml.v3@latest"}, + {spec: "github.com/acme/lib/v2@v2.1.0", want: "github.com/acme/lib/v2@v2.1.0"}, + // A bare owner/repo slug, the shape copied out of a GitHub page. + {spec: "flanksource/commons", want: module + "@latest"}, + {spec: "flanksource/commons@v1.2.3", want: module + "@v1.2.3"}, + {spec: " flanksource/commons ", want: module + "@latest"}, + // The three URL shapes: browser, https clone, ssh clone. + {spec: "https://github.com/flanksource/commons", want: module + "@latest"}, + {spec: "http://github.com/flanksource/commons/", want: module + "@latest"}, + {spec: "https://github.com/flanksource/commons.git@v1.2.3", want: module + "@v1.2.3"}, + {spec: "git@github.com:flanksource/commons.git", want: module + "@latest"}, + {spec: "ssh://git@github.com/flanksource/commons.git", want: module + "@latest"}, + {spec: "git://github.com/flanksource/commons.git", want: module + "@latest"}, + // A slug on another forge keeps its host rather than gaining github.com. + {spec: "https://gitlab.com/acme/lib@v1.0.0", want: "gitlab.com/acme/lib@v1.0.0"}, + } + for _, tc := range cases { + t.Run(tc.spec, func(t *testing.T) { + got, err := (Warmer{}).NormalizeSpec(tc.spec) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Fatalf("NormalizeSpec(%q) = %q, want %q", tc.spec, got, tc.want) + } + }) + } +} + +func TestNormalizeSpecRejectsWhatCannotBeAModulePath(t *testing.T) { + cases := []struct { + name string + spec string + }{ + // One element names no host, and inventing github.com/commons would warm + // something the user never asked for. + {name: "single element", spec: "commons"}, + {name: "single element with version", spec: "commons@v1.2.3"}, + {name: "port is not a module path", spec: "github.com/acme/lib:8080"}, + {name: "empty", spec: " "}, + {name: "no version after the separator", spec: "flanksource/commons@"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := (Warmer{}).NormalizeSpec(tc.spec) + if err == nil { + t.Fatalf("NormalizeSpec(%q) = %q, want an error", tc.spec, got) + } + }) + } +} + func TestWarmerIdentity(t *testing.T) { if got := (Warmer{}).Manager(); got != manifest.ManagerGo { t.Errorf("Manager() = %q, want %q", got, manifest.ManagerGo) diff --git a/deps/manager/npm/warm.go b/deps/manager/npm/warm.go index c0b487e..94857e1 100644 --- a/deps/manager/npm/warm.go +++ b/deps/manager/npm/warm.go @@ -16,6 +16,10 @@ func (Warmer) Binary() string { return "npm" } // version check is needed. func (Warmer) Probe() *manifest.Command { return nil } +// NormalizeSpec returns the spec verbatim: an npm name is already the name the +// registry knows, and rewriting would only risk mangling a scoped @scope/pkg. +func (Warmer) NormalizeSpec(spec string) (string, error) { return spec, nil } + func (Warmer) Steps(req manifest.WarmRequest, _ string) ([]manifest.Step, error) { return node.Steps(req, node.Commands{ Binary: "npm", diff --git a/deps/manager/pnpm/warm.go b/deps/manager/pnpm/warm.go index c5ddc8f..4664642 100644 --- a/deps/manager/pnpm/warm.go +++ b/deps/manager/pnpm/warm.go @@ -31,6 +31,10 @@ func (Warmer) Probe() *manifest.Command { return &manifest.Command{Name: "pnpm", Args: []string{"--version"}} } +// NormalizeSpec returns the spec verbatim: an npm name is already the name the +// registry knows, and rewriting would only risk mangling a scoped @scope/pkg. +func (Warmer) NormalizeSpec(spec string) (string, error) { return spec, nil } + func (Warmer) Steps(req manifest.WarmRequest, probe string) ([]manifest.Step, error) { cmds := node.Commands{ Binary: "pnpm", diff --git a/deps/manifest/warm.go b/deps/manifest/warm.go index 1011239..6e6df52 100644 --- a/deps/manifest/warm.go +++ b/deps/manifest/warm.go @@ -1,5 +1,33 @@ package manifest +import ( + "fmt" + "strings" +) + +// SplitSpec splits "name@version". The split uses the last @ at a non-zero index +// so a scoped npm name such as @scope/pkg keeps its leading @. An omitted version +// becomes "latest" and the manager decides what that means; the concrete version +// is read back after warming. +func SplitSpec(spec string) (name, version string, err error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return "", "", fmt.Errorf("empty dependency spec: expected name@version") + } + at := strings.LastIndex(spec, "@") + if at <= 0 { + if spec == "@" { + return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) + } + return spec, "latest", nil + } + name, version = spec[:at], spec[at+1:] + if name == "" || version == "" { + return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) + } + return name, version, nil +} + // WarmRequest describes one dependency to warm into the machine's shared // package cache. Dir is a scratch project the orchestrator has already created; // a Warmer only decides what to run inside it. @@ -78,5 +106,10 @@ type Warmer interface { // fail before it creates a scratch directory. Binary() string Probe() *Command + // NormalizeSpec canonicalises a user-supplied spec before it is split into + // name and version, so a manager can accept the shorthand its ecosystem's + // users actually type. Managers that take the spec verbatim return it + // unchanged. + NormalizeSpec(spec string) (string, error) Steps(req WarmRequest, probe string) ([]Step, error) } diff --git a/deps/manifest/warm_test.go b/deps/manifest/warm_test.go new file mode 100644 index 0000000..21bb10b --- /dev/null +++ b/deps/manifest/warm_test.go @@ -0,0 +1,47 @@ +package manifest + +import "testing" + +func TestSplitSpec(t *testing.T) { + cases := []struct { + spec string + wantName string + wantVersion string + wantErr bool + }{ + {spec: "github.com/acme/lib@v1.2.3", wantName: "github.com/acme/lib", wantVersion: "v1.2.3"}, + // No version means "whatever the manager considers current"; the concrete + // version is read back after warming. + {spec: "github.com/acme/lib", wantName: "github.com/acme/lib", wantVersion: "latest"}, + {spec: "left-pad@1.3.0", wantName: "left-pad", wantVersion: "1.3.0"}, + {spec: "left-pad@^1.3.0", wantName: "left-pad", wantVersion: "^1.3.0"}, + {spec: "left-pad@latest", wantName: "left-pad", wantVersion: "latest"}, + // A scoped npm name leads with @, so splitting must use the last @ and + // ignore one at index 0. + {spec: "@scope/pkg@1.0.0", wantName: "@scope/pkg", wantVersion: "1.0.0"}, + {spec: "@scope/pkg", wantName: "@scope/pkg", wantVersion: "latest"}, + // A Go branch or commit reference must survive untouched. + {spec: "github.com/acme/lib@main", wantName: "github.com/acme/lib", wantVersion: "main"}, + {spec: "", wantErr: true}, + {spec: " ", wantErr: true}, + {spec: "left-pad@", wantErr: true}, + {spec: "@", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.spec, func(t *testing.T) { + name, version, err := SplitSpec(tc.spec) + if tc.wantErr { + if err == nil { + t.Fatalf("SplitSpec(%q) = (%q, %q), want an error", tc.spec, name, version) + } + return + } + if err != nil { + t.Fatal(err) + } + if name != tc.wantName || version != tc.wantVersion { + t.Fatalf("SplitSpec(%q) = (%q, %q), want (%q, %q)", tc.spec, name, version, tc.wantName, tc.wantVersion) + } + }) + } +}